1//===-- X86Counter.cpp ------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "X86Counter.h"
10
11#if defined(__linux__) && defined(HAVE_LIBPFM) && \
12 defined(LIBPFM_HAS_FIELD_CYCLES)
13
14// FIXME: Use appropriate wrappers for poll.h and mman.h
15// to support Windows and remove this linux-only guard.
16
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/Support/Endian.h"
19#include "llvm/Support/Errc.h"
20
21#include <perfmon/perf_event.h>
22#include <perfmon/pfmlib.h>
23#include <perfmon/pfmlib_perf_event.h>
24
25#include <atomic>
26#include <chrono>
27#include <cstddef>
28#include <cstdint>
29#include <limits>
30#include <memory>
31
32#include <poll.h>
33#include <sys/mman.h>
34#include <unistd.h>
35
36namespace llvm {
37namespace exegesis {
38
39// Number of entries in the LBR.
40static constexpr int kLbrEntries = 16;
41static constexpr size_t kBufferPages = 8;
42static const size_t kDataBufferSize = kBufferPages * getpagesize();
43
44// First page is reserved for perf_event_mmap_page. Data buffer starts on
45// the next page, so we allocate one more page.
46static const size_t kMappedBufferSize = (kBufferPages + 1) * getpagesize();
47
48static constexpr int kPollTimeoutMs = 1000;
49static constexpr int kMaxPolls = 3;
50
51// Waits for the LBR perf events.
52static int pollLbrPerfEvent(const int FileDescriptor) {
53 struct pollfd PollFd;
54 PollFd.fd = FileDescriptor;
55 PollFd.events = POLLIN;
56 PollFd.revents = 0;
57 return poll(&PollFd, 1 /* num of fds */, kPollTimeoutMs);
58}
59
60// Copies the data-buffer into Buf, given the pointer to MMapped.
61static void copyDataBuffer(void *MMappedBuffer, char *Buf, uint64_t Tail,
62 size_t DataSize) {
63 // First page is reserved for perf_event_mmap_page. Data buffer starts on
64 // the next page.
65 char *Start = reinterpret_cast<char *>(MMappedBuffer) + getpagesize();
66 // The LBR buffer is a cyclic buffer, we copy data to another buffer.
67 uint64_t Offset = Tail % kDataBufferSize;
68 size_t CopySize = kDataBufferSize - Offset;
69 memcpy(Buf, Start + Offset, CopySize);
70 if (CopySize >= DataSize)
71 return;
72
73 memcpy(Buf + CopySize, Start, Offset);
74 return;
75}
76
77// Parses the given data-buffer for stats and fill the CycleArray.
78// If data has been extracted successfully, also modifies the code to jump
79// out the benchmark loop.
80static Error parseDataBuffer(const char *DataBuf, size_t DataSize,
81 const void *From, const void *To,
82 SmallVector<int64_t, 4> *CycleArray) {
83 const char *DataPtr = DataBuf;
84 while (DataPtr < DataBuf + DataSize) {
85 struct perf_event_header Header;
86 memcpy(&Header, DataPtr, sizeof(struct perf_event_header));
87 if (Header.type != PERF_RECORD_SAMPLE) {
88 // Ignores non-sample records.
89 DataPtr += Header.size;
90 continue;
91 }
92 DataPtr += sizeof(Header);
93 uint64_t Count = support::endian::read64(DataPtr, endianness::native);
94 DataPtr += sizeof(Count);
95
96 struct perf_branch_entry Entry;
97 memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));
98
99 // Read the perf_branch_entry array.
100 for (uint64_t i = 0; i < Count; ++i) {
101 const uint64_t BlockStart = From == nullptr
102 ? std::numeric_limits<uint64_t>::min()
103 : reinterpret_cast<uint64_t>(From);
104 const uint64_t BlockEnd = To == nullptr
105 ? std::numeric_limits<uint64_t>::max()
106 : reinterpret_cast<uint64_t>(To);
107
108 if (BlockStart <= Entry.from && BlockEnd >= Entry.to)
109 CycleArray->push_back(Entry.cycles);
110
111 if (i == Count - 1)
112 // We've reached the last entry.
113 return Error::success();
114
115 // Advance to next entry
116 DataPtr += sizeof(Entry);
117 memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));
118 }
119 }
120 return make_error<StringError>("Unable to parse databuffer.", errc::io_error);
121}
122
123X86LbrPerfEvent::X86LbrPerfEvent(unsigned SamplingPeriod) {
124 assert(SamplingPeriod > 0 && "SamplingPeriod must be positive");
125 EventString = "BR_INST_RETIRED.NEAR_TAKEN";
126 Attr = new perf_event_attr();
127 Attr->size = sizeof(*Attr);
128 Attr->type = PERF_TYPE_RAW;
129 // FIXME This is SKL's encoding. Not sure if it'll change.
130 Attr->config = 0x20c4; // BR_INST_RETIRED.NEAR_TAKEN
131 Attr->sample_type = PERF_SAMPLE_BRANCH_STACK;
132 // Don't need to specify "USER" because we've already excluded HV and Kernel.
133 Attr->branch_sample_type = PERF_SAMPLE_BRANCH_ANY;
134 Attr->sample_period = SamplingPeriod;
135 Attr->wakeup_events = 1; // We need this even when using ioctl REFRESH.
136 Attr->disabled = 1;
137 Attr->exclude_kernel = 1;
138 Attr->exclude_hv = 1;
139 Attr->read_format = PERF_FORMAT_GROUP;
140
141 FullQualifiedEventString = EventString;
142}
143
144X86LbrCounter::X86LbrCounter(pfm::PerfEvent &&NewEvent)
145 : CounterGroup(std::move(NewEvent), {}) {
146 MMappedBuffer = mmap(nullptr, kMappedBufferSize, PROT_READ | PROT_WRITE,
147 MAP_SHARED, getFileDescriptor(), 0);
148 if (MMappedBuffer == MAP_FAILED)
149 errs() << "Failed to mmap buffer.";
150}
151
152X86LbrCounter::~X86LbrCounter() {
153 if (0 != munmap(MMappedBuffer, kMappedBufferSize))
154 errs() << "Failed to munmap buffer.";
155}
156
157void X86LbrCounter::start() {
158 ioctl(getFileDescriptor(), PERF_EVENT_IOC_REFRESH, 1024 /* kMaxPollsPerFd */);
159}
160
161Error X86LbrCounter::checkLbrSupport() {
162 // Do a sample read and check if the results contain non-zero values.
163
164 X86LbrCounter counter(X86LbrPerfEvent(123));
165 counter.start();
166
167 // Prevent the compiler from unrolling the loop and get rid of all the
168 // branches. We need at least 16 iterations.
169 int Sum = 0;
170 int V = 1;
171
172 volatile int *P = &V;
173 auto TimeLimit =
174 std::chrono::high_resolution_clock::now() + std::chrono::microseconds(5);
175
176 for (int I = 0;
177 I < kLbrEntries || std::chrono::high_resolution_clock::now() < TimeLimit;
178 ++I) {
179 Sum += *P;
180 }
181
182 counter.stop();
183 (void)Sum;
184
185 // A read that fails just means LBR is unusable here. If there is at least one
186 // non-zero entry, then LBR is supported.
187 if (auto Result = expectedToOptional(counter.doReadCounter(nullptr, nullptr)))
188 if (any_of(*Result, [](int64_t Value) { return Value != 0; }))
189 return Error::success();
190
191 return make_error<StringError>(
192 "LBR format with cycles is not suppported on the host.",
193 errc::not_supported);
194}
195
196Expected<SmallVector<int64_t, 4>>
197X86LbrCounter::readOrError(StringRef FunctionBytes) const {
198 // Disable the event before reading
199 ioctl(getFileDescriptor(), PERF_EVENT_IOC_DISABLE, 0);
200
201 // Find the boundary of the function so that we could filter the LBRs
202 // to keep only the relevant records.
203 if (FunctionBytes.empty())
204 return make_error<StringError>("Empty function bytes",
205 errc::invalid_argument);
206 const void *From = reinterpret_cast<const void *>(FunctionBytes.data());
207 const void *To = reinterpret_cast<const void *>(FunctionBytes.data() +
208 FunctionBytes.size());
209 return doReadCounter(From, To);
210}
211
212Expected<SmallVector<int64_t, 4>>
213X86LbrCounter::doReadCounter(const void *From, const void *To) const {
214 // Parses the LBR buffer and fills CycleArray with the sequence of cycle
215 // counts from the buffer.
216 SmallVector<int64_t, 4> CycleArray;
217 auto DataBuf = std::make_unique<char[]>(kDataBufferSize);
218
219 // The event is disabled before we get here, so a sample either is already in
220 // the ring buffer -- wakeup_events == 1 leaves POLLIN asserted -- or it will
221 // never arrive. The budget only absorbs wake-up latency.
222 int PollResult = 0;
223 for (int I = 0; I != kMaxPolls && PollResult == 0; ++I)
224 PollResult = pollLbrPerfEvent(getFileDescriptor());
225
226 if (PollResult < 0)
227 return make_error<StringError>("Cannot poll LBR perf event.",
228 errc::io_error);
229 if (PollResult == 0)
230 return make_error<StringError>(
231 "LBR polling still timed out after max number of attempts.",
232 errc::device_or_resource_busy);
233
234 struct perf_event_mmap_page Page;
235 memcpy(&Page, MMappedBuffer, sizeof(struct perf_event_mmap_page));
236
237 const uint64_t DataTail = Page.data_tail;
238 const uint64_t DataHead = Page.data_head;
239 // We're supposed to use a barrier after reading data_head.
240 std::atomic_thread_fence(std::memory_order_acq_rel);
241 const size_t DataSize = DataHead - DataTail;
242 if (DataSize > kDataBufferSize)
243 return make_error<StringError>("DataSize larger than buffer size.",
244 errc::invalid_argument);
245
246 copyDataBuffer(MMappedBuffer, DataBuf.get(), DataTail, DataSize);
247 Error error = parseDataBuffer(DataBuf.get(), DataSize, From, To, &CycleArray);
248 if (!error)
249 return CycleArray;
250 return std::move(error);
251}
252
253} // namespace exegesis
254} // namespace llvm
255
256#endif // defined(__linux__) && defined(HAVE_LIBPFM) &&
257 // defined(LIBPFM_HAS_FIELD_CYCLES)
258