1//===-- Assembler.cpp -------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Assembler.h"
10
11#include "SnippetRepetitor.h"
12#include "SubprocessMemory.h"
13#include "Target.h"
14#include "llvm/Analysis/TargetLibraryInfo.h"
15#include "llvm/CodeGen/FunctionLoweringInfo.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/MachineInstrBuilder.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/TargetInstrInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetPassConfig.h"
24#include "llvm/CodeGen/TargetSubtargetInfo.h"
25#include "llvm/ExecutionEngine/Orc/LLJIT.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/LegacyPassManager.h"
29#include "llvm/MC/MCInstrInfo.h"
30#include "llvm/Object/SymbolSize.h"
31#include "llvm/Support/Alignment.h"
32#include "llvm/Support/Error.h"
33#include "llvm/Support/MemoryBuffer.h"
34#include "llvm/Support/raw_ostream.h"
35
36#ifdef HAVE_LIBPFM
37#include "perfmon/perf_event.h"
38#endif // HAVE_LIBPFM
39
40#ifdef __linux__
41#include <unistd.h>
42#endif
43
44namespace llvm {
45namespace exegesis {
46
47static constexpr const char ModuleID[] = "ExegesisInfoTest";
48static constexpr const char FunctionID[] = "foo";
49static const Align kFunctionAlignment(4096);
50
51// Fills the given basic block with register setup code, and returns true if
52// all registers could be setup correctly.
53static bool generateSnippetSetupCode(const ExegesisTarget &ET,
54 const MCSubtargetInfo *const MSI,
55 BasicBlockFiller &BBF,
56 const BenchmarkKey &Key,
57 bool GenerateMemoryInstructions) {
58 bool IsSnippetSetupComplete = true;
59 if (GenerateMemoryInstructions) {
60 BBF.addInstructions(Insts: ET.generateMemoryInitialSetup());
61 for (const MemoryMapping &MM : Key.MemoryMappings) {
62#ifdef __linux__
63 // The frontend that generates that parses the memory mapping information
64 // from the user should validate that the requested address is a multiple
65 // of the page size. Assert that this is true here.
66 assert(MM.Address % getpagesize() == 0 &&
67 "Memory mappings need to be aligned to page boundaries.");
68#endif
69 const MemoryValue &MemVal = Key.MemoryValues.at(k: MM.MemoryValueName);
70 BBF.addInstructions(Insts: ET.generateMmap(
71 Address: MM.Address, Length: MemVal.SizeBytes,
72 FileDescriptorAddress: ET.getAuxiliaryMemoryStartAddress() +
73 sizeof(int) *
74 (MemVal.Index + SubprocessMemory::AuxiliaryMemoryOffset)));
75 }
76 BBF.addInstructions(Insts: ET.setStackRegisterToAuxMem());
77 }
78 Register StackPointerRegister = BBF.MF.getSubtarget()
79 .getTargetLowering()
80 ->getStackPointerRegisterToSaveRestore();
81 for (const RegisterValue &RV : Key.RegisterInitialValues) {
82 if (GenerateMemoryInstructions) {
83 // If we're generating memory instructions, don't load in the value for
84 // the register with the stack pointer as it will be used later to finish
85 // the setup.
86 if (Register(RV.Register) == StackPointerRegister)
87 continue;
88 }
89 // Load a constant in the register.
90 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
91 if (SetRegisterCode.empty())
92 IsSnippetSetupComplete = false;
93 BBF.addInstructions(Insts: SetRegisterCode);
94 }
95 if (GenerateMemoryInstructions) {
96#ifdef HAVE_LIBPFM
97 BBF.addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_RESET, true));
98#endif // HAVE_LIBPFM
99 for (const RegisterValue &RV : Key.RegisterInitialValues) {
100 // Load in the stack register now as we're done using it elsewhere
101 // and need to set the value in preparation for executing the
102 // snippet.
103 if (Register(RV.Register) != StackPointerRegister)
104 continue;
105 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
106 if (SetRegisterCode.empty())
107 IsSnippetSetupComplete = false;
108 BBF.addInstructions(Insts: SetRegisterCode);
109 break;
110 }
111 }
112 return IsSnippetSetupComplete;
113}
114
115// Small utility function to add named passes.
116static bool addPass(PassManagerBase &PM, StringRef PassName,
117 TargetPassConfig &TPC) {
118 const PassRegistry *PR = PassRegistry::getPassRegistry();
119 const PassInfo *PI = PR->getPassInfo(Arg: PassName);
120 if (!PI) {
121 errs() << " run-pass " << PassName << " is not registered.\n";
122 return true;
123 }
124
125 if (!PI->getNormalCtor()) {
126 errs() << " cannot create pass: " << PI->getPassName() << "\n";
127 return true;
128 }
129 Pass *P = PI->getNormalCtor()();
130 std::string Banner = std::string("After ") + std::string(P->getPassName());
131 PM.add(P);
132 TPC.printAndVerify(Banner);
133
134 return false;
135}
136
137MachineFunction &createVoidVoidPtrMachineFunction(StringRef FunctionName,
138 Module *Module,
139 MachineModuleInfo *MMI) {
140 Type *const ReturnType = Type::getInt32Ty(C&: Module->getContext());
141 Type *const MemParamType =
142 PointerType::get(C&: Module->getContext(), AddressSpace: 0 /*default address space*/);
143 FunctionType *FunctionType =
144 FunctionType::get(Result: ReturnType, Params: {MemParamType}, isVarArg: false);
145 Function *const F = Function::Create(
146 Ty: FunctionType, Linkage: GlobalValue::ExternalLinkage, N: FunctionName, M: Module);
147 BasicBlock *BB = BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: F);
148 new UnreachableInst(Module->getContext(), BB);
149 return MMI->getOrCreateMachineFunction(F&: *F);
150}
151
152BasicBlockFiller::BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB,
153 const MCInstrInfo *MCII)
154 : MF(MF), MBB(MBB), MCII(MCII) {}
155
156void BasicBlockFiller::addInstruction(const MCInst &Inst, const DebugLoc &DL) {
157 const unsigned Opcode = Inst.getOpcode();
158 const MCInstrDesc &MCID = MCII->get(Opcode);
159 MachineInstrBuilder Builder = BuildMI(BB: MBB, MIMD: DL, MCID);
160 for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;
161 ++OpIndex) {
162 const MCOperand &Op = Inst.getOperand(i: OpIndex);
163 if (Op.isReg()) {
164 const bool IsDef = OpIndex < MCID.getNumDefs();
165 unsigned Flags = 0;
166 const MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];
167 if (IsDef && !OpInfo.isOptionalDef())
168 Flags |= RegState::Define;
169 Builder.addReg(RegNo: Op.getReg(), flags: Flags);
170 } else if (Op.isImm()) {
171 Builder.addImm(Val: Op.getImm());
172 } else if (!Op.isValid()) {
173 llvm_unreachable("Operand is not set");
174 } else {
175 llvm_unreachable("Not yet implemented");
176 }
177 }
178}
179
180void BasicBlockFiller::addInstructions(ArrayRef<MCInst> Insts,
181 const DebugLoc &DL) {
182 for (const MCInst &Inst : Insts)
183 addInstruction(Inst, DL);
184}
185
186void BasicBlockFiller::addReturn(const ExegesisTarget &ET,
187 bool SubprocessCleanup, const DebugLoc &DL) {
188 // Insert cleanup code
189 if (SubprocessCleanup) {
190#ifdef HAVE_LIBPFM
191 addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_DISABLE, false));
192#endif // HAVE_LIBPFM
193#ifdef __linux__
194 addInstructions(Insts: ET.generateExitSyscall(ExitCode: 0));
195#endif // __linux__
196 }
197 // Insert the return code.
198 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
199 if (TII->getReturnOpcode() < TII->getNumOpcodes()) {
200 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TII->getReturnOpcode()));
201 } else {
202 MachineIRBuilder MIB(MF);
203 MIB.setMBB(*MBB);
204
205 FunctionLoweringInfo FuncInfo;
206 FuncInfo.CanLowerReturn = true;
207 MF.getSubtarget().getCallLowering()->lowerReturn(MIRBuilder&: MIB, Val: nullptr, VRegs: {}, FLI&: FuncInfo,
208 SwiftErrorVReg: 0);
209 }
210}
211
212FunctionFiller::FunctionFiller(MachineFunction &MF,
213 std::vector<MCRegister> RegistersSetUp)
214 : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()),
215 RegistersSetUp(std::move(RegistersSetUp)) {}
216
217BasicBlockFiller FunctionFiller::addBasicBlock() {
218 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
219 MF.push_back(MBB);
220 return BasicBlockFiller(MF, MBB, MCII);
221}
222
223ArrayRef<MCRegister> FunctionFiller::getRegistersSetUp() const {
224 return RegistersSetUp;
225}
226
227static std::unique_ptr<Module>
228createModule(const std::unique_ptr<LLVMContext> &Context, const DataLayout &DL) {
229 auto Mod = std::make_unique<Module>(args: ModuleID, args&: *Context);
230 Mod->setDataLayout(DL);
231 return Mod;
232}
233
234BitVector getFunctionReservedRegs(const TargetMachine &TM) {
235 std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>();
236 std::unique_ptr<Module> Module = createModule(Context, DL: TM.createDataLayout());
237 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: &TM);
238 MachineFunction &MF = createVoidVoidPtrMachineFunction(
239 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP->getMMI());
240 // Saving reserved registers for client.
241 return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF);
242}
243
244Error assembleToStream(const ExegesisTarget &ET,
245 std::unique_ptr<TargetMachine> TM,
246 ArrayRef<MCRegister> LiveIns, const FillFunction &Fill,
247 raw_pwrite_stream &AsmStream, const BenchmarkKey &Key,
248 bool GenerateMemoryInstructions) {
249 auto Context = std::make_unique<LLVMContext>();
250 std::unique_ptr<Module> Module =
251 createModule(Context, DL: TM->createDataLayout());
252 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: TM.get());
253 MachineFunction &MF = createVoidVoidPtrMachineFunction(
254 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP.get()->getMMI());
255 MF.ensureAlignment(A: kFunctionAlignment);
256
257 // We need to instruct the passes that we're done with SSA and virtual
258 // registers.
259 auto &Properties = MF.getProperties();
260 Properties.setNoVRegs().resetIsSSA().setNoPHIs();
261
262 for (const MCRegister Reg : LiveIns)
263 MF.getRegInfo().addLiveIn(Reg);
264
265 if (GenerateMemoryInstructions) {
266 for (const MCRegister Reg : ET.getArgumentRegisters())
267 MF.getRegInfo().addLiveIn(Reg);
268 // Add a live in for registers that need saving so that the machine verifier
269 // doesn't fail if the register is never defined.
270 for (const MCRegister Reg : ET.getRegistersNeedSaving())
271 MF.getRegInfo().addLiveIn(Reg);
272 }
273
274 std::vector<MCRegister> RegistersSetUp;
275 RegistersSetUp.reserve(n: Key.RegisterInitialValues.size());
276 for (const auto &InitValue : Key.RegisterInitialValues) {
277 RegistersSetUp.push_back(x: InitValue.Register);
278 }
279 FunctionFiller Sink(MF, std::move(RegistersSetUp));
280 auto Entry = Sink.getEntry();
281
282 for (const MCRegister Reg : LiveIns)
283 Entry.MBB->addLiveIn(PhysReg: Reg);
284
285 if (GenerateMemoryInstructions) {
286 for (const MCRegister Reg : ET.getArgumentRegisters())
287 Entry.MBB->addLiveIn(PhysReg: Reg);
288 // Add a live in for registers that need saving so that the machine verifier
289 // doesn't fail if the register is never defined.
290 for (const MCRegister Reg : ET.getRegistersNeedSaving())
291 Entry.MBB->addLiveIn(PhysReg: Reg);
292 }
293
294 const bool IsSnippetSetupComplete = generateSnippetSetupCode(
295 ET, MSI: TM->getMCSubtargetInfo(), BBF&: Entry, Key, GenerateMemoryInstructions);
296
297 // If the snippet setup is not complete, we disable liveliness tracking. This
298 // means that we won't know what values are in the registers.
299 // FIXME: this should probably be an assertion.
300 if (!IsSnippetSetupComplete)
301 Properties.resetTracksLiveness();
302
303 Fill(Sink);
304
305 // prologue/epilogue pass needs the reserved registers to be frozen, this
306 // is usually done by the SelectionDAGISel pass.
307 MF.getRegInfo().freezeReservedRegs();
308
309 // We create the pass manager, run the passes to populate AsmBuffer.
310 MCContext &MCContext = MMIWP->getMMI().getContext();
311 legacy::PassManager PM;
312
313 TargetLibraryInfoImpl TLII(Module->getTargetTriple());
314 PM.add(P: new TargetLibraryInfoWrapperPass(TLII));
315
316 TargetPassConfig *TPC = TM->createPassConfig(PM);
317 PM.add(P: TPC);
318 PM.add(P: MMIWP.release());
319 TPC->printAndVerify(Banner: "MachineFunctionGenerator::assemble");
320 // Add target-specific passes.
321 ET.addTargetSpecificPasses(PM);
322 TPC->printAndVerify(Banner: "After ExegesisTarget::addTargetSpecificPasses");
323 // Adding the following passes:
324 // - postrapseudos: expands pseudo return instructions used on some targets.
325 // - prologepilog: saves and restore callee saved registers.
326 for (const char *PassName : {"postrapseudos", "prologepilog"})
327 if (addPass(PM, PassName, TPC&: *TPC))
328 return make_error<Failure>(Args: "Unable to add a mandatory pass");
329 TPC->setInitialized();
330
331 // AsmPrinter is responsible for generating the assembly into AsmBuffer.
332 if (TM->addAsmPrinter(PM, Out&: AsmStream, DwoOut: nullptr, FileType: CodeGenFileType::ObjectFile,
333 Context&: MCContext))
334 return make_error<Failure>(Args: "Cannot add AsmPrinter passes");
335
336 PM.run(M&: *Module); // Run all the passes
337 bool MFWellFormed =
338 MF.verify(p: nullptr, Banner: "llvm-exegesis Assembly", OS: &outs(), AbortOnError: false);
339 if (!MFWellFormed)
340 return make_error<Failure>(Args: "The machine function failed verification.");
341 return Error::success();
342}
343
344object::OwningBinary<object::ObjectFile>
345getObjectFromBuffer(StringRef InputData) {
346 // Storing the generated assembly into a MemoryBuffer that owns the memory.
347 std::unique_ptr<MemoryBuffer> Buffer =
348 MemoryBuffer::getMemBufferCopy(InputData);
349 // Create the ObjectFile from the MemoryBuffer.
350 std::unique_ptr<object::ObjectFile> Obj =
351 cantFail(ValOrErr: object::ObjectFile::createObjectFile(Object: Buffer->getMemBufferRef()));
352 // Returning both the MemoryBuffer and the ObjectFile.
353 return object::OwningBinary<object::ObjectFile>(std::move(Obj),
354 std::move(Buffer));
355}
356
357object::OwningBinary<object::ObjectFile> getObjectFromFile(StringRef Filename) {
358 return cantFail(ValOrErr: object::ObjectFile::createObjectFile(ObjectPath: Filename));
359}
360
361Expected<ExecutableFunction> ExecutableFunction::create(
362 std::unique_ptr<TargetMachine> TM,
363 object::OwningBinary<object::ObjectFile> &&ObjectFileHolder) {
364 assert(ObjectFileHolder.getBinary() && "cannot create object file");
365 std::unique_ptr<LLVMContext> Ctx = std::make_unique<LLVMContext>();
366
367 auto SymbolSizes = object::computeSymbolSizes(O: *ObjectFileHolder.getBinary());
368 // Get the size of the function that we want to call into (with the name of
369 // FunctionID).
370 auto SymbolIt = find_if(Range&: SymbolSizes, P: [&](const auto &Pair) {
371 auto SymbolName = Pair.first.getName();
372 if (SymbolName)
373 return *SymbolName == FunctionID;
374 // We should always succeed in finding the FunctionID, hence we suppress
375 // the error here and assert later on the search result, rather than
376 // propagating the Expected<> error back to the caller.
377 consumeError(SymbolName.takeError());
378 return false;
379 });
380 assert(SymbolIt != SymbolSizes.end() &&
381 "Cannot find the symbol for FunctionID");
382 uintptr_t CodeSize = SymbolIt->second;
383
384 auto EJITOrErr = orc::LLJITBuilder().create();
385 if (!EJITOrErr)
386 return EJITOrErr.takeError();
387
388 auto EJIT = std::move(*EJITOrErr);
389
390 if (auto ObjErr =
391 EJIT->addObjectFile(Obj: std::get<1>(in: ObjectFileHolder.takeBinary())))
392 return std::move(ObjErr);
393
394 auto FunctionAddressOrErr = EJIT->lookup(UnmangledName: FunctionID);
395 if (!FunctionAddressOrErr)
396 return FunctionAddressOrErr.takeError();
397
398 const uint64_t FunctionAddress = FunctionAddressOrErr->getValue();
399
400 assert(isAligned(kFunctionAlignment, FunctionAddress) &&
401 "function is not properly aligned");
402
403 StringRef FBytes =
404 StringRef(reinterpret_cast<const char *>(FunctionAddress), CodeSize);
405 return ExecutableFunction(std::move(Ctx), std::move(EJIT), FBytes);
406}
407
408ExecutableFunction::ExecutableFunction(std::unique_ptr<LLVMContext> Ctx,
409 std::unique_ptr<orc::LLJIT> EJIT,
410 StringRef FB)
411 : FunctionBytes(FB), Context(std::move(Ctx)), ExecJIT(std::move(EJIT)) {}
412
413Error getBenchmarkFunctionBytes(const StringRef InputData,
414 std::vector<uint8_t> &Bytes) {
415 const auto Holder = getObjectFromBuffer(InputData);
416 const auto *Obj = Holder.getBinary();
417 // See RuntimeDyldImpl::loadObjectImpl(Obj) for much more complete
418 // implementation.
419
420 // Find the only function in the object file.
421 SmallVector<object::SymbolRef, 1> Functions;
422 for (auto &Sym : Obj->symbols()) {
423 auto SymType = Sym.getType();
424 if (SymType && *SymType == object::SymbolRef::Type::ST_Function)
425 Functions.push_back(Elt: Sym);
426 }
427 if (Functions.size() != 1)
428 return make_error<Failure>(Args: "Exactly one function expected");
429
430 // Find the containing section - it is assumed to contain only this function.
431 auto SectionOrErr = Functions.front().getSection();
432 if (!SectionOrErr || *SectionOrErr == Obj->section_end())
433 return make_error<Failure>(Args: "Section not found");
434
435 auto Address = Functions.front().getAddress();
436 if (!Address || *Address != SectionOrErr.get()->getAddress())
437 return make_error<Failure>(Args: "Unexpected layout");
438
439 auto ContentsOrErr = SectionOrErr.get()->getContents();
440 if (!ContentsOrErr)
441 return ContentsOrErr.takeError();
442 Bytes.assign(first: ContentsOrErr->begin(), last: ContentsOrErr->end());
443 return Error::success();
444}
445
446} // namespace exegesis
447} // namespace llvm
448