1//===-- Assembler.cpp -------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Assembler.h"
10
11#include "SnippetRepetitor.h"
12#include "SubprocessMemory.h"
13#include "Target.h"
14#include "llvm/Analysis/TargetLibraryInfo.h"
15#include "llvm/CodeGen/FunctionLoweringInfo.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/MachineInstrBuilder.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/TargetInstrInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetPassConfig.h"
24#include "llvm/CodeGen/TargetSubtargetInfo.h"
25#include "llvm/ExecutionEngine/Orc/LLJIT.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/LegacyPassManager.h"
29#include "llvm/MC/MCInstrInfo.h"
30#include "llvm/Object/SymbolSize.h"
31#include "llvm/Support/Alignment.h"
32#include "llvm/Support/MemoryBuffer.h"
33#include "llvm/Support/raw_ostream.h"
34
35#ifdef HAVE_LIBPFM
36#include "perfmon/perf_event.h"
37#endif // HAVE_LIBPFM
38
39#ifdef __linux__
40#include <unistd.h>
41#endif
42
43namespace llvm {
44namespace exegesis {
45
46static constexpr const char ModuleID[] = "ExegesisInfoTest";
47static constexpr const char FunctionID[] = "foo";
48static const Align kFunctionAlignment(4096);
49
50// Fills the given basic block with register setup code, and returns true if
51// all registers could be setup correctly.
52static bool generateSnippetSetupCode(const ExegesisTarget &ET,
53 const MCSubtargetInfo *const MSI,
54 BasicBlockFiller &BBF,
55 const BenchmarkKey &Key,
56 bool GenerateMemoryInstructions) {
57 bool IsSnippetSetupComplete = true;
58 if (GenerateMemoryInstructions) {
59 BBF.addInstructions(Insts: ET.generateMemoryInitialSetup());
60 for (const MemoryMapping &MM : Key.MemoryMappings) {
61#ifdef __linux__
62 // The frontend that generates that parses the memory mapping information
63 // from the user should validate that the requested address is a multiple
64 // of the page size. Assert that this is true here.
65 assert(MM.Address % getpagesize() == 0 &&
66 "Memory mappings need to be aligned to page boundaries.");
67#endif
68 BBF.addInstructions(Insts: ET.generateMmap(
69 Address: MM.Address, Length: Key.MemoryValues.at(k: MM.MemoryValueName).SizeBytes,
70 FileDescriptorAddress: ET.getAuxiliaryMemoryStartAddress() +
71 sizeof(int) * (Key.MemoryValues.at(k: MM.MemoryValueName).Index +
72 SubprocessMemory::AuxiliaryMemoryOffset)));
73 }
74 BBF.addInstructions(Insts: ET.setStackRegisterToAuxMem());
75 }
76 Register StackPointerRegister = BBF.MF.getSubtarget()
77 .getTargetLowering()
78 ->getStackPointerRegisterToSaveRestore();
79 for (const RegisterValue &RV : Key.RegisterInitialValues) {
80 if (GenerateMemoryInstructions) {
81 // If we're generating memory instructions, don't load in the value for
82 // the register with the stack pointer as it will be used later to finish
83 // the setup.
84 if (RV.Register == StackPointerRegister)
85 continue;
86 }
87 // Load a constant in the register.
88 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
89 if (SetRegisterCode.empty())
90 IsSnippetSetupComplete = false;
91 BBF.addInstructions(Insts: SetRegisterCode);
92 }
93 if (GenerateMemoryInstructions) {
94#ifdef HAVE_LIBPFM
95 BBF.addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_RESET, true));
96#endif // HAVE_LIBPFM
97 for (const RegisterValue &RV : Key.RegisterInitialValues) {
98 // Load in the stack register now as we're done using it elsewhere
99 // and need to set the value in preparation for executing the
100 // snippet.
101 if (RV.Register != StackPointerRegister)
102 continue;
103 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
104 if (SetRegisterCode.empty())
105 IsSnippetSetupComplete = false;
106 BBF.addInstructions(Insts: SetRegisterCode);
107 break;
108 }
109 }
110 return IsSnippetSetupComplete;
111}
112
113// Small utility function to add named passes.
114static bool addPass(PassManagerBase &PM, StringRef PassName,
115 TargetPassConfig &TPC) {
116 const PassRegistry *PR = PassRegistry::getPassRegistry();
117 const PassInfo *PI = PR->getPassInfo(Arg: PassName);
118 if (!PI) {
119 errs() << " run-pass " << PassName << " is not registered.\n";
120 return true;
121 }
122
123 if (!PI->getNormalCtor()) {
124 errs() << " cannot create pass: " << PI->getPassName() << "\n";
125 return true;
126 }
127 Pass *P = PI->getNormalCtor()();
128 std::string Banner = std::string("After ") + std::string(P->getPassName());
129 PM.add(P);
130 TPC.printAndVerify(Banner);
131
132 return false;
133}
134
135MachineFunction &createVoidVoidPtrMachineFunction(StringRef FunctionName,
136 Module *Module,
137 MachineModuleInfo *MMI) {
138 Type *const ReturnType = Type::getInt32Ty(C&: Module->getContext());
139 Type *const MemParamType = PointerType::get(
140 ElementType: Type::getInt8Ty(C&: Module->getContext()), AddressSpace: 0 /*default address space*/);
141 FunctionType *FunctionType =
142 FunctionType::get(Result: ReturnType, Params: {MemParamType}, isVarArg: false);
143 Function *const F = Function::Create(
144 Ty: FunctionType, Linkage: GlobalValue::ExternalLinkage, N: FunctionName, M: Module);
145 BasicBlock *BB = BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: F);
146 new UnreachableInst(Module->getContext(), BB);
147 return MMI->getOrCreateMachineFunction(F&: *F);
148}
149
150BasicBlockFiller::BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB,
151 const MCInstrInfo *MCII)
152 : MF(MF), MBB(MBB), MCII(MCII) {}
153
154void BasicBlockFiller::addInstruction(const MCInst &Inst, const DebugLoc &DL) {
155 const unsigned Opcode = Inst.getOpcode();
156 const MCInstrDesc &MCID = MCII->get(Opcode);
157 MachineInstrBuilder Builder = BuildMI(BB: MBB, MIMD: DL, MCID);
158 for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;
159 ++OpIndex) {
160 const MCOperand &Op = Inst.getOperand(i: OpIndex);
161 if (Op.isReg()) {
162 const bool IsDef = OpIndex < MCID.getNumDefs();
163 unsigned Flags = 0;
164 const MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];
165 if (IsDef && !OpInfo.isOptionalDef())
166 Flags |= RegState::Define;
167 Builder.addReg(RegNo: Op.getReg(), flags: Flags);
168 } else if (Op.isImm()) {
169 Builder.addImm(Val: Op.getImm());
170 } else if (!Op.isValid()) {
171 llvm_unreachable("Operand is not set");
172 } else {
173 llvm_unreachable("Not yet implemented");
174 }
175 }
176}
177
178void BasicBlockFiller::addInstructions(ArrayRef<MCInst> Insts,
179 const DebugLoc &DL) {
180 for (const MCInst &Inst : Insts)
181 addInstruction(Inst, DL);
182}
183
184void BasicBlockFiller::addReturn(const ExegesisTarget &ET,
185 bool SubprocessCleanup, const DebugLoc &DL) {
186 // Insert cleanup code
187 if (SubprocessCleanup) {
188#ifdef HAVE_LIBPFM
189 addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_DISABLE, false));
190#endif // HAVE_LIBPFM
191#ifdef __linux__
192 addInstructions(Insts: ET.generateExitSyscall(ExitCode: 0));
193#endif // __linux__
194 }
195 // Insert the return code.
196 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
197 if (TII->getReturnOpcode() < TII->getNumOpcodes()) {
198 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TII->getReturnOpcode()));
199 } else {
200 MachineIRBuilder MIB(MF);
201 MIB.setMBB(*MBB);
202
203 FunctionLoweringInfo FuncInfo;
204 FuncInfo.CanLowerReturn = true;
205 MF.getSubtarget().getCallLowering()->lowerReturn(MIRBuilder&: MIB, Val: nullptr, VRegs: {}, FLI&: FuncInfo,
206 SwiftErrorVReg: 0);
207 }
208}
209
210FunctionFiller::FunctionFiller(MachineFunction &MF,
211 std::vector<unsigned> RegistersSetUp)
212 : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()),
213 RegistersSetUp(std::move(RegistersSetUp)) {}
214
215BasicBlockFiller FunctionFiller::addBasicBlock() {
216 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
217 MF.push_back(MBB);
218 return BasicBlockFiller(MF, MBB, MCII);
219}
220
221ArrayRef<unsigned> FunctionFiller::getRegistersSetUp() const {
222 return RegistersSetUp;
223}
224
225static std::unique_ptr<Module>
226createModule(const std::unique_ptr<LLVMContext> &Context, const DataLayout &DL) {
227 auto Mod = std::make_unique<Module>(args: ModuleID, args&: *Context);
228 Mod->setDataLayout(DL);
229 return Mod;
230}
231
232BitVector getFunctionReservedRegs(const TargetMachine &TM) {
233 std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>();
234 std::unique_ptr<Module> Module = createModule(Context, DL: TM.createDataLayout());
235 // TODO: This only works for targets implementing LLVMTargetMachine.
236 const LLVMTargetMachine &LLVMTM = static_cast<const LLVMTargetMachine &>(TM);
237 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: &LLVMTM);
238 MachineFunction &MF = createVoidVoidPtrMachineFunction(
239 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP->getMMI());
240 // Saving reserved registers for client.
241 return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF);
242}
243
244Error assembleToStream(const ExegesisTarget &ET,
245 std::unique_ptr<LLVMTargetMachine> TM,
246 ArrayRef<unsigned> LiveIns, const FillFunction &Fill,
247 raw_pwrite_stream &AsmStream, const BenchmarkKey &Key,
248 bool GenerateMemoryInstructions) {
249 auto Context = std::make_unique<LLVMContext>();
250 std::unique_ptr<Module> Module =
251 createModule(Context, DL: TM->createDataLayout());
252 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: TM.get());
253 MachineFunction &MF = createVoidVoidPtrMachineFunction(
254 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP.get()->getMMI());
255 MF.ensureAlignment(A: kFunctionAlignment);
256
257 // We need to instruct the passes that we're done with SSA and virtual
258 // registers.
259 auto &Properties = MF.getProperties();
260 Properties.set(MachineFunctionProperties::Property::NoVRegs);
261 Properties.reset(P: MachineFunctionProperties::Property::IsSSA);
262 Properties.set(MachineFunctionProperties::Property::NoPHIs);
263
264 for (const unsigned Reg : LiveIns)
265 MF.getRegInfo().addLiveIn(Reg);
266
267 if (GenerateMemoryInstructions) {
268 for (const unsigned Reg : ET.getArgumentRegisters())
269 MF.getRegInfo().addLiveIn(Reg);
270 // Add a live in for registers that need saving so that the machine verifier
271 // doesn't fail if the register is never defined.
272 for (const unsigned Reg : ET.getRegistersNeedSaving())
273 MF.getRegInfo().addLiveIn(Reg);
274 }
275
276 std::vector<unsigned> RegistersSetUp;
277 for (const auto &InitValue : Key.RegisterInitialValues) {
278 RegistersSetUp.push_back(x: InitValue.Register);
279 }
280 FunctionFiller Sink(MF, std::move(RegistersSetUp));
281 auto Entry = Sink.getEntry();
282
283 for (const unsigned Reg : LiveIns)
284 Entry.MBB->addLiveIn(PhysReg: Reg);
285
286 if (GenerateMemoryInstructions) {
287 for (const unsigned Reg : ET.getArgumentRegisters())
288 Entry.MBB->addLiveIn(PhysReg: Reg);
289 // Add a live in for registers that need saving so that the machine verifier
290 // doesn't fail if the register is never defined.
291 for (const unsigned Reg : ET.getRegistersNeedSaving())
292 Entry.MBB->addLiveIn(PhysReg: Reg);
293 }
294
295 const bool IsSnippetSetupComplete = generateSnippetSetupCode(
296 ET, MSI: TM->getMCSubtargetInfo(), BBF&: Entry, Key, GenerateMemoryInstructions);
297
298 // If the snippet setup is not complete, we disable liveliness tracking. This
299 // means that we won't know what values are in the registers.
300 // FIXME: this should probably be an assertion.
301 if (!IsSnippetSetupComplete)
302 Properties.reset(P: MachineFunctionProperties::Property::TracksLiveness);
303
304 Fill(Sink);
305
306 // prologue/epilogue pass needs the reserved registers to be frozen, this
307 // is usually done by the SelectionDAGISel pass.
308 MF.getRegInfo().freezeReservedRegs();
309
310 // We create the pass manager, run the passes to populate AsmBuffer.
311 MCContext &MCContext = MMIWP->getMMI().getContext();
312 legacy::PassManager PM;
313
314 TargetLibraryInfoImpl TLII(Triple(Module->getTargetTriple()));
315 PM.add(P: new TargetLibraryInfoWrapperPass(TLII));
316
317 TargetPassConfig *TPC = TM->createPassConfig(PM);
318 PM.add(P: TPC);
319 PM.add(P: MMIWP.release());
320 TPC->printAndVerify(Banner: "MachineFunctionGenerator::assemble");
321 // Add target-specific passes.
322 ET.addTargetSpecificPasses(PM);
323 TPC->printAndVerify(Banner: "After ExegesisTarget::addTargetSpecificPasses");
324 // Adding the following passes:
325 // - postrapseudos: expands pseudo return instructions used on some targets.
326 // - machineverifier: checks that the MachineFunction is well formed.
327 // - prologepilog: saves and restore callee saved registers.
328 for (const char *PassName :
329 {"postrapseudos", "machineverifier", "prologepilog"})
330 if (addPass(PM, PassName, TPC&: *TPC))
331 return make_error<Failure>(Args: "Unable to add a mandatory pass");
332 TPC->setInitialized();
333
334 // AsmPrinter is responsible for generating the assembly into AsmBuffer.
335 if (TM->addAsmPrinter(PM, Out&: AsmStream, DwoOut: nullptr, FileType: CodeGenFileType::ObjectFile,
336 Context&: MCContext))
337 return make_error<Failure>(Args: "Cannot add AsmPrinter passes");
338
339 PM.run(M&: *Module); // Run all the passes
340 return Error::success();
341}
342
343object::OwningBinary<object::ObjectFile>
344getObjectFromBuffer(StringRef InputData) {
345 // Storing the generated assembly into a MemoryBuffer that owns the memory.
346 std::unique_ptr<MemoryBuffer> Buffer =
347 MemoryBuffer::getMemBufferCopy(InputData);
348 // Create the ObjectFile from the MemoryBuffer.
349 std::unique_ptr<object::ObjectFile> Obj =
350 cantFail(ValOrErr: object::ObjectFile::createObjectFile(Object: Buffer->getMemBufferRef()));
351 // Returning both the MemoryBuffer and the ObjectFile.
352 return object::OwningBinary<object::ObjectFile>(std::move(Obj),
353 std::move(Buffer));
354}
355
356object::OwningBinary<object::ObjectFile> getObjectFromFile(StringRef Filename) {
357 return cantFail(ValOrErr: object::ObjectFile::createObjectFile(ObjectPath: Filename));
358}
359
360Expected<ExecutableFunction> ExecutableFunction::create(
361 std::unique_ptr<LLVMTargetMachine> TM,
362 object::OwningBinary<object::ObjectFile> &&ObjectFileHolder) {
363 assert(ObjectFileHolder.getBinary() && "cannot create object file");
364 std::unique_ptr<LLVMContext> Ctx = std::make_unique<LLVMContext>();
365
366 auto SymbolSizes = object::computeSymbolSizes(O: *ObjectFileHolder.getBinary());
367 // Get the size of the function that we want to call into (with the name of
368 // FunctionID).
369 auto SymbolIt = find_if(Range&: SymbolSizes, P: [&](const auto &Pair) {
370 auto SymbolName = Pair.first.getName();
371 if (SymbolName)
372 return *SymbolName == FunctionID;
373 // We should always succeed in finding the FunctionID, hence we suppress
374 // the error here and assert later on the search result, rather than
375 // propagating the Expected<> error back to the caller.
376 consumeError(SymbolName.takeError());
377 return false;
378 });
379 assert(SymbolIt != SymbolSizes.end() &&
380 "Cannot find the symbol for FunctionID");
381 uintptr_t CodeSize = SymbolIt->second;
382
383 auto EJITOrErr = orc::LLJITBuilder().create();
384 if (!EJITOrErr)
385 return EJITOrErr.takeError();
386
387 auto EJIT = std::move(*EJITOrErr);
388
389 if (auto ObjErr =
390 EJIT->addObjectFile(Obj: std::get<1>(in: ObjectFileHolder.takeBinary())))
391 return std::move(ObjErr);
392
393 auto FunctionAddressOrErr = EJIT->lookup(UnmangledName: FunctionID);
394 if (!FunctionAddressOrErr)
395 return FunctionAddressOrErr.takeError();
396
397 const uint64_t FunctionAddress = FunctionAddressOrErr->getValue();
398
399 assert(isAligned(kFunctionAlignment, FunctionAddress) &&
400 "function is not properly aligned");
401
402 StringRef FBytes =
403 StringRef(reinterpret_cast<const char *>(FunctionAddress), CodeSize);
404 return ExecutableFunction(std::move(Ctx), std::move(EJIT), FBytes);
405}
406
407ExecutableFunction::ExecutableFunction(std::unique_ptr<LLVMContext> Ctx,
408 std::unique_ptr<orc::LLJIT> EJIT,
409 StringRef FB)
410 : FunctionBytes(FB), Context(std::move(Ctx)), ExecJIT(std::move(EJIT)) {}
411
412Error getBenchmarkFunctionBytes(const StringRef InputData,
413 std::vector<uint8_t> &Bytes) {
414 const auto Holder = getObjectFromBuffer(InputData);
415 const auto *Obj = Holder.getBinary();
416 // See RuntimeDyldImpl::loadObjectImpl(Obj) for much more complete
417 // implementation.
418
419 // Find the only function in the object file.
420 SmallVector<object::SymbolRef, 1> Functions;
421 for (auto &Sym : Obj->symbols()) {
422 auto SymType = Sym.getType();
423 if (SymType && *SymType == object::SymbolRef::Type::ST_Function)
424 Functions.push_back(Elt: Sym);
425 }
426 if (Functions.size() != 1)
427 return make_error<Failure>(Args: "Exactly one function expected");
428
429 // Find the containing section - it is assumed to contain only this function.
430 auto SectionOrErr = Functions.front().getSection();
431 if (!SectionOrErr || *SectionOrErr == Obj->section_end())
432 return make_error<Failure>(Args: "Section not found");
433
434 auto Address = Functions.front().getAddress();
435 if (!Address || *Address != SectionOrErr.get()->getAddress())
436 return make_error<Failure>(Args: "Unexpected layout");
437
438 auto ContentsOrErr = SectionOrErr.get()->getContents();
439 if (!ContentsOrErr)
440 return ContentsOrErr.takeError();
441 Bytes.assign(first: ContentsOrErr->begin(), last: ContentsOrErr->end());
442 return Error::success();
443}
444
445} // namespace exegesis
446} // namespace llvm
447