1//===-- Assembler.cpp -------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Assembler.h"
10
11#include "SnippetRepetitor.h"
12#include "SubprocessMemory.h"
13#include "Target.h"
14#include "llvm/Analysis/TargetLibraryInfo.h"
15#include "llvm/CodeGen/FunctionLoweringInfo.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/MachineInstrBuilder.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/TargetInstrInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetPassConfig.h"
24#include "llvm/CodeGen/TargetSubtargetInfo.h"
25#include "llvm/ExecutionEngine/Orc/LLJIT.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/LegacyPassManager.h"
29#include "llvm/MC/MCInstrInfo.h"
30#include "llvm/Object/SymbolSize.h"
31#include "llvm/Support/Alignment.h"
32#include "llvm/Support/Error.h"
33#include "llvm/Support/LibraryOptionsContext.h"
34#include "llvm/Support/MemoryBuffer.h"
35#include "llvm/Support/raw_ostream.h"
36
37#ifdef HAVE_LIBPFM
38#include "perfmon/perf_event.h"
39#endif // HAVE_LIBPFM
40
41#ifdef __linux__
42#include <unistd.h>
43#endif
44
45namespace llvm {
46namespace exegesis {
47
48static constexpr char ModuleID[] = "ExegesisInfoTest";
49static constexpr char FunctionID[] = "foo";
50static const Align kFunctionAlignment(4096);
51
52// Fills the given basic block with register setup code, and returns true if
53// all registers could be setup correctly.
54static bool generateSnippetSetupCode(const ExegesisTarget &ET,
55 const MCSubtargetInfo *const MSI,
56 BasicBlockFiller &BBF,
57 const BenchmarkKey &Key,
58 bool GenerateMemoryInstructions) {
59 bool IsSnippetSetupComplete = true;
60 if (GenerateMemoryInstructions) {
61 BBF.addInstructions(Insts: ET.generateMemoryInitialSetup());
62 for (const MemoryMapping &MM : Key.MemoryMappings) {
63#ifdef __linux__
64 // The frontend that generates that parses the memory mapping information
65 // from the user should validate that the requested address is a multiple
66 // of the page size. Assert that this is true here.
67 assert(MM.Address % getpagesize() == 0 &&
68 "Memory mappings need to be aligned to page boundaries.");
69#endif
70 const MemoryValue &MemVal = Key.MemoryValues.at(Val: MM.MemoryValueName);
71 BBF.addInstructions(Insts: ET.generateMmap(
72 Address: MM.Address, Length: MemVal.SizeBytes,
73 FileDescriptorAddress: ET.getAuxiliaryMemoryStartAddress() +
74 sizeof(int) *
75 (MemVal.Index + SubprocessMemory::AuxiliaryMemoryOffset)));
76 }
77 BBF.addInstructions(Insts: ET.setStackRegisterToAuxMem());
78 }
79 Register StackPointerRegister = BBF.MF.getSubtarget()
80 .getTargetLowering()
81 ->getStackPointerRegisterToSaveRestore();
82 for (const RegisterValue &RV : Key.RegisterInitialValues) {
83 if (GenerateMemoryInstructions) {
84 // If we're generating memory instructions, don't load in the value for
85 // the register with the stack pointer as it will be used later to finish
86 // the setup.
87 if (Register(RV.Register) == StackPointerRegister)
88 continue;
89 }
90 // Load a constant in the register.
91 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
92 if (SetRegisterCode.empty())
93 IsSnippetSetupComplete = false;
94 BBF.addInstructions(Insts: SetRegisterCode);
95 }
96 if (GenerateMemoryInstructions) {
97#ifdef HAVE_LIBPFM
98 BBF.addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_RESET, true));
99#endif // HAVE_LIBPFM
100 for (const RegisterValue &RV : Key.RegisterInitialValues) {
101 // Load in the stack register now as we're done using it elsewhere
102 // and need to set the value in preparation for executing the
103 // snippet.
104 if (Register(RV.Register) != StackPointerRegister)
105 continue;
106 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
107 if (SetRegisterCode.empty())
108 IsSnippetSetupComplete = false;
109 BBF.addInstructions(Insts: SetRegisterCode);
110 break;
111 }
112 }
113 return IsSnippetSetupComplete;
114}
115
116// Small utility function to add named passes.
117static bool addPass(PassManagerBase &PM, StringRef PassName,
118 TargetPassConfig &TPC) {
119 const PassRegistry *PR = PassRegistry::getPassRegistry();
120 const PassInfo *PI = PR->getPassInfo(Arg: PassName);
121 if (!PI) {
122 errs() << " run-pass " << PassName << " is not registered.\n";
123 return true;
124 }
125
126 if (!PI->getNormalCtor()) {
127 errs() << " cannot create pass: " << PI->getPassName() << "\n";
128 return true;
129 }
130 Pass *P = PI->getNormalCtor()();
131 std::string Banner = std::string("After ") + std::string(P->getPassName());
132 PM.add(P);
133 TPC.printAndVerify(Banner);
134
135 return false;
136}
137
138MachineFunction &createVoidVoidPtrMachineFunction(StringRef FunctionName,
139 Module *Module,
140 MachineModuleInfo *MMI) {
141 Type *const ReturnType = Type::getInt32Ty(C&: Module->getContext());
142 Type *const MemParamType =
143 PointerType::get(C&: Module->getContext(), AddressSpace: 0 /*default address space*/);
144 FunctionType *FunctionType =
145 FunctionType::get(Result: ReturnType, Params: {MemParamType}, isVarArg: false);
146 Function *const F = Function::Create(
147 Ty: FunctionType, Linkage: GlobalValue::ExternalLinkage, N: FunctionName, M: Module);
148 BasicBlock *BB = BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: F);
149 new UnreachableInst(Module->getContext(), BB);
150 return MMI->getOrCreateMachineFunction(F&: *F);
151}
152
153BasicBlockFiller::BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB,
154 const MCInstrInfo *MCII)
155 : MF(MF), MBB(MBB), MCII(MCII) {}
156
157void BasicBlockFiller::addInstruction(const MCInst &Inst, const DebugLoc &DL) {
158 const unsigned Opcode = Inst.getOpcode();
159 const MCInstrDesc &MCID = MCII->get(Opcode);
160 MachineInstrBuilder Builder = BuildMI(BB: MBB, MIMD: DL, MCID);
161 for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;
162 ++OpIndex) {
163 const MCOperand &Op = Inst.getOperand(i: OpIndex);
164 if (Op.isReg()) {
165 const bool IsDef = OpIndex < MCID.getNumDefs();
166 RegState Flags = {};
167 const MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];
168 if (IsDef && !OpInfo.isOptionalDef())
169 Flags |= RegState::Define;
170 Builder.addReg(RegNo: Op.getReg(), Flags);
171 } else if (Op.isImm()) {
172 Builder.addImm(Val: Op.getImm());
173 } else if (!Op.isValid()) {
174 llvm_unreachable("Operand is not set");
175 } else {
176 llvm_unreachable("Not yet implemented");
177 }
178 }
179}
180
181void BasicBlockFiller::addInstructions(ArrayRef<MCInst> Insts,
182 const DebugLoc &DL) {
183 for (const MCInst &Inst : Insts)
184 addInstruction(Inst, DL);
185}
186
187void BasicBlockFiller::addReturn(const ExegesisTarget &ET,
188 bool SubprocessCleanup, const DebugLoc &DL) {
189 // Insert cleanup code
190 if (SubprocessCleanup) {
191#ifdef HAVE_LIBPFM
192 addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_DISABLE, false));
193#endif // HAVE_LIBPFM
194#ifdef __linux__
195 addInstructions(Insts: ET.generateExitSyscall(ExitCode: 0));
196#endif // __linux__
197 }
198 // Insert the return code.
199 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
200 if (TII->getReturnOpcode() < TII->getNumOpcodes()) {
201 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TII->getReturnOpcode()));
202 } else {
203 MachineIRBuilder MIB(MF);
204 MIB.setMBB(*MBB);
205
206 FunctionLoweringInfo FuncInfo;
207 FuncInfo.CanLowerReturn = true;
208 MF.getSubtarget().getCallLowering()->lowerReturn(MIRBuilder&: MIB, Val: nullptr, VRegs: {}, FLI&: FuncInfo,
209 SwiftErrorVReg: 0);
210 }
211}
212
213FunctionFiller::FunctionFiller(MachineFunction &MF,
214 std::vector<MCRegister> RegistersSetUp)
215 : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()),
216 RegistersSetUp(std::move(RegistersSetUp)) {}
217
218BasicBlockFiller FunctionFiller::addBasicBlock() {
219 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
220 MF.push_back(MBB);
221 return BasicBlockFiller(MF, MBB, MCII);
222}
223
224ArrayRef<MCRegister> FunctionFiller::getRegistersSetUp() const {
225 return RegistersSetUp;
226}
227
228static std::unique_ptr<Module>
229createModule(const std::unique_ptr<LLVMContext> &Context,
230 const TargetMachine &TM) {
231 if (const cl::LibraryOptionsContext *C = TM.Options.MCOptions.LibraryOptions)
232 Context->getLibraryOptions().assign(Other: *C);
233 auto Mod = std::make_unique<Module>(args: ModuleID, args&: *Context);
234 const Triple &TT = TM.getTargetTriple();
235 Mod->setTargetTriple(TT);
236 StringRef ABIName = TM.Options.MCOptions.getABIName();
237 if (!ABIName.empty()) {
238 Mod->addModuleFlag(Behavior: Module::Error, Key: "target-abi",
239 Val: MDString::get(Context&: *Context, Str: ABIName));
240 }
241
242 Mod->setDataLayout(DataLayout(TT.computeDataLayout(ABIName)));
243 return Mod;
244}
245
246BitVector getFunctionReservedRegs(const TargetMachine &TM) {
247 std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>();
248 std::unique_ptr<Module> Module = createModule(Context, TM);
249 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: &TM);
250 MachineFunction &MF = createVoidVoidPtrMachineFunction(
251 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP->getMMI());
252 // Saving reserved registers for client.
253 return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF);
254}
255
256Error assembleToStream(const ExegesisTarget &ET,
257 std::unique_ptr<TargetMachine> TM,
258 ArrayRef<MCRegister> LiveIns, const FillFunction &Fill,
259 raw_pwrite_stream &AsmStream, const BenchmarkKey &Key,
260 bool GenerateMemoryInstructions) {
261 auto Context = std::make_unique<LLVMContext>();
262 std::unique_ptr<Module> Module = createModule(Context, TM: *TM);
263 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: TM.get());
264 MachineFunction &MF = createVoidVoidPtrMachineFunction(
265 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP.get()->getMMI());
266 MF.ensureAlignment(A: kFunctionAlignment);
267
268 // We need to instruct the passes that we're done with SSA and virtual
269 // registers.
270 auto &Properties = MF.getProperties();
271 Properties.setNoVRegs().resetIsSSA().setNoPHIs();
272
273 for (const MCRegister Reg : LiveIns)
274 MF.getRegInfo().addLiveIn(Reg);
275
276 if (GenerateMemoryInstructions) {
277 for (const MCRegister Reg : ET.getArgumentRegisters())
278 MF.getRegInfo().addLiveIn(Reg);
279 // Add a live in for registers that need saving so that the machine verifier
280 // doesn't fail if the register is never defined.
281 for (const MCRegister Reg : ET.getRegistersNeedSaving())
282 MF.getRegInfo().addLiveIn(Reg);
283 }
284
285 std::vector<MCRegister> RegistersSetUp;
286 RegistersSetUp.reserve(n: Key.RegisterInitialValues.size());
287 for (const auto &InitValue : Key.RegisterInitialValues) {
288 RegistersSetUp.push_back(x: InitValue.Register);
289 }
290 FunctionFiller Sink(MF, std::move(RegistersSetUp));
291 auto Entry = Sink.getEntry();
292
293 for (const MCRegister Reg : LiveIns)
294 Entry.MBB->addLiveIn(PhysReg: Reg);
295
296 if (GenerateMemoryInstructions) {
297 for (const MCRegister Reg : ET.getArgumentRegisters())
298 Entry.MBB->addLiveIn(PhysReg: Reg);
299 // Add a live in for registers that need saving so that the machine verifier
300 // doesn't fail if the register is never defined.
301 for (const MCRegister Reg : ET.getRegistersNeedSaving())
302 Entry.MBB->addLiveIn(PhysReg: Reg);
303 }
304
305 const bool IsSnippetSetupComplete = generateSnippetSetupCode(
306 ET, MSI: &TM->getMCSubtargetInfo(), BBF&: Entry, Key, GenerateMemoryInstructions);
307
308 // If the snippet setup is not complete, we disable liveliness tracking. This
309 // means that we won't know what values are in the registers.
310 // FIXME: this should probably be an assertion.
311 if (!IsSnippetSetupComplete)
312 Properties.resetTracksLiveness();
313
314 Fill(Sink);
315
316 // prologue/epilogue pass needs the reserved registers to be frozen, this
317 // is usually done by the SelectionDAGISel pass.
318 MF.getRegInfo().freezeReservedRegs();
319
320 // We create the pass manager, run the passes to populate AsmBuffer.
321 MCContext &MCContext = MMIWP->getMMI().getContext();
322 legacy::PassManager PM;
323
324 TargetLibraryInfoImpl TLII(Module->getTargetTriple());
325 PM.add(P: new TargetLibraryInfoWrapperPass(TLII));
326
327 TargetPassConfig *TPC = TM->createPassConfig(PM);
328 PM.add(P: TPC);
329 PM.add(P: MMIWP.release());
330 TPC->printAndVerify(Banner: "MachineFunctionGenerator::assemble");
331 // Add target-specific passes.
332 ET.addTargetSpecificPasses(PM);
333 TPC->printAndVerify(Banner: "After ExegesisTarget::addTargetSpecificPasses");
334 // Adding the following passes:
335 // - post-ra-pseudos: expands pseudo return instructions used on some targets.
336 // - prologepilog: saves and restore callee saved registers.
337 for (const char *PassName : {"post-ra-pseudos", "prolog-epilog"})
338 if (addPass(PM, PassName, TPC&: *TPC))
339 return make_error<Failure>(Args: "Unable to add a mandatory pass");
340 TPC->setInitialized();
341
342 // AsmPrinter is responsible for generating the assembly into AsmBuffer.
343 if (TM->addAsmPrinter(PM, Out&: AsmStream, DwoOut: nullptr, FileType: CodeGenFileType::ObjectFile,
344 Context&: MCContext))
345 return make_error<Failure>(Args: "Cannot add AsmPrinter passes");
346
347 PM.run(M&: *Module); // Run all the passes
348 bool MFWellFormed =
349 MF.verify(p: nullptr, Banner: "llvm-exegesis Assembly", OS: &outs(), AbortOnError: false);
350 if (!MFWellFormed)
351 return make_error<Failure>(Args: "The machine function failed verification.");
352 return Error::success();
353}
354
355object::OwningBinary<object::ObjectFile>
356getObjectFromBuffer(StringRef InputData) {
357 // Storing the generated assembly into a MemoryBuffer that owns the memory.
358 std::unique_ptr<MemoryBuffer> Buffer =
359 MemoryBuffer::getMemBufferCopy(InputData);
360 // Create the ObjectFile from the MemoryBuffer.
361 std::unique_ptr<object::ObjectFile> Obj =
362 cantFail(ValOrErr: object::ObjectFile::createObjectFile(Object: Buffer->getMemBufferRef()));
363 // Returning both the MemoryBuffer and the ObjectFile.
364 return object::OwningBinary<object::ObjectFile>(std::move(Obj),
365 std::move(Buffer));
366}
367
368object::OwningBinary<object::ObjectFile> getObjectFromFile(StringRef Filename) {
369 return cantFail(ValOrErr: object::ObjectFile::createObjectFile(ObjectPath: Filename));
370}
371
372Expected<ExecutableFunction> ExecutableFunction::create(
373 std::unique_ptr<TargetMachine> TM,
374 object::OwningBinary<object::ObjectFile> &&ObjectFileHolder) {
375 assert(ObjectFileHolder.getBinary() && "cannot create object file");
376 std::unique_ptr<LLVMContext> Ctx = std::make_unique<LLVMContext>();
377
378 auto SymbolSizes = object::computeSymbolSizes(O: *ObjectFileHolder.getBinary());
379 // Get the size of the function that we want to call into (with the name of
380 // FunctionID).
381 auto SymbolIt = find_if(Range&: SymbolSizes, P: [&](const auto &Pair) {
382 auto SymbolName = Pair.first.getName();
383 if (SymbolName)
384 return *SymbolName == FunctionID;
385 // We should always succeed in finding the FunctionID, hence we suppress
386 // the error here and assert later on the search result, rather than
387 // propagating the Expected<> error back to the caller.
388 consumeError(SymbolName.takeError());
389 return false;
390 });
391 assert(SymbolIt != SymbolSizes.end() &&
392 "Cannot find the symbol for FunctionID");
393 uintptr_t CodeSize = SymbolIt->second;
394
395 auto EJITOrErr =
396 orc::LLJITBuilder()
397 .setDataLayout(DataLayout(TM->getTargetTriple().computeDataLayout(
398 ABIName: TM->Options.MCOptions.getABIName())))
399 .create();
400 if (!EJITOrErr)
401 return EJITOrErr.takeError();
402
403 auto EJIT = std::move(*EJITOrErr);
404
405 if (auto ObjErr =
406 EJIT->addObjectFile(Obj: std::get<1>(in: ObjectFileHolder.takeBinary())))
407 return std::move(ObjErr);
408
409 auto FunctionAddressOrErr = EJIT->lookup(UnmangledName: FunctionID);
410 if (!FunctionAddressOrErr)
411 return FunctionAddressOrErr.takeError();
412
413 const uint64_t FunctionAddress = FunctionAddressOrErr->getValue();
414
415 assert(isAligned(kFunctionAlignment, FunctionAddress) &&
416 "function is not properly aligned");
417
418 StringRef FBytes =
419 StringRef(reinterpret_cast<const char *>(FunctionAddress), CodeSize);
420 return ExecutableFunction(std::move(Ctx), std::move(EJIT), FBytes);
421}
422
423ExecutableFunction::ExecutableFunction(std::unique_ptr<LLVMContext> Ctx,
424 std::unique_ptr<orc::LLJIT> EJIT,
425 StringRef FB)
426 : FunctionBytes(FB), Context(std::move(Ctx)), ExecJIT(std::move(EJIT)) {}
427
428Error getBenchmarkFunctionBytes(const StringRef InputData,
429 std::vector<uint8_t> &Bytes) {
430 const auto Holder = getObjectFromBuffer(InputData);
431 const auto *Obj = Holder.getBinary();
432 // See RuntimeDyldImpl::loadObjectImpl(Obj) for much more complete
433 // implementation.
434
435 // Find the only function in the object file.
436 SmallVector<object::SymbolRef, 1> Functions;
437 for (auto &Sym : Obj->symbols()) {
438 auto SymType = Sym.getType();
439 if (SymType && *SymType == object::SymbolRef::Type::ST_Function)
440 Functions.push_back(Elt: Sym);
441 }
442 if (Functions.size() != 1)
443 return make_error<Failure>(Args: "Exactly one function expected");
444
445 // Find the containing section - it is assumed to contain only this function.
446 auto SectionOrErr = Functions.front().getSection();
447 if (!SectionOrErr || *SectionOrErr == Obj->section_end())
448 return make_error<Failure>(Args: "Section not found");
449
450 auto Address = Functions.front().getAddress();
451 if (!Address || *Address != SectionOrErr.get()->getAddress())
452 return make_error<Failure>(Args: "Unexpected layout");
453
454 auto ContentsOrErr = SectionOrErr.get()->getContents();
455 if (!ContentsOrErr)
456 return ContentsOrErr.takeError();
457 Bytes.assign(first: ContentsOrErr->begin(), last: ContentsOrErr->end());
458 return Error::success();
459}
460
461} // namespace exegesis
462} // namespace llvm
463