1//===-- Assembler.cpp -------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Assembler.h"
10
11#include "SnippetRepetitor.h"
12#include "SubprocessMemory.h"
13#include "Target.h"
14#include "llvm/Analysis/TargetLibraryInfo.h"
15#include "llvm/CodeGen/FunctionLoweringInfo.h"
16#include "llvm/CodeGen/GlobalISel/CallLowering.h"
17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
18#include "llvm/CodeGen/MachineInstrBuilder.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/TargetInstrInfo.h"
22#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetPassConfig.h"
24#include "llvm/CodeGen/TargetSubtargetInfo.h"
25#include "llvm/ExecutionEngine/Orc/LLJIT.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/LegacyPassManager.h"
29#include "llvm/MC/MCInstrInfo.h"
30#include "llvm/Object/SymbolSize.h"
31#include "llvm/Support/Alignment.h"
32#include "llvm/Support/Error.h"
33#include "llvm/Support/MemoryBuffer.h"
34#include "llvm/Support/raw_ostream.h"
35
36#ifdef HAVE_LIBPFM
37#include "perfmon/perf_event.h"
38#endif // HAVE_LIBPFM
39
40#ifdef __linux__
41#include <unistd.h>
42#endif
43
44namespace llvm {
45namespace exegesis {
46
47static constexpr char ModuleID[] = "ExegesisInfoTest";
48static constexpr char FunctionID[] = "foo";
49static const Align kFunctionAlignment(4096);
50
51// Fills the given basic block with register setup code, and returns true if
52// all registers could be setup correctly.
53static bool generateSnippetSetupCode(const ExegesisTarget &ET,
54 const MCSubtargetInfo *const MSI,
55 BasicBlockFiller &BBF,
56 const BenchmarkKey &Key,
57 bool GenerateMemoryInstructions) {
58 bool IsSnippetSetupComplete = true;
59 if (GenerateMemoryInstructions) {
60 BBF.addInstructions(Insts: ET.generateMemoryInitialSetup());
61 for (const MemoryMapping &MM : Key.MemoryMappings) {
62#ifdef __linux__
63 // The frontend that generates that parses the memory mapping information
64 // from the user should validate that the requested address is a multiple
65 // of the page size. Assert that this is true here.
66 assert(MM.Address % getpagesize() == 0 &&
67 "Memory mappings need to be aligned to page boundaries.");
68#endif
69 const MemoryValue &MemVal = Key.MemoryValues.at(Val: MM.MemoryValueName);
70 BBF.addInstructions(Insts: ET.generateMmap(
71 Address: MM.Address, Length: MemVal.SizeBytes,
72 FileDescriptorAddress: ET.getAuxiliaryMemoryStartAddress() +
73 sizeof(int) *
74 (MemVal.Index + SubprocessMemory::AuxiliaryMemoryOffset)));
75 }
76 BBF.addInstructions(Insts: ET.setStackRegisterToAuxMem());
77 }
78 Register StackPointerRegister = BBF.MF.getSubtarget()
79 .getTargetLowering()
80 ->getStackPointerRegisterToSaveRestore();
81 for (const RegisterValue &RV : Key.RegisterInitialValues) {
82 if (GenerateMemoryInstructions) {
83 // If we're generating memory instructions, don't load in the value for
84 // the register with the stack pointer as it will be used later to finish
85 // the setup.
86 if (Register(RV.Register) == StackPointerRegister)
87 continue;
88 }
89 // Load a constant in the register.
90 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
91 if (SetRegisterCode.empty())
92 IsSnippetSetupComplete = false;
93 BBF.addInstructions(Insts: SetRegisterCode);
94 }
95 if (GenerateMemoryInstructions) {
96#ifdef HAVE_LIBPFM
97 BBF.addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_RESET, true));
98#endif // HAVE_LIBPFM
99 for (const RegisterValue &RV : Key.RegisterInitialValues) {
100 // Load in the stack register now as we're done using it elsewhere
101 // and need to set the value in preparation for executing the
102 // snippet.
103 if (Register(RV.Register) != StackPointerRegister)
104 continue;
105 const auto SetRegisterCode = ET.setRegTo(STI: *MSI, Reg: RV.Register, Value: RV.Value);
106 if (SetRegisterCode.empty())
107 IsSnippetSetupComplete = false;
108 BBF.addInstructions(Insts: SetRegisterCode);
109 break;
110 }
111 }
112 return IsSnippetSetupComplete;
113}
114
115// Small utility function to add named passes.
116static bool addPass(PassManagerBase &PM, StringRef PassName,
117 TargetPassConfig &TPC) {
118 const PassRegistry *PR = PassRegistry::getPassRegistry();
119 const PassInfo *PI = PR->getPassInfo(Arg: PassName);
120 if (!PI) {
121 errs() << " run-pass " << PassName << " is not registered.\n";
122 return true;
123 }
124
125 if (!PI->getNormalCtor()) {
126 errs() << " cannot create pass: " << PI->getPassName() << "\n";
127 return true;
128 }
129 Pass *P = PI->getNormalCtor()();
130 std::string Banner = std::string("After ") + std::string(P->getPassName());
131 PM.add(P);
132 TPC.printAndVerify(Banner);
133
134 return false;
135}
136
137MachineFunction &createVoidVoidPtrMachineFunction(StringRef FunctionName,
138 Module *Module,
139 MachineModuleInfo *MMI) {
140 Type *const ReturnType = Type::getInt32Ty(C&: Module->getContext());
141 Type *const MemParamType =
142 PointerType::get(C&: Module->getContext(), AddressSpace: 0 /*default address space*/);
143 FunctionType *FunctionType =
144 FunctionType::get(Result: ReturnType, Params: {MemParamType}, isVarArg: false);
145 Function *const F = Function::Create(
146 Ty: FunctionType, Linkage: GlobalValue::ExternalLinkage, N: FunctionName, M: Module);
147 BasicBlock *BB = BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: F);
148 new UnreachableInst(Module->getContext(), BB);
149 return MMI->getOrCreateMachineFunction(F&: *F);
150}
151
152BasicBlockFiller::BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB,
153 const MCInstrInfo *MCII)
154 : MF(MF), MBB(MBB), MCII(MCII) {}
155
156void BasicBlockFiller::addInstruction(const MCInst &Inst, const DebugLoc &DL) {
157 const unsigned Opcode = Inst.getOpcode();
158 const MCInstrDesc &MCID = MCII->get(Opcode);
159 MachineInstrBuilder Builder = BuildMI(BB: MBB, MIMD: DL, MCID);
160 for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;
161 ++OpIndex) {
162 const MCOperand &Op = Inst.getOperand(i: OpIndex);
163 if (Op.isReg()) {
164 const bool IsDef = OpIndex < MCID.getNumDefs();
165 RegState Flags = {};
166 const MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];
167 if (IsDef && !OpInfo.isOptionalDef())
168 Flags |= RegState::Define;
169 Builder.addReg(RegNo: Op.getReg(), Flags);
170 } else if (Op.isImm()) {
171 Builder.addImm(Val: Op.getImm());
172 } else if (!Op.isValid()) {
173 llvm_unreachable("Operand is not set");
174 } else {
175 llvm_unreachable("Not yet implemented");
176 }
177 }
178}
179
180void BasicBlockFiller::addInstructions(ArrayRef<MCInst> Insts,
181 const DebugLoc &DL) {
182 for (const MCInst &Inst : Insts)
183 addInstruction(Inst, DL);
184}
185
186void BasicBlockFiller::addReturn(const ExegesisTarget &ET,
187 bool SubprocessCleanup, const DebugLoc &DL) {
188 // Insert cleanup code
189 if (SubprocessCleanup) {
190#ifdef HAVE_LIBPFM
191 addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_DISABLE, false));
192#endif // HAVE_LIBPFM
193#ifdef __linux__
194 addInstructions(Insts: ET.generateExitSyscall(ExitCode: 0));
195#endif // __linux__
196 }
197 // Insert the return code.
198 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
199 if (TII->getReturnOpcode() < TII->getNumOpcodes()) {
200 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TII->getReturnOpcode()));
201 } else {
202 MachineIRBuilder MIB(MF);
203 MIB.setMBB(*MBB);
204
205 FunctionLoweringInfo FuncInfo;
206 FuncInfo.CanLowerReturn = true;
207 MF.getSubtarget().getCallLowering()->lowerReturn(MIRBuilder&: MIB, Val: nullptr, VRegs: {}, FLI&: FuncInfo,
208 SwiftErrorVReg: 0);
209 }
210}
211
212FunctionFiller::FunctionFiller(MachineFunction &MF,
213 std::vector<MCRegister> RegistersSetUp)
214 : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()),
215 RegistersSetUp(std::move(RegistersSetUp)) {}
216
217BasicBlockFiller FunctionFiller::addBasicBlock() {
218 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
219 MF.push_back(MBB);
220 return BasicBlockFiller(MF, MBB, MCII);
221}
222
223ArrayRef<MCRegister> FunctionFiller::getRegistersSetUp() const {
224 return RegistersSetUp;
225}
226
227static std::unique_ptr<Module>
228createModule(const std::unique_ptr<LLVMContext> &Context,
229 const TargetMachine &TM) {
230 auto Mod = std::make_unique<Module>(args: ModuleID, args&: *Context);
231 const Triple &TT = TM.getTargetTriple();
232 Mod->setTargetTriple(TT);
233 StringRef ABIName = TM.Options.MCOptions.getABIName();
234 if (!ABIName.empty()) {
235 Mod->addModuleFlag(Behavior: Module::Error, Key: "target-abi",
236 Val: MDString::get(Context&: *Context, Str: ABIName));
237 }
238
239 Mod->setDataLayout(DataLayout(TT.computeDataLayout(ABIName)));
240 return Mod;
241}
242
243BitVector getFunctionReservedRegs(const TargetMachine &TM) {
244 std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>();
245 std::unique_ptr<Module> Module = createModule(Context, TM);
246 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: &TM);
247 MachineFunction &MF = createVoidVoidPtrMachineFunction(
248 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP->getMMI());
249 // Saving reserved registers for client.
250 return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF);
251}
252
253Error assembleToStream(const ExegesisTarget &ET,
254 std::unique_ptr<TargetMachine> TM,
255 ArrayRef<MCRegister> LiveIns, const FillFunction &Fill,
256 raw_pwrite_stream &AsmStream, const BenchmarkKey &Key,
257 bool GenerateMemoryInstructions) {
258 auto Context = std::make_unique<LLVMContext>();
259 std::unique_ptr<Module> Module = createModule(Context, TM: *TM);
260 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(args: TM.get());
261 MachineFunction &MF = createVoidVoidPtrMachineFunction(
262 FunctionName: FunctionID, Module: Module.get(), MMI: &MMIWP.get()->getMMI());
263 MF.ensureAlignment(A: kFunctionAlignment);
264
265 // We need to instruct the passes that we're done with SSA and virtual
266 // registers.
267 auto &Properties = MF.getProperties();
268 Properties.setNoVRegs().resetIsSSA().setNoPHIs();
269
270 for (const MCRegister Reg : LiveIns)
271 MF.getRegInfo().addLiveIn(Reg);
272
273 if (GenerateMemoryInstructions) {
274 for (const MCRegister Reg : ET.getArgumentRegisters())
275 MF.getRegInfo().addLiveIn(Reg);
276 // Add a live in for registers that need saving so that the machine verifier
277 // doesn't fail if the register is never defined.
278 for (const MCRegister Reg : ET.getRegistersNeedSaving())
279 MF.getRegInfo().addLiveIn(Reg);
280 }
281
282 std::vector<MCRegister> RegistersSetUp;
283 RegistersSetUp.reserve(n: Key.RegisterInitialValues.size());
284 for (const auto &InitValue : Key.RegisterInitialValues) {
285 RegistersSetUp.push_back(x: InitValue.Register);
286 }
287 FunctionFiller Sink(MF, std::move(RegistersSetUp));
288 auto Entry = Sink.getEntry();
289
290 for (const MCRegister Reg : LiveIns)
291 Entry.MBB->addLiveIn(PhysReg: Reg);
292
293 if (GenerateMemoryInstructions) {
294 for (const MCRegister Reg : ET.getArgumentRegisters())
295 Entry.MBB->addLiveIn(PhysReg: Reg);
296 // Add a live in for registers that need saving so that the machine verifier
297 // doesn't fail if the register is never defined.
298 for (const MCRegister Reg : ET.getRegistersNeedSaving())
299 Entry.MBB->addLiveIn(PhysReg: Reg);
300 }
301
302 const bool IsSnippetSetupComplete = generateSnippetSetupCode(
303 ET, MSI: &TM->getMCSubtargetInfo(), BBF&: Entry, Key, GenerateMemoryInstructions);
304
305 // If the snippet setup is not complete, we disable liveliness tracking. This
306 // means that we won't know what values are in the registers.
307 // FIXME: this should probably be an assertion.
308 if (!IsSnippetSetupComplete)
309 Properties.resetTracksLiveness();
310
311 Fill(Sink);
312
313 // prologue/epilogue pass needs the reserved registers to be frozen, this
314 // is usually done by the SelectionDAGISel pass.
315 MF.getRegInfo().freezeReservedRegs();
316
317 // We create the pass manager, run the passes to populate AsmBuffer.
318 MCContext &MCContext = MMIWP->getMMI().getContext();
319 legacy::PassManager PM;
320
321 TargetLibraryInfoImpl TLII(Module->getTargetTriple());
322 PM.add(P: new TargetLibraryInfoWrapperPass(TLII));
323
324 TargetPassConfig *TPC = TM->createPassConfig(PM);
325 PM.add(P: TPC);
326 PM.add(P: MMIWP.release());
327 TPC->printAndVerify(Banner: "MachineFunctionGenerator::assemble");
328 // Add target-specific passes.
329 ET.addTargetSpecificPasses(PM);
330 TPC->printAndVerify(Banner: "After ExegesisTarget::addTargetSpecificPasses");
331 // Adding the following passes:
332 // - post-ra-pseudos: expands pseudo return instructions used on some targets.
333 // - prologepilog: saves and restore callee saved registers.
334 for (const char *PassName : {"post-ra-pseudos", "prolog-epilog"})
335 if (addPass(PM, PassName, TPC&: *TPC))
336 return make_error<Failure>(Args: "Unable to add a mandatory pass");
337 TPC->setInitialized();
338
339 // AsmPrinter is responsible for generating the assembly into AsmBuffer.
340 if (TM->addAsmPrinter(PM, Out&: AsmStream, DwoOut: nullptr, FileType: CodeGenFileType::ObjectFile,
341 Context&: MCContext))
342 return make_error<Failure>(Args: "Cannot add AsmPrinter passes");
343
344 PM.run(M&: *Module); // Run all the passes
345 bool MFWellFormed =
346 MF.verify(p: nullptr, Banner: "llvm-exegesis Assembly", OS: &outs(), AbortOnError: false);
347 if (!MFWellFormed)
348 return make_error<Failure>(Args: "The machine function failed verification.");
349 return Error::success();
350}
351
352object::OwningBinary<object::ObjectFile>
353getObjectFromBuffer(StringRef InputData) {
354 // Storing the generated assembly into a MemoryBuffer that owns the memory.
355 std::unique_ptr<MemoryBuffer> Buffer =
356 MemoryBuffer::getMemBufferCopy(InputData);
357 // Create the ObjectFile from the MemoryBuffer.
358 std::unique_ptr<object::ObjectFile> Obj =
359 cantFail(ValOrErr: object::ObjectFile::createObjectFile(Object: Buffer->getMemBufferRef()));
360 // Returning both the MemoryBuffer and the ObjectFile.
361 return object::OwningBinary<object::ObjectFile>(std::move(Obj),
362 std::move(Buffer));
363}
364
365object::OwningBinary<object::ObjectFile> getObjectFromFile(StringRef Filename) {
366 return cantFail(ValOrErr: object::ObjectFile::createObjectFile(ObjectPath: Filename));
367}
368
369Expected<ExecutableFunction> ExecutableFunction::create(
370 std::unique_ptr<TargetMachine> TM,
371 object::OwningBinary<object::ObjectFile> &&ObjectFileHolder) {
372 assert(ObjectFileHolder.getBinary() && "cannot create object file");
373 std::unique_ptr<LLVMContext> Ctx = std::make_unique<LLVMContext>();
374
375 auto SymbolSizes = object::computeSymbolSizes(O: *ObjectFileHolder.getBinary());
376 // Get the size of the function that we want to call into (with the name of
377 // FunctionID).
378 auto SymbolIt = find_if(Range&: SymbolSizes, P: [&](const auto &Pair) {
379 auto SymbolName = Pair.first.getName();
380 if (SymbolName)
381 return *SymbolName == FunctionID;
382 // We should always succeed in finding the FunctionID, hence we suppress
383 // the error here and assert later on the search result, rather than
384 // propagating the Expected<> error back to the caller.
385 consumeError(SymbolName.takeError());
386 return false;
387 });
388 assert(SymbolIt != SymbolSizes.end() &&
389 "Cannot find the symbol for FunctionID");
390 uintptr_t CodeSize = SymbolIt->second;
391
392 auto EJITOrErr =
393 orc::LLJITBuilder()
394 .setDataLayout(DataLayout(TM->getTargetTriple().computeDataLayout(
395 ABIName: TM->Options.MCOptions.getABIName())))
396 .create();
397 if (!EJITOrErr)
398 return EJITOrErr.takeError();
399
400 auto EJIT = std::move(*EJITOrErr);
401
402 if (auto ObjErr =
403 EJIT->addObjectFile(Obj: std::get<1>(in: ObjectFileHolder.takeBinary())))
404 return std::move(ObjErr);
405
406 auto FunctionAddressOrErr = EJIT->lookup(UnmangledName: FunctionID);
407 if (!FunctionAddressOrErr)
408 return FunctionAddressOrErr.takeError();
409
410 const uint64_t FunctionAddress = FunctionAddressOrErr->getValue();
411
412 assert(isAligned(kFunctionAlignment, FunctionAddress) &&
413 "function is not properly aligned");
414
415 StringRef FBytes =
416 StringRef(reinterpret_cast<const char *>(FunctionAddress), CodeSize);
417 return ExecutableFunction(std::move(Ctx), std::move(EJIT), FBytes);
418}
419
420ExecutableFunction::ExecutableFunction(std::unique_ptr<LLVMContext> Ctx,
421 std::unique_ptr<orc::LLJIT> EJIT,
422 StringRef FB)
423 : FunctionBytes(FB), Context(std::move(Ctx)), ExecJIT(std::move(EJIT)) {}
424
425Error getBenchmarkFunctionBytes(const StringRef InputData,
426 std::vector<uint8_t> &Bytes) {
427 const auto Holder = getObjectFromBuffer(InputData);
428 const auto *Obj = Holder.getBinary();
429 // See RuntimeDyldImpl::loadObjectImpl(Obj) for much more complete
430 // implementation.
431
432 // Find the only function in the object file.
433 SmallVector<object::SymbolRef, 1> Functions;
434 for (auto &Sym : Obj->symbols()) {
435 auto SymType = Sym.getType();
436 if (SymType && *SymType == object::SymbolRef::Type::ST_Function)
437 Functions.push_back(Elt: Sym);
438 }
439 if (Functions.size() != 1)
440 return make_error<Failure>(Args: "Exactly one function expected");
441
442 // Find the containing section - it is assumed to contain only this function.
443 auto SectionOrErr = Functions.front().getSection();
444 if (!SectionOrErr || *SectionOrErr == Obj->section_end())
445 return make_error<Failure>(Args: "Section not found");
446
447 auto Address = Functions.front().getAddress();
448 if (!Address || *Address != SectionOrErr.get()->getAddress())
449 return make_error<Failure>(Args: "Unexpected layout");
450
451 auto ContentsOrErr = SectionOrErr.get()->getContents();
452 if (!ContentsOrErr)
453 return ContentsOrErr.takeError();
454 Bytes.assign(first: ContentsOrErr->begin(), last: ContentsOrErr->end());
455 return Error::success();
456}
457
458} // namespace exegesis
459} // namespace llvm
460