1//===-- SnippetRepetitor.cpp ------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SnippetRepetitor.h"
10#include "Target.h"
11#include "llvm/ADT/Sequence.h"
12#include "llvm/CodeGen/TargetInstrInfo.h"
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/CodeGen/TargetSubtargetInfo.h"
15
16namespace llvm {
17namespace exegesis {
18namespace {
19
20class DuplicateSnippetRepetitor : public SnippetRepetitor {
21public:
22 using SnippetRepetitor::SnippetRepetitor;
23
24 // Repeats the snippet until there are at least MinInstructions in the
25 // resulting code.
26 FillFunction Repeat(ArrayRef<MCInst> Instructions, unsigned MinInstructions,
27 unsigned LoopBodySize,
28 bool CleanupMemory) const override {
29 return [this, Instructions, MinInstructions,
30 CleanupMemory](FunctionFiller &Filler) {
31 auto Entry = Filler.getEntry();
32 if (!Instructions.empty()) {
33 const unsigned NumRepetitions =
34 divideCeil(Numerator: MinInstructions, Denominator: Instructions.size());
35 for (unsigned I = 0; I < NumRepetitions; ++I) {
36 Entry.addInstructions(Insts: Instructions);
37 }
38 }
39 Entry.addReturn(ET: State.getExegesisTarget(), SubprocessCleanup: CleanupMemory);
40 };
41 }
42
43 BitVector getReservedRegs() const override {
44 // We're using no additional registers.
45 return State.getRATC().emptyRegisters();
46 }
47};
48
49class LoopSnippetRepetitor : public SnippetRepetitor {
50public:
51 explicit LoopSnippetRepetitor(const LLVMState &State, MCRegister LoopRegister)
52 : SnippetRepetitor(State), LoopCounter(LoopRegister) {}
53
54 // Loop over the snippet ceil(MinInstructions / Instructions.Size()) times.
55 FillFunction Repeat(ArrayRef<MCInst> Instructions, unsigned MinInstructions,
56 unsigned LoopBodySize,
57 bool CleanupMemory) const override {
58 return [this, Instructions, MinInstructions, LoopBodySize,
59 CleanupMemory](FunctionFiller &Filler) {
60 const auto &ET = State.getExegesisTarget();
61 auto Entry = Filler.getEntry();
62
63 // We can not use loop snippet repetitor for terminator instructions.
64 for (const MCInst &Inst : Instructions) {
65 const unsigned Opcode = Inst.getOpcode();
66 const MCInstrDesc &MCID = Filler.MCII->get(Opcode);
67 if (!MCID.isTerminator())
68 continue;
69 Entry.addReturn(ET: State.getExegesisTarget(), SubprocessCleanup: CleanupMemory);
70 return;
71 }
72
73 auto Loop = Filler.addBasicBlock();
74 auto Exit = Filler.addBasicBlock();
75
76 // Align the loop machine basic block to a target-specific boundary
77 // to promote optimal instruction fetch/predecoding conditions.
78 Loop.MBB->setAlignment(
79 Filler.MF.getSubtarget().getTargetLowering()->getPrefLoopAlignment());
80
81 const unsigned LoopUnrollFactor =
82 LoopBodySize <= Instructions.size()
83 ? 1
84 : divideCeil(Numerator: LoopBodySize, Denominator: Instructions.size());
85 assert(LoopUnrollFactor >= 1 && "Should end up with at least 1 snippet.");
86
87 // Set loop counter to the right value:
88 const APInt LoopCount(
89 32,
90 divideCeil(Numerator: MinInstructions, Denominator: LoopUnrollFactor * Instructions.size()));
91 assert(LoopCount.uge(1) && "Trip count should be at least 1.");
92 for (const MCInst &Inst :
93 ET.setRegTo(STI: State.getSubtargetInfo(), Reg: LoopCounter, Value: LoopCount))
94 Entry.addInstruction(Inst);
95
96 // Set up the loop basic block.
97 Entry.MBB->addSuccessor(Succ: Loop.MBB, Prob: BranchProbability::getOne());
98 Loop.MBB->addSuccessor(Succ: Loop.MBB, Prob: BranchProbability::getOne());
99 // If the snippet setup completed, then we can track liveness.
100 if (Loop.MF.getProperties().hasTracksLiveness()) {
101 // The live ins are: the loop counter, the registers that were setup by
102 // the entry block, and entry block live ins.
103 Loop.MBB->addLiveIn(PhysReg: LoopCounter);
104 for (MCRegister Reg : Filler.getRegistersSetUp())
105 Loop.MBB->addLiveIn(PhysReg: Reg);
106 for (const auto &LiveIn : Entry.MBB->liveins())
107 Loop.MBB->addLiveIn(RegMaskPair: LiveIn);
108 }
109 for (auto _ : seq(Size: LoopUnrollFactor)) {
110 (void)_;
111 Loop.addInstructions(Insts: Instructions);
112 }
113 ET.decrementLoopCounterAndJump(MBB&: *Loop.MBB, TargetMBB&: *Loop.MBB, MII: State.getInstrInfo(),
114 LoopRegister: LoopCounter);
115
116 // Set up the exit basic block.
117 Loop.MBB->addSuccessor(Succ: Exit.MBB, Prob: BranchProbability::getZero());
118 Exit.addReturn(ET: State.getExegesisTarget(), SubprocessCleanup: CleanupMemory);
119 };
120 }
121
122 BitVector getReservedRegs() const override {
123 // We're using a single loop counter, but we have to reserve all aliasing
124 // registers.
125 return State.getRATC().getRegister(Reg: LoopCounter).aliasedBits();
126 }
127
128private:
129 const MCRegister LoopCounter;
130};
131
132} // namespace
133
134SnippetRepetitor::~SnippetRepetitor() {}
135
136std::unique_ptr<const SnippetRepetitor>
137SnippetRepetitor::Create(Benchmark::RepetitionModeE Mode,
138 const LLVMState &State, MCRegister LoopRegister) {
139 switch (Mode) {
140 case Benchmark::Duplicate:
141 case Benchmark::MiddleHalfDuplicate:
142 return std::make_unique<DuplicateSnippetRepetitor>(args: State);
143 case Benchmark::Loop:
144 case Benchmark::MiddleHalfLoop:
145 return std::make_unique<LoopSnippetRepetitor>(args: State, args&: LoopRegister);
146 case Benchmark::AggregateMin:
147 break;
148 }
149 llvm_unreachable("Unknown RepetitionModeE enum");
150}
151
152} // namespace exegesis
153} // namespace llvm
154