1//===-- SnippetRepetitor.cpp ------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SnippetRepetitor.h"
10#include "Target.h"
11#include "llvm/ADT/Sequence.h"
12#include "llvm/CodeGen/TargetInstrInfo.h"
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/CodeGen/TargetSubtargetInfo.h"
15
16namespace llvm {
17namespace exegesis {
18namespace {
19
20class DuplicateSnippetRepetitor : public SnippetRepetitor {
21public:
22 using SnippetRepetitor::SnippetRepetitor;
23
24 // Repeats the snippet until there are at least MinInstructions in the
25 // resulting code.
26 FillFunction Repeat(ArrayRef<MCInst> Instructions, unsigned MinInstructions,
27 unsigned LoopBodySize,
28 bool CleanupMemory) const override {
29 return [this, Instructions, MinInstructions,
30 CleanupMemory](FunctionFiller &Filler) {
31 auto Entry = Filler.getEntry();
32 if (!Instructions.empty()) {
33 const unsigned NumRepetitions =
34 divideCeil(Numerator: MinInstructions, Denominator: Instructions.size());
35 for (unsigned I = 0; I < NumRepetitions; ++I) {
36 Entry.addInstructions(Insts: Instructions);
37 }
38 }
39 Entry.addReturn(ET: State.getExegesisTarget(), SubprocessCleanup: CleanupMemory);
40 };
41 }
42
43 BitVector getReservedRegs() const override {
44 // We're using no additional registers.
45 return State.getRATC().emptyRegisters();
46 }
47};
48
49class LoopSnippetRepetitor : public SnippetRepetitor {
50public:
51 explicit LoopSnippetRepetitor(const LLVMState &State, unsigned LoopRegister)
52 : SnippetRepetitor(State), LoopCounter(LoopRegister) {}
53
54 // Loop over the snippet ceil(MinInstructions / Instructions.Size()) times.
55 FillFunction Repeat(ArrayRef<MCInst> Instructions, unsigned MinInstructions,
56 unsigned LoopBodySize,
57 bool CleanupMemory) const override {
58 return [this, Instructions, MinInstructions, LoopBodySize,
59 CleanupMemory](FunctionFiller &Filler) {
60 const auto &ET = State.getExegesisTarget();
61 auto Entry = Filler.getEntry();
62
63 // We can not use loop snippet repetitor for terminator instructions.
64 for (const MCInst &Inst : Instructions) {
65 const unsigned Opcode = Inst.getOpcode();
66 const MCInstrDesc &MCID = Filler.MCII->get(Opcode);
67 if (!MCID.isTerminator())
68 continue;
69 Entry.addReturn(ET: State.getExegesisTarget(), SubprocessCleanup: CleanupMemory);
70 return;
71 }
72
73 auto Loop = Filler.addBasicBlock();
74 auto Exit = Filler.addBasicBlock();
75
76 // Align the loop machine basic block to a target-specific boundary
77 // to promote optimal instruction fetch/predecoding conditions.
78 Loop.MBB->setAlignment(
79 Filler.MF.getSubtarget().getTargetLowering()->getPrefLoopAlignment());
80
81 const unsigned LoopUnrollFactor =
82 LoopBodySize <= Instructions.size()
83 ? 1
84 : divideCeil(Numerator: LoopBodySize, Denominator: Instructions.size());
85 assert(LoopUnrollFactor >= 1 && "Should end up with at least 1 snippet.");
86
87 // Set loop counter to the right value:
88 const APInt LoopCount(
89 32,
90 divideCeil(Numerator: MinInstructions, Denominator: LoopUnrollFactor * Instructions.size()));
91 assert(LoopCount.uge(1) && "Trip count should be at least 1.");
92 for (const MCInst &Inst :
93 ET.setRegTo(STI: State.getSubtargetInfo(), Reg: LoopCounter, Value: LoopCount))
94 Entry.addInstruction(Inst);
95
96 // Set up the loop basic block.
97 Entry.MBB->addSuccessor(Succ: Loop.MBB, Prob: BranchProbability::getOne());
98 Loop.MBB->addSuccessor(Succ: Loop.MBB, Prob: BranchProbability::getOne());
99 // If the snippet setup completed, then we can track liveness.
100 if (Loop.MF.getProperties().hasProperty(
101 P: MachineFunctionProperties::Property::TracksLiveness)) {
102 // The live ins are: the loop counter, the registers that were setup by
103 // the entry block, and entry block live ins.
104 Loop.MBB->addLiveIn(PhysReg: LoopCounter);
105 for (unsigned Reg : Filler.getRegistersSetUp())
106 Loop.MBB->addLiveIn(PhysReg: Reg);
107 for (const auto &LiveIn : Entry.MBB->liveins())
108 Loop.MBB->addLiveIn(RegMaskPair: LiveIn);
109 }
110 for (auto _ : seq(Size: LoopUnrollFactor)) {
111 (void)_;
112 Loop.addInstructions(Insts: Instructions);
113 }
114 ET.decrementLoopCounterAndJump(MBB&: *Loop.MBB, TargetMBB&: *Loop.MBB, MII: State.getInstrInfo(),
115 LoopRegister: LoopCounter);
116
117 // Set up the exit basic block.
118 Loop.MBB->addSuccessor(Succ: Exit.MBB, Prob: BranchProbability::getZero());
119 Exit.addReturn(ET: State.getExegesisTarget(), SubprocessCleanup: CleanupMemory);
120 };
121 }
122
123 BitVector getReservedRegs() const override {
124 // We're using a single loop counter, but we have to reserve all aliasing
125 // registers.
126 return State.getRATC().getRegister(Reg: LoopCounter).aliasedBits();
127 }
128
129private:
130 const unsigned LoopCounter;
131};
132
133} // namespace
134
135SnippetRepetitor::~SnippetRepetitor() {}
136
137std::unique_ptr<const SnippetRepetitor>
138SnippetRepetitor::Create(Benchmark::RepetitionModeE Mode,
139 const LLVMState &State, unsigned LoopRegister) {
140 switch (Mode) {
141 case Benchmark::Duplicate:
142 case Benchmark::MiddleHalfDuplicate:
143 return std::make_unique<DuplicateSnippetRepetitor>(args: State);
144 case Benchmark::Loop:
145 case Benchmark::MiddleHalfLoop:
146 return std::make_unique<LoopSnippetRepetitor>(args: State, args&: LoopRegister);
147 case Benchmark::AggregateMin:
148 break;
149 }
150 llvm_unreachable("Unknown RepetitionModeE enum");
151}
152
153} // namespace exegesis
154} // namespace llvm
155