1//===-- Target.cpp ----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#include "../Target.h"
9#include "AArch64.h"
10#include "AArch64RegisterInfo.h"
11#include "llvm/CodeGen/MachineInstrBuilder.h"
12
13#if defined(__aarch64__) && defined(__linux__)
14#include <sys/prctl.h> // For PR_PAC_* constants
15#ifndef PR_PAC_APIAKEY
16#define PR_PAC_APIAKEY (1UL << 0)
17#endif
18#ifndef PR_PAC_APIBKEY
19#define PR_PAC_APIBKEY (1UL << 1)
20#endif
21#ifndef PR_PAC_APDAKEY
22#define PR_PAC_APDAKEY (1UL << 2)
23#endif
24#ifndef PR_PAC_APDBKEY
25#define PR_PAC_APDBKEY (1UL << 3)
26#endif
27#endif
28
29#define GET_AVAILABLE_OPCODE_CHECKER
30#include "AArch64GenInstrInfo.inc"
31
32namespace llvm {
33namespace exegesis {
34
35static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
36 switch (RegBitWidth) {
37 case 32:
38 return AArch64::MOVi32imm;
39 case 64:
40 return AArch64::MOVi64imm;
41 }
42 llvm_unreachable("Invalid Value Width");
43}
44
45// Generates instruction to load an immediate value into a register.
46static MCInst loadImmediate(MCRegister Reg, unsigned RegBitWidth,
47 const APInt &Value) {
48 assert(Value.getBitWidth() <= RegBitWidth &&
49 "Value must fit in the Register");
50 return MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
51 .addReg(Reg)
52 .addImm(Val: Value.getZExtValue());
53}
54
55static MCInst loadZPRImmediate(MCRegister Reg, unsigned RegBitWidth,
56 const APInt &Value) {
57 assert(Value.getZExtValue() < (1 << 7) &&
58 "Value must be in the range of the immediate opcode");
59 return MCInstBuilder(AArch64::DUP_ZI_D)
60 .addReg(Reg)
61 .addImm(Val: Value.getZExtValue())
62 .addImm(Val: 0);
63}
64
65static MCInst loadPPRImmediate(MCRegister Reg, unsigned RegBitWidth,
66 const APInt &Value) {
67 // For PPR, we typically use PTRUE instruction to set predicate registers
68 return MCInstBuilder(AArch64::PTRUE_B)
69 .addReg(Reg)
70 .addImm(Val: 31); // All lanes true for 16 bits
71}
72
73static MCInst loadFFRImmediate(MCRegister Reg, unsigned RegBitWidth,
74 const APInt &Value) {
75 assert(Value.getZExtValue() == 0 && "Expected initialisation value 0");
76 // For first-fault register FFR, we set it to a true
77 return MCInstBuilder(AArch64::SETFFR);
78}
79
80// Generates instructions to load an immediate value into an FPCR register.
81static std::vector<MCInst>
82loadFPCRImmediate(MCRegister Reg, unsigned RegBitWidth, const APInt &Value) {
83 MCRegister TempReg = AArch64::X8;
84 MCInst LoadImm = MCInstBuilder(AArch64::MOVi64imm).addReg(Reg: TempReg).addImm(Val: 0);
85 MCInst MoveToFPCR =
86 MCInstBuilder(AArch64::MSR).addImm(Val: AArch64SysReg::FPCR).addReg(Reg: TempReg);
87 return {LoadImm, MoveToFPCR};
88}
89
90// Generates instructions to load an immediate value into a pair of W registers
91static std::vector<MCInst> loadWSeqPairImmediate(MCRegister Reg,
92 unsigned RegBitWidth,
93 const APInt &Value) {
94 MCRegister EvenReg = (Reg - AArch64::W0_W1) * 2 + AArch64::W0 + 0;
95 MCRegister OddReg = (Reg - AArch64::W0_W1) * 2 + AArch64::W0 + 1;
96 assert(Value.getBitWidth() <= RegBitWidth &&
97 "Value must fit in the Register");
98
99 MCInst LoadEven = MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
100 .addReg(Reg: EvenReg)
101 .addImm(Val: Value.getZExtValue());
102 MCInst LoadOdd = MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
103 .addReg(Reg: OddReg)
104 .addImm(Val: Value.getZExtValue());
105 return {LoadEven, LoadOdd};
106}
107
108// Generates instructions to load an immediate value into a pair of X registers
109static std::vector<MCInst> loadXSeqPairImmediate(MCRegister Reg,
110 unsigned RegBitWidth,
111 const APInt &Value) {
112 MCRegister EvenReg = (Reg - AArch64::X0_X1) * 2 + AArch64::X0 + 0;
113 MCRegister OddReg = (Reg - AArch64::X0_X1) * 2 + AArch64::X0 + 1;
114 assert(Value.getBitWidth() <= RegBitWidth &&
115 "Value must fit in the Register");
116
117 MCInst LoadEven = MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
118 .addReg(Reg: EvenReg)
119 .addImm(Val: Value.getZExtValue());
120 MCInst LoadOdd = MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
121 .addReg(Reg: OddReg)
122 .addImm(Val: Value.getZExtValue());
123 return {LoadEven, LoadOdd};
124}
125
126// Fetch base-instruction to load an FP immediate value into a register.
127static unsigned getLoadFPImmediateOpcode(unsigned RegBitWidth) {
128 switch (RegBitWidth) {
129 case 16:
130 return AArch64::FMOVH0; // FMOVHi;
131 case 32:
132 return AArch64::FMOVS0; // FMOVSi;
133 case 64:
134 return AArch64::MOVID; // FMOVDi;
135 case 128:
136 return AArch64::MOVIv2d_ns;
137 }
138 llvm_unreachable("Invalid Value Width");
139}
140
141// Generates instruction to load an FP immediate value into a register.
142static MCInst loadFPImmediate(MCRegister Reg, unsigned RegBitWidth,
143 const APInt &Value) {
144 assert(Value.getZExtValue() == 0 && "Expected initialisation value 0");
145 MCInst Instructions =
146 MCInstBuilder(getLoadFPImmediateOpcode(RegBitWidth)).addReg(Reg);
147 if (RegBitWidth >= 64)
148 Instructions.addOperand(Op: MCOperand::createImm(Val: Value.getZExtValue()));
149 return Instructions;
150}
151
152// Generates instructions to load an immediate value into a DD, DDD, DDDD,
153// QQ, QQQ or QQQQ Reg
154static std::vector<MCInst>
155loadDQ234RegImmediate(MCRegister Reg, unsigned RegBitWidth, const APInt &Value,
156 MCRegister BaseReg, unsigned RegCount) {
157 MCRegister RegDorQ0 = AArch64::D0;
158 if (RegBitWidth == 128)
159 RegDorQ0 = AArch64::Q0;
160
161 MCRegister RegDQ0 = RegDorQ0 + ((Reg - BaseReg + 0) % 32);
162 MCRegister RegDQ1 = RegDorQ0 + ((Reg - BaseReg + 1) % 32);
163 MCRegister RegDQ2 = RegDorQ0 + ((Reg - BaseReg + 2) % 32);
164 MCRegister RegDQ3 = RegDorQ0 + ((Reg - BaseReg + 3) % 32);
165
166 MCInst LoadDQ0 = loadFPImmediate(Reg: RegDQ0, RegBitWidth, Value);
167 MCInst LoadDQ1 = loadFPImmediate(Reg: RegDQ1, RegBitWidth, Value);
168 if (RegCount == 2)
169 return {LoadDQ0, LoadDQ1};
170 MCInst LoadDQ2 = loadFPImmediate(Reg: RegDQ2, RegBitWidth, Value);
171 if (RegCount == 3)
172 return {LoadDQ0, LoadDQ1, LoadDQ2};
173 MCInst LoadDQ3 = loadFPImmediate(Reg: RegDQ3, RegBitWidth, Value);
174 assert((RegCount == 4) && "ExpectedRegCount 2, 3 or 4");
175 return {LoadDQ0, LoadDQ1, LoadDQ2, LoadDQ3};
176}
177
178// Generates instructions to load immediate in the flags register
179static std::vector<MCInst>
180loadNZCVImmediate(MCRegister Reg, unsigned RegBitWidth, const APInt &Value) {
181 MCRegister TempReg1 = AArch64::X8;
182 MCRegister TempReg2 = AArch64::X9;
183
184 MCInst MoveFromNZCV =
185 MCInstBuilder(AArch64::MRS).addReg(Reg: TempReg1).addImm(Val: AArch64SysReg::NZCV);
186 MCInst LoadMask =
187 MCInstBuilder(AArch64::MOVi64imm).addReg(Reg: TempReg2).addImm(Val: 0xf0000000);
188 MCInst BitClear = MCInstBuilder(AArch64::BICXrr)
189 .addReg(Reg: TempReg1)
190 .addReg(Reg: TempReg1)
191 .addReg(Reg: TempReg2);
192 MCInst MoveToNZCV =
193 MCInstBuilder(AArch64::MSR).addImm(Val: AArch64SysReg::NZCV).addReg(Reg: TempReg1);
194
195 if (Value.getZExtValue() == 0)
196 return {MoveFromNZCV, LoadMask, BitClear, MoveToNZCV};
197
198 MCInst OrrMask = MCInstBuilder(AArch64::ORRXrr)
199 .addReg(Reg: TempReg1)
200 .addReg(Reg: TempReg1)
201 .addImm(Val: Value.getZExtValue());
202 return {MoveFromNZCV, LoadMask, BitClear, OrrMask, MoveToNZCV};
203}
204
205#include "AArch64GenExegesis.inc"
206
207namespace {
208
209// Use X19 as the loop counter register since it's a callee-saved register
210// that's available for temporary use.
211constexpr MCPhysReg kDefaultLoopCounterReg = AArch64::X19;
212
213class ExegesisAArch64Target : public ExegesisTarget {
214public:
215 ExegesisAArch64Target()
216 : ExegesisTarget(AArch64CpuPfmCounters, AArch64_MC::isOpcodeAvailable) {}
217
218 Error randomizeTargetMCOperand(const Instruction &Instr, const Variable &Var,
219 MCOperand &AssignedValue,
220 const BitVector &ForbiddenRegs) const override;
221
222private:
223 std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, MCRegister Reg,
224 const APInt &Value) const override {
225 if (AArch64::GPR32RegClass.contains(Reg))
226 return {loadImmediate(Reg, RegBitWidth: 32, Value)};
227 if (AArch64::GPR64RegClass.contains(Reg))
228 return {loadImmediate(Reg, RegBitWidth: 64, Value)};
229 if (AArch64::PPRRegClass.contains(Reg))
230 return {loadPPRImmediate(Reg, RegBitWidth: 16, Value)};
231 if (AArch64::FPR8RegClass.contains(Reg))
232 return {loadFPImmediate(Reg: Reg - AArch64::B0 + AArch64::D0, RegBitWidth: 64, Value)};
233 if (AArch64::FPR16RegClass.contains(Reg))
234 return {loadFPImmediate(Reg, RegBitWidth: 16, Value)};
235 if (AArch64::FPR32RegClass.contains(Reg))
236 return {loadFPImmediate(Reg, RegBitWidth: 32, Value)};
237 if (AArch64::FPR64RegClass.contains(Reg))
238 return {loadFPImmediate(Reg, RegBitWidth: 64, Value)};
239 if (AArch64::FPR128RegClass.contains(Reg))
240 return {loadFPImmediate(Reg, RegBitWidth: 128, Value)};
241 if (AArch64::ZPRRegClass.contains(Reg))
242 return {loadZPRImmediate(Reg, RegBitWidth: 128, Value)};
243 if (Reg == AArch64::FPCR)
244 return {loadFPCRImmediate(Reg, RegBitWidth: 32, Value)};
245 if (Reg == AArch64::NZCV)
246 return {loadNZCVImmediate(Reg, RegBitWidth: 32, Value)};
247 if (Reg == AArch64::FFR)
248 return {loadFFRImmediate(Reg, RegBitWidth: 32, Value)};
249 if (AArch64::WSeqPairsClassRegClass.contains(Reg))
250 return {loadWSeqPairImmediate(Reg, RegBitWidth: 32, Value)};
251 if (AArch64::XSeqPairsClassRegClass.contains(Reg))
252 return {loadXSeqPairImmediate(Reg, RegBitWidth: 64, Value)};
253 if (AArch64::DDRegClass.contains(Reg))
254 return loadDQ234RegImmediate(Reg, RegBitWidth: 64, Value, BaseReg: AArch64::D0_D1, RegCount: 2);
255 if (AArch64::DDDRegClass.contains(Reg))
256 return loadDQ234RegImmediate(Reg, RegBitWidth: 64, Value, BaseReg: AArch64::D0_D1_D2, RegCount: 3);
257 if (AArch64::DDDDRegClass.contains(Reg))
258 return loadDQ234RegImmediate(Reg, RegBitWidth: 64, Value, BaseReg: AArch64::D0_D1_D2_D3, RegCount: 4);
259 if (AArch64::QQRegClass.contains(Reg))
260 return loadDQ234RegImmediate(Reg, RegBitWidth: 128, Value, BaseReg: AArch64::Q0_Q1, RegCount: 2);
261 if (AArch64::QQQRegClass.contains(Reg))
262 return loadDQ234RegImmediate(Reg, RegBitWidth: 128, Value, BaseReg: AArch64::Q0_Q1_Q2, RegCount: 3);
263 if (AArch64::QQQQRegClass.contains(Reg))
264 return loadDQ234RegImmediate(Reg, RegBitWidth: 128, Value, BaseReg: AArch64::Q0_Q1_Q2_Q3, RegCount: 4);
265 // TODO if (AArch64::PNRRegClass.contains(Reg))
266 // TODO if (AArch64::ZPRRegClass.contains(Reg))
267 // TODO if (AArch64::ZPR2RegClass.contains(Reg))
268 // TODO if (AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg))
269
270 errs() << "setRegTo is not implemented, results will be unreliable\n";
271 return {};
272 }
273 MCRegister getDefaultLoopCounterRegister(const Triple &) const override {
274 return kDefaultLoopCounterReg;
275 }
276
277 void decrementLoopCounterAndJump(MachineBasicBlock &MBB,
278 MachineBasicBlock &TargetMBB,
279 const MCInstrInfo &MII,
280 MCRegister LoopRegister) const override {
281 // subs LoopRegister, LoopRegister, #1
282 BuildMI(BB: &MBB, MIMD: DebugLoc(), MCID: MII.get(Opcode: AArch64::SUBSXri))
283 .addDef(RegNo: LoopRegister)
284 .addUse(RegNo: LoopRegister)
285 .addImm(Val: 1) // Subtract 1
286 .addImm(Val: 0); // No shift amount
287 // b.ne TargetMBB
288 BuildMI(BB: &MBB, MIMD: DebugLoc(), MCID: MII.get(Opcode: AArch64::Bcc))
289 .addImm(Val: AArch64CC::NE)
290 .addMBB(MBB: &TargetMBB);
291 }
292
293 // Registers that should not be selected for use in snippets.
294 const MCPhysReg UnavailableRegisters[1] = {kDefaultLoopCounterReg};
295 ArrayRef<MCPhysReg> getUnavailableRegisters() const override {
296 return UnavailableRegisters;
297 }
298
299 bool matchesArch(Triple::ArchType Arch) const override {
300 return Arch == Triple::aarch64 || Arch == Triple::aarch64_be;
301 }
302
303 void addTargetSpecificPasses(PassManagerBase &PM) const override {
304 // Function return is a pseudo-instruction that needs to be expanded
305 PM.add(P: createAArch64ExpandPseudoPass());
306 }
307};
308
309Error ExegesisAArch64Target::randomizeTargetMCOperand(
310 const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue,
311 const BitVector &ForbiddenRegs) const {
312 const Operand &Op = Instr.getPrimaryOperand(Var);
313 const auto OperandType = Op.getExplicitOperandInfo().OperandType;
314 // NOTE: To resolve "Not all operands were initialized by snippet generator"
315 // Requires OperandType to be defined for such opcode's operands in AArch64
316 // tablegen files. And omit introduced OperandType(s).
317
318 // Hacky Fix: Defaulting all OPERAND_UNKNOWN to immediate value 0 works with a
319 // limitation that it introduces illegal instruction error for system
320 // instructions. System instructions will need to be omitted with OperandType
321 // or opcode specific values to avoid generating invalid encodings or
322 // unreliable benchmark results for these system-level instructions.
323 // Implement opcode-specific immediate value handling for system instrs:
324 // - MRS/MSR: Use valid system register encodings (e.g., NZCV, FPCR, FPSR)
325 // - MSRpstatesvcrImm1: Use valid PSTATE field encodings (e.g., SPSel,
326 // DAIFSet)
327 // - SYSLxt/SYSxt: Use valid system instruction encodings with proper
328 // CRn/CRm/op values
329 // - UDF: Use valid undefined instruction immediate ranges (0-65535)
330
331 switch (OperandType) {
332 // MSL (Masking Shift Left) imm operand for 32-bit splatted SIMD constants
333 // Correspond to AArch64InstructionSelector::tryAdvSIMDModImm321s()
334 case llvm::AArch64::OPERAND_SHIFT_MSL: {
335 // There are two valid encodings:
336 // - Type 7: imm at [15:8], [47:40], shift = 264 (0x108) → msl #8
337 // - Type 8: imm at [23:16], [55:48], shift = 272 (0x110) → msl #16
338 // Corresponds AArch64_AM::encodeAdvSIMDModImmType7()
339 // But, v2s_msl and v4s_msl instructions accept either form,
340 // Thus, Arbitrarily chosing 264 (msl #8) for simplicity.
341 AssignedValue = MCOperand::createImm(Val: 264);
342 return Error::success();
343 }
344 case llvm::AArch64::OPERAND_IMPLICIT_IMM_0:
345 AssignedValue = MCOperand::createImm(Val: 0);
346 return Error::success();
347 case llvm::AArch64::OPERAND_SHIFTED_REGISTER:
348 // TODO it would be better if these operands were randomized
349 AssignedValue = MCOperand::createReg(Reg: 0);
350 return Error::success();
351 case llvm::AArch64::OPERAND_SHIFTED_IMMEDIATE:
352 AssignedValue = MCOperand::createImm(Val: 0);
353 return Error::success();
354 case MCOI::OperandType::OPERAND_PCREL:
355 AssignedValue = MCOperand::createImm(Val: 8);
356 return Error::success();
357 default:
358 break;
359 }
360
361 return make_error<Failure>(
362 Args: Twine("Unimplemented operand type: MCOI::OperandType:")
363 .concat(Suffix: Twine(static_cast<int>(OperandType))));
364}
365
366} // namespace
367
368static ExegesisTarget *getTheExegesisAArch64Target() {
369 static ExegesisAArch64Target Target;
370 return &Target;
371}
372
373void InitializeAArch64ExegesisTarget() {
374 ExegesisTarget::registerTarget(T: getTheExegesisAArch64Target());
375}
376
377} // namespace exegesis
378} // namespace llvm
379