1//===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that BPF uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "BPFISelLowering.h"
15#include "BPF.h"
16#include "BPFSubtarget.h"
17#include "llvm/CodeGen/CallingConvLower.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineJumpTableInfo.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
24#include "llvm/CodeGen/ValueTypes.h"
25#include "llvm/IR/DIBuilder.h"
26#include "llvm/IR/DiagnosticInfo.h"
27#include "llvm/IR/DiagnosticPrinter.h"
28#include "llvm/IR/Module.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Support/raw_ostream.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "bpf-lower"
37
38static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
39 cl::Hidden, cl::init(Val: false),
40 cl::desc("Expand memcpy into load/store pairs in order"));
41
42static cl::opt<unsigned> BPFMinimumJumpTableEntries(
43 "bpf-min-jump-table-entries", cl::init(Val: 13), cl::Hidden,
44 cl::desc("Set minimum number of entries to use a jump table on BPF"));
45
46static cl::opt<bool> BPFAllowsLibcalls(
47 "bpf-allows-libcalls", cl::Hidden, cl::init(Val: false),
48 cl::desc("Allow libcalls instead of rejecting unsupported built-in "
49 "functions"));
50
51static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg,
52 SDValue Val = {}) {
53 std::string Str;
54 if (Val) {
55 raw_string_ostream OS(Str);
56 Val->print(OS);
57 OS << ' ';
58 }
59 MachineFunction &MF = DAG.getMachineFunction();
60 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
61 MF.getFunction(), Twine(Str).concat(Suffix: Msg), DL.getDebugLoc()));
62}
63
64BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
65 const BPFSubtarget &STI)
66 : TargetLowering(TM, STI) {
67
68 // Set up the register classes.
69 addRegisterClass(VT: MVT::i64, RC: &BPF::GPRRegClass);
70 if (STI.getHasAlu32())
71 addRegisterClass(VT: MVT::i32, RC: &BPF::GPR32RegClass);
72
73 // Compute derived properties from the register classes
74 computeRegisterProperties(TRI: STI.getRegisterInfo());
75
76 setStackPointerRegisterToSaveRestore(BPF::R11);
77
78 setOperationAction(Op: ISD::BR_CC, VT: MVT::i64, Action: Custom);
79 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: STI.hasGotox() ? Custom : Expand);
80 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
81
82 if (!STI.hasGotox())
83 setOperationAction(Op: ISD::BRIND, VT: MVT::Other, Action: Expand);
84
85 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Custom);
86
87 setOperationAction(Ops: {ISD::GlobalAddress, ISD::ConstantPool}, VT: MVT::i64, Action: Custom);
88 if (STI.hasGotox())
89 setOperationAction(Ops: {ISD::JumpTable, ISD::BlockAddress}, VT: MVT::i64, Action: Custom);
90
91 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i64, Action: Custom);
92 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
93 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
94
95 // Set unsupported atomic operations as Custom so
96 // we can emit better error messages than fatal error
97 // from selectiondag.
98 for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
99 if (VT == MVT::i32) {
100 if (STI.getHasAlu32())
101 continue;
102 } else {
103 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT, Action: Custom);
104 }
105
106 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT, Action: Custom);
107 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT, Action: Custom);
108 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT, Action: Custom);
109 setOperationAction(Op: ISD::ATOMIC_SWAP, VT, Action: Custom);
110 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Action: Custom);
111 }
112
113 for (auto VT : {MVT::i32, MVT::i64}) {
114 setOperationAction(Op: ISD::ATOMIC_LOAD, VT, Action: Custom);
115 setOperationAction(Op: ISD::ATOMIC_STORE, VT, Action: Custom);
116 // No atomic sub insn. Expand it into atomic add of the negated value.
117 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Expand);
118 }
119
120 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
121
122 for (auto VT : { MVT::i32, MVT::i64 }) {
123 if (VT == MVT::i32 && !STI.getHasAlu32())
124 continue;
125
126 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
127 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
128 if (!STI.hasSdivSmod()) {
129 setOperationAction(Op: ISD::SDIV, VT, Action: Custom);
130 setOperationAction(Op: ISD::SREM, VT, Action: Custom);
131 }
132 if (VT == MVT::i32 || !STI.hasHmul()) {
133 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
134 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
135 }
136 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
137 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
138 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
139 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
140 setOperationAction(Op: ISD::SHL_PARTS, VT, Action: Custom);
141 setOperationAction(Op: ISD::SRL_PARTS, VT, Action: Custom);
142 setOperationAction(Op: ISD::SRA_PARTS, VT, Action: Custom);
143 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
144 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
145 setOperationAction(Op: ISD::CTLZ, VT, Action: Expand);
146 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT, Action: Expand);
147 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT, Action: Expand);
148
149 setOperationAction(Op: ISD::SETCC, VT, Action: Expand);
150 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
151 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
152 }
153
154 if (STI.getHasAlu32()) {
155 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Promote);
156 setOperationAction(Op: ISD::BR_CC, VT: MVT::i32,
157 Action: STI.getHasJmp32() ? Custom : Promote);
158 }
159
160 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
161 if (!STI.hasMovsx()) {
162 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
163 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
164 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i32, Action: Expand);
165 }
166
167 // Extended load operations for i1 types must be promoted
168 for (MVT VT : MVT::integer_valuetypes()) {
169 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
170 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
171 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
172
173 if (!STI.hasLdsx()) {
174 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i8, Action: Expand);
175 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i16, Action: Expand);
176 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
177 }
178 }
179
180 setBooleanContents(ZeroOrOneBooleanContent);
181 setMaxAtomicSizeInBitsSupported(64);
182 // AtomicExpand turns operations on 8 and 16 bits into cmpxchg of the word
183 // that they are in. Its loops start with a relaxed atomic load.
184 if (STI.hasLoadAcqStoreRel())
185 setMinCmpXchgSizeInBits(32);
186 setMinimumJumpTableEntries(BPFMinimumJumpTableEntries);
187
188 // Function alignments
189 setMinFunctionAlignment(Align(8));
190 setPrefFunctionAlignment(Align(8));
191
192 if (BPFExpandMemcpyInOrder) {
193 // LLVM generic code will try to expand memcpy into load/store pairs at this
194 // stage which is before quite a few IR optimization passes, therefore the
195 // loads and stores could potentially be moved apart from each other which
196 // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
197 // compilers.
198 //
199 // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
200 // of memcpy to later stage in IR optimization pipeline so those load/store
201 // pairs won't be touched and could be kept in order. Hence, we set
202 // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
203 // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
204 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = 0;
205 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = 0;
206 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 0;
207 MaxLoadsPerMemcmp = 0;
208 } else {
209 // inline memcpy() for kernel to see explicit copy
210 unsigned CommonMaxStores =
211 STI.getSelectionDAGInfo()->getCommonMaxStoresPerMemFunc();
212
213 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
214 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
215 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = CommonMaxStores;
216 MaxLoadsPerMemcmp = MaxLoadsPerMemcmpOptSize = CommonMaxStores;
217 }
218
219 // CPU/Feature control
220 HasAlu32 = STI.getHasAlu32();
221 HasJmp32 = STI.getHasJmp32();
222 HasJmpExt = STI.getHasJmpExt();
223 HasMovsx = STI.hasMovsx();
224 HasLoadAcqStoreRel = STI.hasLoadAcqStoreRel();
225
226 AllowsMisalignedMemAccess = STI.getAllowsMisalignedMemAccess();
227}
228
229bool BPFTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
230 MachineMemOperand::Flags,
231 unsigned *Fast) const {
232 // allows-misaligned-mem-access is disabled
233 if (!AllowsMisalignedMemAccess)
234 return false;
235
236 // only allow misalignment for simple value types
237 if (!VT.isSimple())
238 return false;
239
240 // always assume fast mode when misalignment is allowed
241 if (Fast)
242 *Fast = true;
243
244 return true;
245}
246
247bool BPFTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
248 return false;
249}
250
251bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
252 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
253 return false;
254 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
255 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
256 return NumBits1 > NumBits2;
257}
258
259bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
260 if (!VT1.isInteger() || !VT2.isInteger())
261 return false;
262 unsigned NumBits1 = VT1.getSizeInBits();
263 unsigned NumBits2 = VT2.getSizeInBits();
264 return NumBits1 > NumBits2;
265}
266
267bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
268 if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
269 return false;
270 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
271 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
272 return NumBits1 == 32 && NumBits2 == 64;
273}
274
275bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
276 if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
277 return false;
278 unsigned NumBits1 = VT1.getSizeInBits();
279 unsigned NumBits2 = VT2.getSizeInBits();
280 return NumBits1 == 32 && NumBits2 == 64;
281}
282
283bool BPFTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
284 EVT VT1 = Val.getValueType();
285 if (Val.getOpcode() == ISD::LOAD && VT1.isSimple() && VT2.isSimple()) {
286 MVT MT1 = VT1.getSimpleVT().SimpleTy;
287 MVT MT2 = VT2.getSimpleVT().SimpleTy;
288 if ((MT1 == MVT::i8 || MT1 == MVT::i16 || MT1 == MVT::i32) &&
289 (MT2 == MVT::i32 || MT2 == MVT::i64))
290 return true;
291 }
292 return TargetLoweringBase::isZExtFree(Val, VT2);
293}
294
295unsigned BPFTargetLowering::getJumpTableEncoding() const {
296 return MachineJumpTableInfo::EK_BlockAddress;
297}
298
299BPFTargetLowering::ConstraintType
300BPFTargetLowering::getConstraintType(StringRef Constraint) const {
301 if (Constraint.size() == 1) {
302 switch (Constraint[0]) {
303 default:
304 break;
305 case 'w':
306 return C_RegisterClass;
307 }
308 }
309
310 return TargetLowering::getConstraintType(Constraint);
311}
312
313std::pair<unsigned, const TargetRegisterClass *>
314BPFTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
315 StringRef Constraint,
316 MVT VT) const {
317 if (Constraint.size() == 1) {
318 // GCC Constraint Letters
319 switch (Constraint[0]) {
320 case 'r': // GENERAL_REGS
321 return std::make_pair(x: 0U, y: &BPF::GPRRegClass);
322 case 'w':
323 if (HasAlu32)
324 return std::make_pair(x: 0U, y: &BPF::GPR32RegClass);
325 break;
326 default:
327 break;
328 }
329 }
330
331 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
332}
333
334void BPFTargetLowering::ReplaceNodeResults(
335 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
336 const char *Msg;
337 uint32_t Opcode = N->getOpcode();
338 switch (Opcode) {
339 default:
340 report_fatal_error(reason: "unhandled custom legalization: " + Twine(Opcode));
341 case ISD::ATOMIC_LOAD_ADD:
342 case ISD::ATOMIC_LOAD_AND:
343 case ISD::ATOMIC_LOAD_OR:
344 case ISD::ATOMIC_LOAD_XOR:
345 case ISD::ATOMIC_SWAP:
346 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
347 if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD)
348 Msg = "unsupported atomic operation, please use 32/64 bit version";
349 else
350 Msg = "unsupported atomic operation, please use 64 bit version";
351 break;
352 case ISD::ATOMIC_LOAD:
353 case ISD::ATOMIC_STORE:
354 return;
355 }
356
357 SDLoc DL(N);
358 fail(DL, DAG, Msg);
359 for (EVT VT : N->values())
360 Results.push_back(Elt: VT == MVT::Other ? N->getOperand(Num: 0) : DAG.getUNDEF(VT));
361}
362
363SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
364 switch (Op.getOpcode()) {
365 default:
366 report_fatal_error(reason: "unimplemented opcode: " + Twine(Op.getOpcode()));
367 case ISD::BR_CC:
368 return LowerBR_CC(Op, DAG);
369 case ISD::JumpTable:
370 return LowerJumpTable(Op, DAG);
371 case ISD::BR_JT:
372 return LowerBR_JT(Op, DAG);
373 case ISD::GlobalAddress:
374 return LowerGlobalAddress(Op, DAG);
375 case ISD::ConstantPool:
376 return LowerConstantPool(Op, DAG);
377 case ISD::BlockAddress:
378 return LowerBlockAddress(Op, DAG);
379 case ISD::SELECT_CC:
380 return LowerSELECT_CC(Op, DAG);
381 case ISD::SDIV:
382 case ISD::SREM:
383 return LowerSDIVSREM(Op, DAG);
384 case ISD::SHL_PARTS:
385 case ISD::SRL_PARTS:
386 case ISD::SRA_PARTS:
387 return LowerShiftParts(Op, DAG);
388 case ISD::DYNAMIC_STACKALLOC:
389 return LowerDYNAMIC_STACKALLOC(Op, DAG);
390 case ISD::ATOMIC_LOAD:
391 case ISD::ATOMIC_STORE:
392 return LowerATOMIC_LOAD_STORE(Op, DAG);
393 case ISD::ATOMIC_FENCE:
394 return LowerATOMIC_FENCE(Op, DAG);
395 case ISD::TRAP:
396 return LowerTRAP(Op, DAG);
397 }
398}
399
400// Calling Convention Implementation
401#define GET_CALLING_CONV_IMPL
402#include "BPFGenCallingConv.inc"
403
404// Apply AssertSext/AssertZext and truncate based on VA's LocInfo.
405static SDValue convertLocValType(SelectionDAG &DAG, const SDLoc &DL,
406 const CCValAssign &VA, EVT RegVT,
407 SDValue ArgValue) {
408 if (VA.getLocInfo() == CCValAssign::SExt)
409 ArgValue = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: RegVT, N1: ArgValue,
410 N2: DAG.getValueType(VA.getValVT()));
411 else if (VA.getLocInfo() == CCValAssign::ZExt)
412 ArgValue = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: RegVT, N1: ArgValue,
413 N2: DAG.getValueType(VA.getValVT()));
414 if (VA.getLocInfo() != CCValAssign::Full)
415 ArgValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: ArgValue);
416 return ArgValue;
417}
418
419SDValue BPFTargetLowering::LowerFormalArguments(
420 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
421 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
422 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
423 switch (CallConv) {
424 default:
425 report_fatal_error(reason: "unimplemented calling convention: " + Twine(CallConv));
426 case CallingConv::C:
427 case CallingConv::Fast:
428 break;
429 }
430
431 MachineFunction &MF = DAG.getMachineFunction();
432 MachineRegisterInfo &RegInfo = MF.getRegInfo();
433
434 // Assign locations to all of the incoming arguments.
435 SmallVector<CCValAssign, 16> ArgLocs;
436 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
437 CCInfo.AnalyzeFormalArguments(Ins, Fn: getHasAlu32() ? CC_BPF32 : CC_BPF64);
438
439 for (size_t I = 0; I < ArgLocs.size(); ++I) {
440 auto &VA = ArgLocs[I];
441 EVT RegVT = VA.getLocVT();
442
443 if (VA.isRegLoc()) {
444 // Arguments passed in registers
445 MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
446 switch (SimpleTy) {
447 default: {
448 std::string Str;
449 {
450 raw_string_ostream OS(Str);
451 RegVT.print(OS);
452 }
453 report_fatal_error(reason: "unhandled argument type: " + Twine(Str));
454 }
455 case MVT::i32:
456 case MVT::i64:
457 Register VReg = RegInfo.createVirtualRegister(
458 RegClass: SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
459 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
460 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: RegVT);
461 InVals.push_back(Elt: convertLocValType(DAG, DL, VA, RegVT, ArgValue));
462 break;
463 }
464 continue;
465 }
466
467 if (VA.isMemLoc()) {
468 // For example, two stack arguments,
469 // arg1: Off = 8
470 // arg2: off = 16
471 int Off = VA.getLocMemOffset() + 8;
472 if (Off > INT16_MAX) {
473 fail(DL, DAG, Msg: "extra parameter stack depth exceeded limit");
474 break;
475 }
476
477 // Physical extra argument slot is always 64-bit.
478 SDValue StackVal = DAG.getNode(Opcode: BPFISD::LOAD_STACK_ARG, DL,
479 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain,
480 N2: DAG.getConstant(Val: Off, DL, VT: MVT::i64));
481 SDValue ArgValue = StackVal.getValue(R: 0);
482 Chain = StackVal.getValue(R: 1);
483 InVals.push_back(Elt: convertLocValType(DAG, DL, VA, RegVT: MVT::i64, ArgValue));
484 continue;
485 }
486 }
487
488 if (IsVarArg)
489 fail(DL, DAG, Msg: "variadic functions are not supported");
490 return Chain;
491}
492
493static void resetRegMaskBit(const TargetRegisterInfo *TRI, uint32_t *RegMask,
494 MCRegister Reg) {
495 for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg))
496 RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
497}
498
499static uint32_t *regMaskFromTemplate(const TargetRegisterInfo *TRI,
500 MachineFunction &MF,
501 const uint32_t *BaseRegMask) {
502 uint32_t *RegMask = MF.allocateRegMask();
503 unsigned RegMaskSize = MachineOperand::getRegMaskSize(NumRegs: TRI->getNumRegs());
504 memcpy(dest: RegMask, src: BaseRegMask, n: sizeof(RegMask[0]) * RegMaskSize);
505 return RegMask;
506}
507
508SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
509 SmallVectorImpl<SDValue> &InVals) const {
510 SelectionDAG &DAG = CLI.DAG;
511 auto &Outs = CLI.Outs;
512 auto &OutVals = CLI.OutVals;
513 auto &Ins = CLI.Ins;
514 SDValue Chain = CLI.Chain;
515 SDValue Callee = CLI.Callee;
516 bool &IsTailCall = CLI.IsTailCall;
517 CallingConv::ID CallConv = CLI.CallConv;
518 bool IsVarArg = CLI.IsVarArg;
519 MachineFunction &MF = DAG.getMachineFunction();
520
521 // BPF target does not support tail call optimization.
522 IsTailCall = false;
523
524 switch (CallConv) {
525 default:
526 report_fatal_error(reason: "unsupported calling convention: " + Twine(CallConv));
527 case CallingConv::Fast:
528 case CallingConv::C:
529 break;
530 }
531
532 // Analyze operands of the call, assigning locations to each operand.
533 SmallVector<CCValAssign, 16> ArgLocs;
534 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
535
536 CCInfo.AnalyzeCallOperands(Outs, Fn: getHasAlu32() ? CC_BPF32 : CC_BPF64);
537
538 unsigned NumBytes = CCInfo.getStackSize();
539
540 for (auto &Arg : Outs) {
541 ISD::ArgFlagsTy Flags = Arg.Flags;
542 if (!Flags.isByVal())
543 continue;
544 fail(DL: CLI.DL, DAG, Msg: "pass by value not supported", Val: Callee);
545 break;
546 }
547
548 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
549 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
550
551 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
552
553 // Walk arg assignments
554 for (size_t i = 0; i < OutVals.size(); ++i) {
555 CCValAssign &VA = ArgLocs[i];
556 SDValue &Arg = OutVals[i];
557
558 // Promote the value if needed.
559 switch (VA.getLocInfo()) {
560 default:
561 report_fatal_error(reason: "unhandled location info: " + Twine(VA.getLocInfo()));
562 case CCValAssign::Full:
563 break;
564 case CCValAssign::SExt:
565 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: CLI.DL, VT: VA.getLocVT(), Operand: Arg);
566 break;
567 case CCValAssign::ZExt:
568 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: CLI.DL, VT: VA.getLocVT(), Operand: Arg);
569 break;
570 case CCValAssign::AExt:
571 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: CLI.DL, VT: VA.getLocVT(), Operand: Arg);
572 break;
573 }
574
575 // Push arguments into RegsToPass vector
576 if (VA.isRegLoc()) {
577 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
578 continue;
579 }
580
581 if (VA.isMemLoc()) {
582 int Off = -8 - VA.getLocMemOffset();
583 if (Off < INT16_MIN) {
584 fail(DL: CLI.DL, DAG, Msg: "extra parameter stack depth exceeded limit");
585 break;
586 }
587
588 assert(Arg.getValueType() == MVT::i64 && "stack args are i64");
589
590 SDValue OffVal = DAG.getConstant(Val: Off, DL: CLI.DL, VT: MVT::i64);
591 Chain = DAG.getNode(Opcode: BPFISD::STORE_STACK_ARG, DL: CLI.DL, VT: MVT::Other, N1: Chain,
592 N2: OffVal, N3: Arg);
593 continue;
594 }
595
596 report_fatal_error(reason: "unhandled argument location");
597 }
598
599 SDValue InGlue;
600
601 // Build a sequence of copy-to-reg nodes chained together with token chain and
602 // flag operands which copy the outgoing args into registers. The InGlue in
603 // necessary since all emitted instructions must be stuck together.
604 for (auto &Reg : RegsToPass) {
605 Chain = DAG.getCopyToReg(Chain, dl: CLI.DL, Reg: Reg.first, N: Reg.second, Glue: InGlue);
606 InGlue = Chain.getValue(R: 1);
607 }
608
609 // If the callee is a GlobalAddress node (quite common, every direct call is)
610 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
611 // Likewise ExternalSymbol -> TargetExternalSymbol.
612 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
613 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL: CLI.DL, VT: PtrVT,
614 offset: G->getOffset(), TargetFlags: 0);
615 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
616 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT, TargetFlags: 0);
617 StringRef Sym = E->getSymbol();
618 if (!BPFAllowsLibcalls && Sym != BPF_TRAP && Sym != "__multi3" &&
619 Sym != "__divti3" && Sym != "__modti3" && Sym != "__udivti3" &&
620 Sym != "__umodti3" && Sym != "memcpy" && Sym != "memset" &&
621 Sym != "memmove")
622 fail(
623 DL: CLI.DL, DAG,
624 Msg: Twine("A call to built-in function '" + Sym + "' is not supported."));
625 }
626
627 // Returns a chain & a flag for retval copy to use.
628 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
629 SmallVector<SDValue, 8> Ops;
630 Ops.push_back(Elt: Chain);
631 Ops.push_back(Elt: Callee);
632
633 // Add argument registers to the end of the list so that they are
634 // known live into the call.
635 for (auto &Reg : RegsToPass)
636 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
637
638 bool HasFastCall =
639 (CLI.CB && isa<CallInst>(Val: CLI.CB) && CLI.CB->hasFnAttr(Kind: "bpf_fastcall"));
640 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
641 if (HasFastCall) {
642 uint32_t *RegMask = regMaskFromTemplate(
643 TRI, MF, BaseRegMask: TRI->getCallPreservedMask(MF, CallingConv::PreserveAll));
644 for (auto const &RegPair : RegsToPass)
645 resetRegMaskBit(TRI, RegMask, Reg: RegPair.first);
646 if (!CLI.CB->getType()->isVoidTy())
647 resetRegMaskBit(TRI, RegMask, Reg: BPF::R0);
648 Ops.push_back(Elt: DAG.getRegisterMask(RegMask));
649 } else {
650 Ops.push_back(
651 Elt: DAG.getRegisterMask(RegMask: TRI->getCallPreservedMask(MF, CLI.CallConv)));
652 }
653
654 if (InGlue.getNode())
655 Ops.push_back(Elt: InGlue);
656
657 Chain = DAG.getNode(Opcode: BPFISD::CALL, DL: CLI.DL, VTList: NodeTys, Ops);
658 InGlue = Chain.getValue(R: 1);
659
660 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
661
662 // Create the CALLSEQ_END node.
663 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL: CLI.DL);
664 InGlue = Chain.getValue(R: 1);
665
666 // Handle result values, copying them out of physregs into vregs that we
667 // return.
668 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL: CLI.DL, DAG,
669 InVals);
670}
671
672SDValue
673BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
674 bool IsVarArg,
675 const SmallVectorImpl<ISD::OutputArg> &Outs,
676 const SmallVectorImpl<SDValue> &OutVals,
677 const SDLoc &DL, SelectionDAG &DAG) const {
678 unsigned Opc = BPFISD::RET_GLUE;
679
680 // CCValAssign - represent the assignment of the return value to a location
681 SmallVector<CCValAssign, 16> RVLocs;
682 MachineFunction &MF = DAG.getMachineFunction();
683
684 // CCState - Info about the registers and stack slot.
685 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
686
687 // Analize return values.
688 CCInfo.AnalyzeReturn(Outs, Fn: getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
689
690 SDValue Glue;
691 SmallVector<SDValue, 4> RetOps(1, Chain);
692
693 // Copy the result values into the output registers.
694 for (size_t i = 0; i != RVLocs.size(); ++i) {
695 CCValAssign &VA = RVLocs[i];
696 if (!VA.isRegLoc())
697 report_fatal_error(reason: "stack return values are not supported");
698
699 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: OutVals[i], Glue);
700
701 // Guarantee that all emitted copies are stuck together,
702 // avoiding something bad.
703 Glue = Chain.getValue(R: 1);
704 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
705 }
706
707 RetOps[0] = Chain; // Update chain.
708
709 // Add the glue if we have it.
710 if (Glue.getNode())
711 RetOps.push_back(Elt: Glue);
712
713 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
714}
715
716SDValue BPFTargetLowering::LowerCallResult(
717 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
718 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
719 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
720
721 MachineFunction &MF = DAG.getMachineFunction();
722 // Assign locations to each value returned by this call.
723 SmallVector<CCValAssign, 16> RVLocs;
724 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
725
726 CCInfo.AnalyzeCallResult(Ins, Fn: getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
727
728 // Copy all of the result registers out of their specified physreg.
729 for (auto &Val : RVLocs) {
730 Chain = DAG.getCopyFromReg(Chain, dl: DL, Reg: Val.getLocReg(),
731 VT: Val.getValVT(), Glue: InGlue).getValue(R: 1);
732 InGlue = Chain.getValue(R: 2);
733 InVals.push_back(Elt: Chain.getValue(R: 0));
734 }
735
736 return Chain;
737}
738
739static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
740 switch (CC) {
741 default:
742 break;
743 case ISD::SETULT:
744 case ISD::SETULE:
745 case ISD::SETLT:
746 case ISD::SETLE:
747 CC = ISD::getSetCCSwappedOperands(Operation: CC);
748 std::swap(a&: LHS, b&: RHS);
749 break;
750 }
751}
752
753SDValue BPFTargetLowering::LowerSDIVSREM(SDValue Op, SelectionDAG &DAG) const {
754 SDLoc DL(Op);
755 fail(DL, DAG,
756 Msg: "unsupported signed division, please convert to unsigned div/mod.");
757 return DAG.getUNDEF(VT: Op->getValueType(ResNo: 0));
758}
759
760SDValue BPFTargetLowering::LowerShiftParts(SDValue Op,
761 SelectionDAG &DAG) const {
762 SDValue Lo, Hi;
763 expandShiftParts(N: Op.getNode(), Lo, Hi, DAG);
764 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: SDLoc(Op));
765}
766
767SDValue BPFTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
768 SelectionDAG &DAG) const {
769 SDLoc DL(Op);
770 fail(DL, DAG, Msg: "unsupported dynamic stack allocation");
771 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
772 return DAG.getMergeValues(Ops, dl: SDLoc());
773}
774
775SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
776 SDValue Chain = Op.getOperand(i: 0);
777 ISD::CondCode CC = Op.getOperandCondCode(i: 1);
778 SDValue LHS = Op.getOperand(i: 2);
779 SDValue RHS = Op.getOperand(i: 3);
780 SDValue Dest = Op.getOperand(i: 4);
781 SDLoc DL(Op);
782
783 if (!getHasJmpExt())
784 NegateCC(LHS, RHS, CC);
785
786 return DAG.getNode(Opcode: BPFISD::BR_CC, DL, VT: Op.getValueType(), N1: Chain, N2: LHS, N3: RHS,
787 N4: DAG.getConstant(Val: CC, DL, VT: LHS.getValueType()), N5: Dest);
788}
789
790SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
791 SDValue LHS = Op.getOperand(i: 0);
792 SDValue RHS = Op.getOperand(i: 1);
793 SDValue TrueV = Op.getOperand(i: 2);
794 SDValue FalseV = Op.getOperand(i: 3);
795 ISD::CondCode CC = Op.getOperandCondCode(i: 4);
796 SDLoc DL(Op);
797
798 if (!getHasJmpExt())
799 NegateCC(LHS, RHS, CC);
800
801 SDValue TargetCC = DAG.getConstant(Val: CC, DL, VT: LHS.getValueType());
802 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
803
804 return DAG.getNode(Opcode: BPFISD::SELECT_CC, DL, VT: Op.getValueType(), Ops);
805}
806
807SDValue BPFTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
808 SelectionDAG &DAG) const {
809 SDNode *N = Op.getNode();
810 SDLoc DL(N);
811
812 // A seq_cst store is xchg by now, see shouldExpandAtomicStoreInIR(), which
813 // makes load acquire enough for a seq_cst load.
814 if (cast<AtomicSDNode>(Val: N)->getMergedOrdering() ==
815 AtomicOrdering::SequentiallyConsistent &&
816 (N->getOpcode() == ISD::ATOMIC_STORE || !HasLoadAcqStoreRel))
817 fail(DL, DAG,
818 Msg: "sequentially consistent (seq_cst) "
819 "atomic load/store is not supported");
820
821 return Op;
822}
823
824TargetLowering::AtomicExpansionKind
825BPFTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
826 if (!HasLoadAcqStoreRel)
827 return TargetLowering::shouldExpandAtomicRMWInIR(RMW: AI);
828
829 switch (AI->getOperation()) {
830 case AtomicRMWInst::Add:
831 case AtomicRMWInst::Sub:
832 case AtomicRMWInst::And:
833 case AtomicRMWInst::Or:
834 case AtomicRMWInst::Xor:
835 case AtomicRMWInst::Xchg:
836 // There are insns for 32 and 64 bits.
837 if (AI->getDataLayout().getTypeSizeInBits(Ty: AI->getType()) >= 32)
838 return AtomicExpansionKind::None;
839 [[fallthrough]];
840 default:
841 return AtomicExpansionKind::CmpXChg;
842 }
843}
844
845TargetLowering::AtomicExpansionKind
846BPFTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
847 // Store release is not ordered with a later load. xchg is fully ordered.
848 if (HasLoadAcqStoreRel &&
849 SI->getOrdering() == AtomicOrdering::SequentiallyConsistent)
850 return AtomicExpansionKind::Expand;
851 return AtomicExpansionKind::None;
852}
853
854SDValue BPFTargetLowering::LowerATOMIC_FENCE(SDValue Op,
855 SelectionDAG &DAG) const {
856 SDLoc DL(Op);
857 SyncScope::ID FenceSSID =
858 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
859
860 if (FenceSSID == SyncScope::SingleThread)
861 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
862 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
863
864 // There is no insn for that. An atomic insn that returns the old value is
865 // fully ordered: add zero to a slot in the stack. Without alu32 that is
866 // 'lock *(u64 *)(r10 - 8) += r1', which is not ordered.
867 if (!HasAlu32)
868 report_fatal_error(reason: "Runtime fence is not supported at the moment");
869
870 MachineFunction &MF = DAG.getMachineFunction();
871 SDValue Slot = DAG.CreateStackTemporary(VT: MVT::i64);
872 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
873 MF, FI: cast<FrameIndexSDNode>(Val&: Slot)->getIndex());
874 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
875 SDValue Chain = DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: Zero, Ptr: Slot, PtrInfo);
876 MachineMemOperand *MMO = MF.getMachineMemOperand(
877 PtrInfo, F: MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
878 MemTy: LLT::scalar(SizeInBits: 64), BaseAlignment: Align(8), Metadata: MMOMetadata(), SSID: SyncScope::System,
879 Ordering: AtomicOrdering::SequentiallyConsistent);
880 return DAG
881 .getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: MVT::i64, Chain, Ptr: Slot, Val: Zero, MMO)
882 .getValue(R: 1);
883}
884
885static Function *createBPFUnreachable(Module *M) {
886 if (auto *Fn = M->getFunction(BPF_TRAP))
887 return Fn;
888
889 FunctionType *FT = FunctionType::get(Result: Type::getVoidTy(C&: M->getContext()), isVarArg: false);
890 Function *NewF =
891 Function::Create(Ty: FT, Linkage: GlobalValue::ExternalWeakLinkage, BPF_TRAP, M);
892 NewF->setDSOLocal(true);
893 NewF->setCallingConv(CallingConv::C);
894 NewF->setSection(".ksyms");
895
896 if (M->debug_compile_units().empty())
897 return NewF;
898
899 DIBuilder DBuilder(*M);
900 DITypeArray ParamTypes =
901 DBuilder.getOrCreateTypeArray(Elements: {nullptr /*void return*/});
902 DISubroutineType *FuncType = DBuilder.createSubroutineType(ParameterTypes: ParamTypes);
903 DICompileUnit *CU = *M->debug_compile_units_begin();
904 DISubprogram *SP =
905 DBuilder.createFunction(Scope: CU, BPF_TRAP, BPF_TRAP, File: nullptr, LineNo: 0, Ty: FuncType, ScopeLine: 0,
906 Flags: DINode::FlagZero, SPFlags: DISubprogram::SPFlagZero);
907 NewF->setSubprogram(SP);
908 return NewF;
909}
910
911SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const {
912 MachineFunction &MF = DAG.getMachineFunction();
913 TargetLowering::CallLoweringInfo CLI(DAG);
914 SmallVector<SDValue> InVals;
915 SDNode *N = Op.getNode();
916 SDLoc DL(N);
917
918 Function *Fn = createBPFUnreachable(M: MF.getFunction().getParent());
919 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
920 CLI.Callee = DAG.getTargetGlobalAddress(GV: Fn, DL, VT: PtrVT);
921 CLI.Chain = N->getOperand(Num: 0);
922 CLI.IsTailCall = false;
923 CLI.CallConv = CallingConv::C;
924 CLI.IsVarArg = false;
925 CLI.DL = std::move(DL);
926 CLI.NoMerge = false;
927 CLI.DoesNotReturn = true;
928 return LowerCall(CLI, InVals);
929}
930
931/// The verifier wants to see that the load from the table is within it. The
932/// range check in front of it doesn't always show that: it can be done on
933/// another copy of the value, e.g. 'w1 = w7; w1 &= 255; if w1 > 24 goto' and
934/// then 'r1 = r7; r1 &= 255' for the index. Make the size of the table a power
935/// of two and mask the index.
936SDValue BPFTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
937 SDLoc DL(Op);
938 SDValue Chain = Op.getOperand(i: 0);
939 unsigned JTI = cast<JumpTableSDNode>(Val: Op.getOperand(i: 1))->getIndex();
940 SDValue Index = Op.getOperand(i: 2);
941 MachineFunction &MF = DAG.getMachineFunction();
942 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
943 std::vector<MachineBasicBlock *> MBBs = MJTI->getJumpTables()[JTI].MBBs;
944 uint64_t Size = PowerOf2Ceil(A: MBBs.size());
945
946 if (Size != MBBs.size()) {
947 // An index that gets there is out of range: any block of the table will do.
948 MBBs.resize(new_size: Size, x: MBBs.back());
949 MJTI->RemoveJumpTable(Idx: JTI);
950 JTI = MJTI->createJumpTableIndex(DestBBs: MBBs);
951 }
952
953 Index = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Index,
954 N2: DAG.getConstant(Val: Size - 1, DL, VT: MVT::i64));
955 Index = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Index,
956 N2: DAG.getConstant(Val: 3, DL, VT: MVT::i64));
957 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64,
958 N1: DAG.getJumpTable(JTI, VT: MVT::i64), N2: Index);
959 SDValue Load = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: Addr,
960 PtrInfo: MachinePointerInfo::getJumpTable(MF));
961 return DAG.getNode(Opcode: ISD::BRIND, DL, VT: MVT::Other, N1: Load.getValue(R: 1), N2: Load);
962}
963
964SDValue BPFTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
965 JumpTableSDNode *N = cast<JumpTableSDNode>(Val&: Op);
966 return getAddr(N, DAG);
967}
968
969static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
970 SelectionDAG &DAG, unsigned Flags) {
971 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
972 Offset: N->getOffset(), TargetFlags: Flags);
973}
974
975static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
976 SelectionDAG &DAG, unsigned Flags) {
977 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
978}
979
980template <class NodeTy>
981SDValue BPFTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
982 unsigned Flags) const {
983 SDLoc DL(N);
984
985 SDValue GA = getTargetNode(N, DL, MVT::i64, DAG, Flags);
986
987 return DAG.getNode(Opcode: BPFISD::Wrapper, DL, VT: MVT::i64, Operand: GA);
988}
989
990SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
991 SelectionDAG &DAG) const {
992 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
993 if (N->getOffset() != 0)
994 report_fatal_error(reason: "invalid offset for global address: " +
995 Twine(N->getOffset()));
996
997 const GlobalValue *GVal = N->getGlobal();
998 SDLoc DL(Op);
999
1000 // Wrap it in a TargetGlobalAddress
1001 SDValue Addr = DAG.getTargetGlobalAddress(GV: GVal, DL, VT: MVT::i64);
1002
1003 // Emit pseudo instruction
1004 return SDValue(DAG.getMachineNode(Opcode: BPF::LDIMM64, dl: DL, VT: MVT::i64, Op1: Addr), 0);
1005}
1006
1007SDValue BPFTargetLowering::LowerConstantPool(SDValue Op,
1008 SelectionDAG &DAG) const {
1009 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Val&: Op);
1010
1011 return getAddr(N, DAG);
1012}
1013
1014SDValue BPFTargetLowering::LowerBlockAddress(SDValue Op,
1015 SelectionDAG &DAG) const {
1016 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
1017 SDLoc DL(Op);
1018
1019 // Wrap it in a TargetBlockAddress
1020 SDValue Addr = DAG.getTargetBlockAddress(BA, VT: MVT::i64);
1021
1022 // Emit pseudo instruction
1023 return SDValue(DAG.getMachineNode(Opcode: BPF::LDIMM64, dl: DL, VT: MVT::i64, Op1: Addr), 0);
1024}
1025
1026unsigned
1027BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
1028 unsigned Reg, bool isSigned) const {
1029 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1030 const TargetRegisterClass *RC = getRegClassFor(VT: MVT::i64);
1031 int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
1032 MachineFunction *F = BB->getParent();
1033 DebugLoc DL = MI.getDebugLoc();
1034
1035 MachineRegisterInfo &RegInfo = F->getRegInfo();
1036
1037 if (!isSigned) {
1038 Register PromotedReg0 = RegInfo.createVirtualRegister(RegClass: RC);
1039 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::MOV_32_64), DestReg: PromotedReg0).addReg(RegNo: Reg);
1040 return PromotedReg0;
1041 }
1042 Register PromotedReg0 = RegInfo.createVirtualRegister(RegClass: RC);
1043 Register PromotedReg1 = RegInfo.createVirtualRegister(RegClass: RC);
1044 Register PromotedReg2 = RegInfo.createVirtualRegister(RegClass: RC);
1045 if (HasMovsx) {
1046 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::MOVSX_rr_32), DestReg: PromotedReg0).addReg(RegNo: Reg);
1047 } else {
1048 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::MOV_32_64), DestReg: PromotedReg0).addReg(RegNo: Reg);
1049 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::SLL_ri), DestReg: PromotedReg1)
1050 .addReg(RegNo: PromotedReg0).addImm(Val: 32);
1051 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: RShiftOp), DestReg: PromotedReg2)
1052 .addReg(RegNo: PromotedReg1).addImm(Val: 32);
1053 }
1054
1055 return PromotedReg2;
1056}
1057
1058MachineBasicBlock *
1059BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
1060 MachineBasicBlock *BB)
1061 const {
1062 MachineFunction *MF = MI.getParent()->getParent();
1063 MachineRegisterInfo &MRI = MF->getRegInfo();
1064 MachineInstrBuilder MIB(*MF, MI);
1065 unsigned ScratchReg;
1066
1067 // This function does custom insertion during lowering BPFISD::MEMCPY which
1068 // only has two register operands from memcpy semantics, the copy source
1069 // address and the copy destination address.
1070 //
1071 // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
1072 // a third scratch register to serve as the destination register of load and
1073 // source register of store.
1074 //
1075 // The scratch register here is with the Define | Dead | EarlyClobber flags.
1076 // The EarlyClobber flag has the semantic property that the operand it is
1077 // attached to is clobbered before the rest of the inputs are read. Hence it
1078 // must be unique among the operands to the instruction. The Define flag is
1079 // needed to coerce the machine verifier that an Undef value isn't a problem
1080 // as we anyway is loading memory into it. The Dead flag is needed as the
1081 // value in scratch isn't supposed to be used by any other instruction.
1082 ScratchReg = MRI.createVirtualRegister(RegClass: &BPF::GPRRegClass);
1083 MIB.addReg(RegNo: ScratchReg,
1084 Flags: RegState::Define | RegState::Dead | RegState::EarlyClobber);
1085
1086 return BB;
1087}
1088
1089MachineBasicBlock *BPFTargetLowering::EmitInstrWithCustomInserterLDimm64(
1090 MachineInstr &MI, MachineBasicBlock *BB) const {
1091 MachineFunction *MF = BB->getParent();
1092 const BPFInstrInfo *TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
1093 const TargetRegisterClass *RC = getRegClassFor(VT: MVT::i64);
1094 MachineRegisterInfo &RegInfo = MF->getRegInfo();
1095 DebugLoc DL = MI.getDebugLoc();
1096
1097 // Build address taken map for Global Varaibles and BlockAddresses
1098 DenseMap<const BasicBlock *, MachineBasicBlock *> AddressTakenBBs;
1099 for (MachineBasicBlock &MBB : *MF) {
1100 if (const BasicBlock *BB = MBB.getBasicBlock())
1101 if (BB->hasAddressTaken())
1102 AddressTakenBBs[BB] = &MBB;
1103 }
1104
1105 MachineOperand &MO = MI.getOperand(i: 1);
1106 assert(MO.isBlockAddress() || MO.isGlobal());
1107
1108 Register ResultReg = MI.getOperand(i: 0).getReg();
1109 Register TmpReg = RegInfo.createVirtualRegister(RegClass: RC);
1110
1111 std::vector<MachineBasicBlock *> Targets;
1112 unsigned JTI;
1113
1114 if (MO.isBlockAddress()) {
1115 auto *BA = MO.getBlockAddress();
1116 MachineBasicBlock *TgtMBB = AddressTakenBBs[BA->getBasicBlock()];
1117 assert(TgtMBB);
1118
1119 Targets.push_back(x: TgtMBB);
1120 JTI = MF->getOrCreateJumpTableInfo(JTEntryKind: getJumpTableEncoding())
1121 ->createJumpTableIndex(DestBBs: Targets);
1122
1123 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BPF::LD_imm64), DestReg: TmpReg)
1124 .addJumpTableIndex(Idx: JTI);
1125 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BPF::LDD), DestReg: ResultReg)
1126 .addReg(RegNo: TmpReg)
1127 .addImm(Val: 0);
1128 MI.eraseFromParent();
1129 return BB;
1130 }
1131
1132 // Helper: emit LD_imm64 with operand GlobalAddress or JumpTable
1133 auto emitLDImm64 = [&](const GlobalValue *GV = nullptr, unsigned JTI = -1) {
1134 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BPF::LD_imm64), DestReg: ResultReg);
1135 if (GV)
1136 MIB.addGlobalAddress(GV);
1137 else
1138 MIB.addJumpTableIndex(Idx: JTI);
1139 MI.eraseFromParent();
1140 return BB;
1141 };
1142
1143 // Must be a global at this point
1144 const GlobalValue *GVal = MO.getGlobal();
1145 const auto *GV = dyn_cast<GlobalVariable>(Val: GVal);
1146
1147 if (!GV || GV->getLinkage() != GlobalValue::PrivateLinkage ||
1148 !GV->isConstant() || !GV->hasInitializer())
1149 return emitLDImm64(GVal);
1150
1151 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1152 if (!CA)
1153 return emitLDImm64(GVal);
1154
1155 for (const Use &Op : CA->operands()) {
1156 if (!isa<BlockAddress>(Val: Op))
1157 return emitLDImm64(GVal);
1158 auto *BA = cast<BlockAddress>(Val: Op);
1159 MachineBasicBlock *TgtMBB = AddressTakenBBs[BA->getBasicBlock()];
1160 assert(TgtMBB);
1161 Targets.push_back(x: TgtMBB);
1162 }
1163
1164 JTI = MF->getOrCreateJumpTableInfo(JTEntryKind: getJumpTableEncoding())
1165 ->createJumpTableIndex(DestBBs: Targets);
1166 return emitLDImm64(nullptr, JTI);
1167}
1168
1169MachineBasicBlock *
1170BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
1171 MachineBasicBlock *BB) const {
1172 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1173 DebugLoc DL = MI.getDebugLoc();
1174 unsigned Opc = MI.getOpcode();
1175 bool isSelectRROp = (Opc == BPF::Select ||
1176 Opc == BPF::Select_64_32 ||
1177 Opc == BPF::Select_32 ||
1178 Opc == BPF::Select_32_64);
1179
1180 bool isMemcpyOp = Opc == BPF::MEMCPY;
1181 bool isLDimm64Op = Opc == BPF::LDIMM64;
1182
1183#ifndef NDEBUG
1184 bool isSelectRIOp = (Opc == BPF::Select_Ri ||
1185 Opc == BPF::Select_Ri_64_32 ||
1186 Opc == BPF::Select_Ri_32 ||
1187 Opc == BPF::Select_Ri_32_64);
1188
1189 if (!(isSelectRROp || isSelectRIOp || isMemcpyOp || isLDimm64Op))
1190 report_fatal_error("unhandled instruction type: " + Twine(Opc));
1191#endif
1192
1193 if (isMemcpyOp)
1194 return EmitInstrWithCustomInserterMemcpy(MI, BB);
1195
1196 if (isLDimm64Op)
1197 return EmitInstrWithCustomInserterLDimm64(MI, BB);
1198
1199 bool is32BitCmp = (Opc == BPF::Select_32 ||
1200 Opc == BPF::Select_32_64 ||
1201 Opc == BPF::Select_Ri_32 ||
1202 Opc == BPF::Select_Ri_32_64);
1203
1204 // To "insert" a SELECT instruction, we actually have to insert the diamond
1205 // control-flow pattern. The incoming instruction knows the destination vreg
1206 // to set, the condition code register to branch on, the true/false values to
1207 // select between, and a branch opcode to use.
1208 const BasicBlock *LLVM_BB = BB->getBasicBlock();
1209 MachineFunction::iterator I = ++BB->getIterator();
1210
1211 // ThisMBB:
1212 // ...
1213 // TrueVal = ...
1214 // jmp_XX r1, r2 goto Copy1MBB
1215 // fallthrough --> Copy0MBB
1216 MachineBasicBlock *ThisMBB = BB;
1217 MachineFunction *F = BB->getParent();
1218 MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
1219 MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
1220
1221 F->insert(MBBI: I, MBB: Copy0MBB);
1222 F->insert(MBBI: I, MBB: Copy1MBB);
1223
1224 // Set the call frame size on entry to the new basic blocks.
1225 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI);
1226 Copy0MBB->setCallFrameSize(CallFrameSize);
1227 Copy1MBB->setCallFrameSize(CallFrameSize);
1228
1229 // Update machine-CFG edges by transferring all successors of the current
1230 // block to the new block which will contain the Phi node for the select.
1231 Copy1MBB->splice(Where: Copy1MBB->begin(), Other: BB,
1232 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
1233 Copy1MBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
1234 // Next, add the true and fallthrough blocks as its successors.
1235 BB->addSuccessor(Succ: Copy0MBB);
1236 BB->addSuccessor(Succ: Copy1MBB);
1237
1238 // Insert Branch if Flag
1239 int CC = MI.getOperand(i: 3).getImm();
1240 int NewCC;
1241 switch (CC) {
1242#define SET_NEWCC(X, Y) \
1243 case ISD::X: \
1244 if (is32BitCmp && HasJmp32) \
1245 NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
1246 else \
1247 NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
1248 break
1249 SET_NEWCC(SETGT, JSGT);
1250 SET_NEWCC(SETUGT, JUGT);
1251 SET_NEWCC(SETGE, JSGE);
1252 SET_NEWCC(SETUGE, JUGE);
1253 SET_NEWCC(SETEQ, JEQ);
1254 SET_NEWCC(SETNE, JNE);
1255 SET_NEWCC(SETLT, JSLT);
1256 SET_NEWCC(SETULT, JULT);
1257 SET_NEWCC(SETLE, JSLE);
1258 SET_NEWCC(SETULE, JULE);
1259 default:
1260 report_fatal_error(reason: "unimplemented select CondCode " + Twine(CC));
1261 }
1262
1263 Register LHS = MI.getOperand(i: 1).getReg();
1264 bool isSignedCmp = (CC == ISD::SETGT ||
1265 CC == ISD::SETGE ||
1266 CC == ISD::SETLT ||
1267 CC == ISD::SETLE);
1268
1269 // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
1270 // to be promoted, however if the 32-bit comparison operands are destination
1271 // registers then they are implicitly zero-extended already, there is no
1272 // need of explicit zero-extend sequence for them.
1273 //
1274 // We simply do extension for all situations in this method, but we will
1275 // try to remove those unnecessary in BPFMIPeephole pass.
1276 if (is32BitCmp && !HasJmp32)
1277 LHS = EmitSubregExt(MI, BB, Reg: LHS, isSigned: isSignedCmp);
1278
1279 if (isSelectRROp) {
1280 Register RHS = MI.getOperand(i: 2).getReg();
1281
1282 if (is32BitCmp && !HasJmp32)
1283 RHS = EmitSubregExt(MI, BB, Reg: RHS, isSigned: isSignedCmp);
1284
1285 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: NewCC)).addReg(RegNo: LHS).addReg(RegNo: RHS).addMBB(MBB: Copy1MBB);
1286 } else {
1287 int64_t imm32 = MI.getOperand(i: 2).getImm();
1288 // Check before we build J*_ri instruction.
1289 if (!isInt<32>(x: imm32))
1290 report_fatal_error(reason: "immediate overflows 32 bits: " + Twine(imm32));
1291 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: NewCC))
1292 .addReg(RegNo: LHS).addImm(Val: imm32).addMBB(MBB: Copy1MBB);
1293 }
1294
1295 // Copy0MBB:
1296 // %FalseValue = ...
1297 // # fallthrough to Copy1MBB
1298 BB = Copy0MBB;
1299
1300 // Update machine-CFG edges
1301 BB->addSuccessor(Succ: Copy1MBB);
1302
1303 // Copy1MBB:
1304 // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
1305 // ...
1306 BB = Copy1MBB;
1307 BuildMI(BB&: *BB, I: BB->begin(), MIMD: DL, MCID: TII.get(Opcode: BPF::PHI), DestReg: MI.getOperand(i: 0).getReg())
1308 .addReg(RegNo: MI.getOperand(i: 5).getReg())
1309 .addMBB(MBB: Copy0MBB)
1310 .addReg(RegNo: MI.getOperand(i: 4).getReg())
1311 .addMBB(MBB: ThisMBB);
1312
1313 MI.eraseFromParent(); // The pseudo instruction is gone now.
1314 return BB;
1315}
1316
1317EVT BPFTargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
1318 EVT VT) const {
1319 return getHasAlu32() ? MVT::i32 : MVT::i64;
1320}
1321
1322MVT BPFTargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1323 EVT VT) const {
1324 return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
1325}
1326
1327bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1328 const AddrMode &AM, Type *Ty,
1329 unsigned AS,
1330 Instruction *I) const {
1331 // No global is ever allowed as a base.
1332 if (AM.BaseGV)
1333 return false;
1334
1335 switch (AM.Scale) {
1336 case 0: // "r+i" or just "i", depending on HasBaseReg.
1337 break;
1338 case 1:
1339 if (!AM.HasBaseReg) // allow "r+i".
1340 break;
1341 return false; // disallow "r+r" or "r+r+i".
1342 default:
1343 return false;
1344 }
1345
1346 return true;
1347}
1348
1349bool BPFTargetLowering::CanLowerReturn(
1350 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
1351 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
1352 const Type *RetTy) const {
1353 SmallVector<CCValAssign, 16> RVLocs;
1354 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
1355 return CCInfo.CheckReturn(Outs, Fn: getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
1356}
1357