1//===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that BPF uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "BPFISelLowering.h"
15#include "BPF.h"
16#include "BPFSubtarget.h"
17#include "llvm/CodeGen/CallingConvLower.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineJumpTableInfo.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
24#include "llvm/CodeGen/ValueTypes.h"
25#include "llvm/IR/DIBuilder.h"
26#include "llvm/IR/DiagnosticInfo.h"
27#include "llvm/IR/DiagnosticPrinter.h"
28#include "llvm/IR/Module.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Support/raw_ostream.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "bpf-lower"
37
38static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
39 cl::Hidden, cl::init(Val: false),
40 cl::desc("Expand memcpy into load/store pairs in order"));
41
42static cl::opt<unsigned> BPFMinimumJumpTableEntries(
43 "bpf-min-jump-table-entries", cl::init(Val: 13), cl::Hidden,
44 cl::desc("Set minimum number of entries to use a jump table on BPF"));
45
46static cl::opt<bool> BPFAllowsLibcalls(
47 "bpf-allows-libcalls", cl::Hidden, cl::init(Val: false),
48 cl::desc("Allow libcalls instead of rejecting unsupported built-in "
49 "functions"));
50
51static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg,
52 SDValue Val = {}) {
53 std::string Str;
54 if (Val) {
55 raw_string_ostream OS(Str);
56 Val->print(OS);
57 OS << ' ';
58 }
59 MachineFunction &MF = DAG.getMachineFunction();
60 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
61 MF.getFunction(), Twine(Str).concat(Suffix: Msg), DL.getDebugLoc()));
62}
63
64BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
65 const BPFSubtarget &STI)
66 : TargetLowering(TM, STI) {
67
68 // Set up the register classes.
69 addRegisterClass(VT: MVT::i64, RC: &BPF::GPRRegClass);
70 if (STI.getHasAlu32())
71 addRegisterClass(VT: MVT::i32, RC: &BPF::GPR32RegClass);
72
73 // Compute derived properties from the register classes
74 computeRegisterProperties(TRI: STI.getRegisterInfo());
75
76 setStackPointerRegisterToSaveRestore(BPF::R11);
77
78 setOperationAction(Op: ISD::BR_CC, VT: MVT::i64, Action: Custom);
79 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
80 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
81
82 if (!STI.hasGotox())
83 setOperationAction(Op: ISD::BRIND, VT: MVT::Other, Action: Expand);
84
85 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Custom);
86
87 setOperationAction(Ops: {ISD::GlobalAddress, ISD::ConstantPool}, VT: MVT::i64, Action: Custom);
88 if (STI.hasGotox())
89 setOperationAction(Ops: {ISD::JumpTable, ISD::BlockAddress}, VT: MVT::i64, Action: Custom);
90
91 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i64, Action: Custom);
92 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
93 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
94
95 // Set unsupported atomic operations as Custom so
96 // we can emit better error messages than fatal error
97 // from selectiondag.
98 for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
99 if (VT == MVT::i32) {
100 if (STI.getHasAlu32())
101 continue;
102 } else {
103 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT, Action: Custom);
104 }
105
106 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT, Action: Custom);
107 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT, Action: Custom);
108 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT, Action: Custom);
109 setOperationAction(Op: ISD::ATOMIC_SWAP, VT, Action: Custom);
110 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Action: Custom);
111 }
112
113 for (auto VT : {MVT::i32, MVT::i64}) {
114 setOperationAction(Op: ISD::ATOMIC_LOAD, VT, Action: Custom);
115 setOperationAction(Op: ISD::ATOMIC_STORE, VT, Action: Custom);
116 }
117
118 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
119
120 for (auto VT : { MVT::i32, MVT::i64 }) {
121 if (VT == MVT::i32 && !STI.getHasAlu32())
122 continue;
123
124 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
125 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
126 if (!STI.hasSdivSmod()) {
127 setOperationAction(Op: ISD::SDIV, VT, Action: Custom);
128 setOperationAction(Op: ISD::SREM, VT, Action: Custom);
129 }
130 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
131 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
132 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
133 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
134 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
135 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
136 setOperationAction(Op: ISD::SHL_PARTS, VT, Action: Custom);
137 setOperationAction(Op: ISD::SRL_PARTS, VT, Action: Custom);
138 setOperationAction(Op: ISD::SRA_PARTS, VT, Action: Custom);
139 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
140 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
141 setOperationAction(Op: ISD::CTLZ, VT, Action: Expand);
142 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT, Action: Expand);
143 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT, Action: Expand);
144
145 setOperationAction(Op: ISD::SETCC, VT, Action: Expand);
146 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
147 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
148 }
149
150 if (STI.getHasAlu32()) {
151 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Promote);
152 setOperationAction(Op: ISD::BR_CC, VT: MVT::i32,
153 Action: STI.getHasJmp32() ? Custom : Promote);
154 }
155
156 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
157 if (!STI.hasMovsx()) {
158 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i8, Action: Expand);
159 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i16, Action: Expand);
160 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i32, Action: Expand);
161 }
162
163 // Extended load operations for i1 types must be promoted
164 for (MVT VT : MVT::integer_valuetypes()) {
165 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
166 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
167 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
168
169 if (!STI.hasLdsx()) {
170 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i8, Action: Expand);
171 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i16, Action: Expand);
172 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i32, Action: Expand);
173 }
174 }
175
176 setBooleanContents(ZeroOrOneBooleanContent);
177 setMaxAtomicSizeInBitsSupported(64);
178 setMinimumJumpTableEntries(BPFMinimumJumpTableEntries);
179
180 // Function alignments
181 setMinFunctionAlignment(Align(8));
182 setPrefFunctionAlignment(Align(8));
183
184 if (BPFExpandMemcpyInOrder) {
185 // LLVM generic code will try to expand memcpy into load/store pairs at this
186 // stage which is before quite a few IR optimization passes, therefore the
187 // loads and stores could potentially be moved apart from each other which
188 // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
189 // compilers.
190 //
191 // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
192 // of memcpy to later stage in IR optimization pipeline so those load/store
193 // pairs won't be touched and could be kept in order. Hence, we set
194 // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
195 // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
196 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = 0;
197 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = 0;
198 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = 0;
199 MaxLoadsPerMemcmp = 0;
200 } else {
201 // inline memcpy() for kernel to see explicit copy
202 unsigned CommonMaxStores =
203 STI.getSelectionDAGInfo()->getCommonMaxStoresPerMemFunc();
204
205 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
206 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
207 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = CommonMaxStores;
208 MaxLoadsPerMemcmp = MaxLoadsPerMemcmpOptSize = CommonMaxStores;
209 }
210
211 // CPU/Feature control
212 HasAlu32 = STI.getHasAlu32();
213 HasJmp32 = STI.getHasJmp32();
214 HasJmpExt = STI.getHasJmpExt();
215 HasMovsx = STI.hasMovsx();
216
217 AllowsMisalignedMemAccess = STI.getAllowsMisalignedMemAccess();
218}
219
220bool BPFTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
221 MachineMemOperand::Flags,
222 unsigned *Fast) const {
223 // allows-misaligned-mem-access is disabled
224 if (!AllowsMisalignedMemAccess)
225 return false;
226
227 // only allow misalignment for simple value types
228 if (!VT.isSimple())
229 return false;
230
231 // always assume fast mode when misalignment is allowed
232 if (Fast)
233 *Fast = true;
234
235 return true;
236}
237
238bool BPFTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
239 return false;
240}
241
242bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
243 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
244 return false;
245 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
246 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
247 return NumBits1 > NumBits2;
248}
249
250bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
251 if (!VT1.isInteger() || !VT2.isInteger())
252 return false;
253 unsigned NumBits1 = VT1.getSizeInBits();
254 unsigned NumBits2 = VT2.getSizeInBits();
255 return NumBits1 > NumBits2;
256}
257
258bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
259 if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
260 return false;
261 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
262 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
263 return NumBits1 == 32 && NumBits2 == 64;
264}
265
266bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
267 if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
268 return false;
269 unsigned NumBits1 = VT1.getSizeInBits();
270 unsigned NumBits2 = VT2.getSizeInBits();
271 return NumBits1 == 32 && NumBits2 == 64;
272}
273
274bool BPFTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
275 EVT VT1 = Val.getValueType();
276 if (Val.getOpcode() == ISD::LOAD && VT1.isSimple() && VT2.isSimple()) {
277 MVT MT1 = VT1.getSimpleVT().SimpleTy;
278 MVT MT2 = VT2.getSimpleVT().SimpleTy;
279 if ((MT1 == MVT::i8 || MT1 == MVT::i16 || MT1 == MVT::i32) &&
280 (MT2 == MVT::i32 || MT2 == MVT::i64))
281 return true;
282 }
283 return TargetLoweringBase::isZExtFree(Val, VT2);
284}
285
286unsigned BPFTargetLowering::getJumpTableEncoding() const {
287 return MachineJumpTableInfo::EK_BlockAddress;
288}
289
290BPFTargetLowering::ConstraintType
291BPFTargetLowering::getConstraintType(StringRef Constraint) const {
292 if (Constraint.size() == 1) {
293 switch (Constraint[0]) {
294 default:
295 break;
296 case 'w':
297 return C_RegisterClass;
298 }
299 }
300
301 return TargetLowering::getConstraintType(Constraint);
302}
303
304std::pair<unsigned, const TargetRegisterClass *>
305BPFTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
306 StringRef Constraint,
307 MVT VT) const {
308 if (Constraint.size() == 1) {
309 // GCC Constraint Letters
310 switch (Constraint[0]) {
311 case 'r': // GENERAL_REGS
312 return std::make_pair(x: 0U, y: &BPF::GPRRegClass);
313 case 'w':
314 if (HasAlu32)
315 return std::make_pair(x: 0U, y: &BPF::GPR32RegClass);
316 break;
317 default:
318 break;
319 }
320 }
321
322 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
323}
324
325void BPFTargetLowering::ReplaceNodeResults(
326 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
327 const char *Msg;
328 uint32_t Opcode = N->getOpcode();
329 switch (Opcode) {
330 default:
331 report_fatal_error(reason: "unhandled custom legalization: " + Twine(Opcode));
332 case ISD::ATOMIC_LOAD_ADD:
333 case ISD::ATOMIC_LOAD_AND:
334 case ISD::ATOMIC_LOAD_OR:
335 case ISD::ATOMIC_LOAD_XOR:
336 case ISD::ATOMIC_SWAP:
337 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
338 if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD)
339 Msg = "unsupported atomic operation, please use 32/64 bit version";
340 else
341 Msg = "unsupported atomic operation, please use 64 bit version";
342 break;
343 case ISD::ATOMIC_LOAD:
344 case ISD::ATOMIC_STORE:
345 return;
346 }
347
348 SDLoc DL(N);
349 // We'll still produce a fatal error downstream, but this diagnostic is more
350 // user-friendly.
351 fail(DL, DAG, Msg);
352}
353
354SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
355 switch (Op.getOpcode()) {
356 default:
357 report_fatal_error(reason: "unimplemented opcode: " + Twine(Op.getOpcode()));
358 case ISD::BR_CC:
359 return LowerBR_CC(Op, DAG);
360 case ISD::JumpTable:
361 return LowerJumpTable(Op, DAG);
362 case ISD::GlobalAddress:
363 return LowerGlobalAddress(Op, DAG);
364 case ISD::ConstantPool:
365 return LowerConstantPool(Op, DAG);
366 case ISD::BlockAddress:
367 return LowerBlockAddress(Op, DAG);
368 case ISD::SELECT_CC:
369 return LowerSELECT_CC(Op, DAG);
370 case ISD::SDIV:
371 case ISD::SREM:
372 return LowerSDIVSREM(Op, DAG);
373 case ISD::SHL_PARTS:
374 case ISD::SRL_PARTS:
375 case ISD::SRA_PARTS:
376 return LowerShiftParts(Op, DAG);
377 case ISD::DYNAMIC_STACKALLOC:
378 return LowerDYNAMIC_STACKALLOC(Op, DAG);
379 case ISD::ATOMIC_LOAD:
380 case ISD::ATOMIC_STORE:
381 return LowerATOMIC_LOAD_STORE(Op, DAG);
382 case ISD::ATOMIC_FENCE:
383 return LowerATOMIC_FENCE(Op, DAG);
384 case ISD::TRAP:
385 return LowerTRAP(Op, DAG);
386 }
387}
388
389// Calling Convention Implementation
390#define GET_CALLING_CONV_IMPL
391#include "BPFGenCallingConv.inc"
392
393// Apply AssertSext/AssertZext and truncate based on VA's LocInfo.
394static SDValue convertLocValType(SelectionDAG &DAG, const SDLoc &DL,
395 const CCValAssign &VA, EVT RegVT,
396 SDValue ArgValue) {
397 if (VA.getLocInfo() == CCValAssign::SExt)
398 ArgValue = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: RegVT, N1: ArgValue,
399 N2: DAG.getValueType(VA.getValVT()));
400 else if (VA.getLocInfo() == CCValAssign::ZExt)
401 ArgValue = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: RegVT, N1: ArgValue,
402 N2: DAG.getValueType(VA.getValVT()));
403 if (VA.getLocInfo() != CCValAssign::Full)
404 ArgValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: ArgValue);
405 return ArgValue;
406}
407
408SDValue BPFTargetLowering::LowerFormalArguments(
409 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
410 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
411 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
412 switch (CallConv) {
413 default:
414 report_fatal_error(reason: "unimplemented calling convention: " + Twine(CallConv));
415 case CallingConv::C:
416 case CallingConv::Fast:
417 break;
418 }
419
420 MachineFunction &MF = DAG.getMachineFunction();
421 MachineRegisterInfo &RegInfo = MF.getRegInfo();
422
423 // Assign locations to all of the incoming arguments.
424 SmallVector<CCValAssign, 16> ArgLocs;
425 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
426 CCInfo.AnalyzeFormalArguments(Ins, Fn: getHasAlu32() ? CC_BPF32 : CC_BPF64);
427
428 for (size_t I = 0; I < ArgLocs.size(); ++I) {
429 auto &VA = ArgLocs[I];
430 EVT RegVT = VA.getLocVT();
431
432 if (VA.isRegLoc()) {
433 // Arguments passed in registers
434 MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
435 switch (SimpleTy) {
436 default: {
437 std::string Str;
438 {
439 raw_string_ostream OS(Str);
440 RegVT.print(OS);
441 }
442 report_fatal_error(reason: "unhandled argument type: " + Twine(Str));
443 }
444 case MVT::i32:
445 case MVT::i64:
446 Register VReg = RegInfo.createVirtualRegister(
447 RegClass: SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
448 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
449 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: RegVT);
450 InVals.push_back(Elt: convertLocValType(DAG, DL, VA, RegVT, ArgValue));
451 break;
452 }
453 continue;
454 }
455
456 if (VA.isMemLoc()) {
457 // For example, two stack arguments,
458 // arg1: Off = 8
459 // arg2: off = 16
460 int Off = VA.getLocMemOffset() + 8;
461 if (Off > INT16_MAX) {
462 fail(DL, DAG, Msg: "extra parameter stack depth exceeded limit");
463 break;
464 }
465
466 // Physical extra argument slot is always 64-bit.
467 SDValue StackVal = DAG.getNode(Opcode: BPFISD::LOAD_STACK_ARG, DL,
468 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain,
469 N2: DAG.getConstant(Val: Off, DL, VT: MVT::i64));
470 SDValue ArgValue = StackVal.getValue(R: 0);
471 Chain = StackVal.getValue(R: 1);
472 InVals.push_back(Elt: convertLocValType(DAG, DL, VA, RegVT: MVT::i64, ArgValue));
473 continue;
474 }
475 }
476
477 if (IsVarArg)
478 fail(DL, DAG, Msg: "variadic functions are not supported");
479 return Chain;
480}
481
482static void resetRegMaskBit(const TargetRegisterInfo *TRI, uint32_t *RegMask,
483 MCRegister Reg) {
484 for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg))
485 RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
486}
487
488static uint32_t *regMaskFromTemplate(const TargetRegisterInfo *TRI,
489 MachineFunction &MF,
490 const uint32_t *BaseRegMask) {
491 uint32_t *RegMask = MF.allocateRegMask();
492 unsigned RegMaskSize = MachineOperand::getRegMaskSize(NumRegs: TRI->getNumRegs());
493 memcpy(dest: RegMask, src: BaseRegMask, n: sizeof(RegMask[0]) * RegMaskSize);
494 return RegMask;
495}
496
497SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
498 SmallVectorImpl<SDValue> &InVals) const {
499 SelectionDAG &DAG = CLI.DAG;
500 auto &Outs = CLI.Outs;
501 auto &OutVals = CLI.OutVals;
502 auto &Ins = CLI.Ins;
503 SDValue Chain = CLI.Chain;
504 SDValue Callee = CLI.Callee;
505 bool &IsTailCall = CLI.IsTailCall;
506 CallingConv::ID CallConv = CLI.CallConv;
507 bool IsVarArg = CLI.IsVarArg;
508 MachineFunction &MF = DAG.getMachineFunction();
509
510 // BPF target does not support tail call optimization.
511 IsTailCall = false;
512
513 switch (CallConv) {
514 default:
515 report_fatal_error(reason: "unsupported calling convention: " + Twine(CallConv));
516 case CallingConv::Fast:
517 case CallingConv::C:
518 break;
519 }
520
521 // Analyze operands of the call, assigning locations to each operand.
522 SmallVector<CCValAssign, 16> ArgLocs;
523 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
524
525 CCInfo.AnalyzeCallOperands(Outs, Fn: getHasAlu32() ? CC_BPF32 : CC_BPF64);
526
527 unsigned NumBytes = CCInfo.getStackSize();
528
529 for (auto &Arg : Outs) {
530 ISD::ArgFlagsTy Flags = Arg.Flags;
531 if (!Flags.isByVal())
532 continue;
533 fail(DL: CLI.DL, DAG, Msg: "pass by value not supported", Val: Callee);
534 break;
535 }
536
537 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
538 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
539
540 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
541
542 // Walk arg assignments
543 for (size_t i = 0; i < OutVals.size(); ++i) {
544 CCValAssign &VA = ArgLocs[i];
545 SDValue &Arg = OutVals[i];
546
547 // Promote the value if needed.
548 switch (VA.getLocInfo()) {
549 default:
550 report_fatal_error(reason: "unhandled location info: " + Twine(VA.getLocInfo()));
551 case CCValAssign::Full:
552 break;
553 case CCValAssign::SExt:
554 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: CLI.DL, VT: VA.getLocVT(), Operand: Arg);
555 break;
556 case CCValAssign::ZExt:
557 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: CLI.DL, VT: VA.getLocVT(), Operand: Arg);
558 break;
559 case CCValAssign::AExt:
560 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: CLI.DL, VT: VA.getLocVT(), Operand: Arg);
561 break;
562 }
563
564 // Push arguments into RegsToPass vector
565 if (VA.isRegLoc()) {
566 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
567 continue;
568 }
569
570 if (VA.isMemLoc()) {
571 int Off = -8 - VA.getLocMemOffset();
572 if (Off < INT16_MIN) {
573 fail(DL: CLI.DL, DAG, Msg: "extra parameter stack depth exceeded limit");
574 break;
575 }
576
577 // STORE_STACK_ARG requires i64 operands. With ALU32 mode, the CC
578 // promotion may only extend to i32, so extend to i64 if needed.
579 if (Arg.getValueType() != MVT::i64)
580 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: CLI.DL, VT: MVT::i64, Operand: Arg);
581
582 SDValue OffVal = DAG.getConstant(Val: Off, DL: CLI.DL, VT: MVT::i64);
583 Chain = DAG.getNode(Opcode: BPFISD::STORE_STACK_ARG, DL: CLI.DL, VT: MVT::Other, N1: Chain,
584 N2: OffVal, N3: Arg);
585 continue;
586 }
587
588 report_fatal_error(reason: "unhandled argument location");
589 }
590
591 SDValue InGlue;
592
593 // Build a sequence of copy-to-reg nodes chained together with token chain and
594 // flag operands which copy the outgoing args into registers. The InGlue in
595 // necessary since all emitted instructions must be stuck together.
596 for (auto &Reg : RegsToPass) {
597 Chain = DAG.getCopyToReg(Chain, dl: CLI.DL, Reg: Reg.first, N: Reg.second, Glue: InGlue);
598 InGlue = Chain.getValue(R: 1);
599 }
600
601 // If the callee is a GlobalAddress node (quite common, every direct call is)
602 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
603 // Likewise ExternalSymbol -> TargetExternalSymbol.
604 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
605 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL: CLI.DL, VT: PtrVT,
606 offset: G->getOffset(), TargetFlags: 0);
607 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
608 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT, TargetFlags: 0);
609 StringRef Sym = E->getSymbol();
610 if (!BPFAllowsLibcalls && Sym != BPF_TRAP && Sym != "__multi3" &&
611 Sym != "__divti3" && Sym != "__modti3" && Sym != "__udivti3" &&
612 Sym != "__umodti3" && Sym != "memcpy" && Sym != "memset" &&
613 Sym != "memmove")
614 fail(
615 DL: CLI.DL, DAG,
616 Msg: Twine("A call to built-in function '" + Sym + "' is not supported."));
617 }
618
619 // Returns a chain & a flag for retval copy to use.
620 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
621 SmallVector<SDValue, 8> Ops;
622 Ops.push_back(Elt: Chain);
623 Ops.push_back(Elt: Callee);
624
625 // Add argument registers to the end of the list so that they are
626 // known live into the call.
627 for (auto &Reg : RegsToPass)
628 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
629
630 bool HasFastCall =
631 (CLI.CB && isa<CallInst>(Val: CLI.CB) && CLI.CB->hasFnAttr(Kind: "bpf_fastcall"));
632 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
633 if (HasFastCall) {
634 uint32_t *RegMask = regMaskFromTemplate(
635 TRI, MF, BaseRegMask: TRI->getCallPreservedMask(MF, CallingConv::PreserveAll));
636 for (auto const &RegPair : RegsToPass)
637 resetRegMaskBit(TRI, RegMask, Reg: RegPair.first);
638 if (!CLI.CB->getType()->isVoidTy())
639 resetRegMaskBit(TRI, RegMask, Reg: BPF::R0);
640 Ops.push_back(Elt: DAG.getRegisterMask(RegMask));
641 } else {
642 Ops.push_back(
643 Elt: DAG.getRegisterMask(RegMask: TRI->getCallPreservedMask(MF, CLI.CallConv)));
644 }
645
646 if (InGlue.getNode())
647 Ops.push_back(Elt: InGlue);
648
649 Chain = DAG.getNode(Opcode: BPFISD::CALL, DL: CLI.DL, VTList: NodeTys, Ops);
650 InGlue = Chain.getValue(R: 1);
651
652 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
653
654 // Create the CALLSEQ_END node.
655 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL: CLI.DL);
656 InGlue = Chain.getValue(R: 1);
657
658 // Handle result values, copying them out of physregs into vregs that we
659 // return.
660 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL: CLI.DL, DAG,
661 InVals);
662}
663
664SDValue
665BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
666 bool IsVarArg,
667 const SmallVectorImpl<ISD::OutputArg> &Outs,
668 const SmallVectorImpl<SDValue> &OutVals,
669 const SDLoc &DL, SelectionDAG &DAG) const {
670 unsigned Opc = BPFISD::RET_GLUE;
671
672 // CCValAssign - represent the assignment of the return value to a location
673 SmallVector<CCValAssign, 16> RVLocs;
674 MachineFunction &MF = DAG.getMachineFunction();
675
676 // CCState - Info about the registers and stack slot.
677 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
678
679 // Analize return values.
680 CCInfo.AnalyzeReturn(Outs, Fn: getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
681
682 SDValue Glue;
683 SmallVector<SDValue, 4> RetOps(1, Chain);
684
685 // Copy the result values into the output registers.
686 for (size_t i = 0; i != RVLocs.size(); ++i) {
687 CCValAssign &VA = RVLocs[i];
688 if (!VA.isRegLoc())
689 report_fatal_error(reason: "stack return values are not supported");
690
691 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: OutVals[i], Glue);
692
693 // Guarantee that all emitted copies are stuck together,
694 // avoiding something bad.
695 Glue = Chain.getValue(R: 1);
696 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
697 }
698
699 RetOps[0] = Chain; // Update chain.
700
701 // Add the glue if we have it.
702 if (Glue.getNode())
703 RetOps.push_back(Elt: Glue);
704
705 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
706}
707
708SDValue BPFTargetLowering::LowerCallResult(
709 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
710 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
711 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
712
713 MachineFunction &MF = DAG.getMachineFunction();
714 // Assign locations to each value returned by this call.
715 SmallVector<CCValAssign, 16> RVLocs;
716 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
717
718 CCInfo.AnalyzeCallResult(Ins, Fn: getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
719
720 // Copy all of the result registers out of their specified physreg.
721 for (auto &Val : RVLocs) {
722 Chain = DAG.getCopyFromReg(Chain, dl: DL, Reg: Val.getLocReg(),
723 VT: Val.getValVT(), Glue: InGlue).getValue(R: 1);
724 InGlue = Chain.getValue(R: 2);
725 InVals.push_back(Elt: Chain.getValue(R: 0));
726 }
727
728 return Chain;
729}
730
731static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
732 switch (CC) {
733 default:
734 break;
735 case ISD::SETULT:
736 case ISD::SETULE:
737 case ISD::SETLT:
738 case ISD::SETLE:
739 CC = ISD::getSetCCSwappedOperands(Operation: CC);
740 std::swap(a&: LHS, b&: RHS);
741 break;
742 }
743}
744
745SDValue BPFTargetLowering::LowerSDIVSREM(SDValue Op, SelectionDAG &DAG) const {
746 SDLoc DL(Op);
747 fail(DL, DAG,
748 Msg: "unsupported signed division, please convert to unsigned div/mod.");
749 return DAG.getUNDEF(VT: Op->getValueType(ResNo: 0));
750}
751
752SDValue BPFTargetLowering::LowerShiftParts(SDValue Op,
753 SelectionDAG &DAG) const {
754 SDValue Lo, Hi;
755 expandShiftParts(N: Op.getNode(), Lo, Hi, DAG);
756 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: SDLoc(Op));
757}
758
759SDValue BPFTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
760 SelectionDAG &DAG) const {
761 SDLoc DL(Op);
762 fail(DL, DAG, Msg: "unsupported dynamic stack allocation");
763 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
764 return DAG.getMergeValues(Ops, dl: SDLoc());
765}
766
767SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
768 SDValue Chain = Op.getOperand(i: 0);
769 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
770 SDValue LHS = Op.getOperand(i: 2);
771 SDValue RHS = Op.getOperand(i: 3);
772 SDValue Dest = Op.getOperand(i: 4);
773 SDLoc DL(Op);
774
775 if (!getHasJmpExt())
776 NegateCC(LHS, RHS, CC);
777
778 return DAG.getNode(Opcode: BPFISD::BR_CC, DL, VT: Op.getValueType(), N1: Chain, N2: LHS, N3: RHS,
779 N4: DAG.getConstant(Val: CC, DL, VT: LHS.getValueType()), N5: Dest);
780}
781
782SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
783 SDValue LHS = Op.getOperand(i: 0);
784 SDValue RHS = Op.getOperand(i: 1);
785 SDValue TrueV = Op.getOperand(i: 2);
786 SDValue FalseV = Op.getOperand(i: 3);
787 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
788 SDLoc DL(Op);
789
790 if (!getHasJmpExt())
791 NegateCC(LHS, RHS, CC);
792
793 SDValue TargetCC = DAG.getConstant(Val: CC, DL, VT: LHS.getValueType());
794 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
795
796 return DAG.getNode(Opcode: BPFISD::SELECT_CC, DL, VT: Op.getValueType(), Ops);
797}
798
799SDValue BPFTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
800 SelectionDAG &DAG) const {
801 SDNode *N = Op.getNode();
802 SDLoc DL(N);
803
804 if (cast<AtomicSDNode>(Val: N)->getMergedOrdering() ==
805 AtomicOrdering::SequentiallyConsistent)
806 fail(DL, DAG,
807 Msg: "sequentially consistent (seq_cst) "
808 "atomic load/store is not supported");
809
810 return Op;
811}
812
813SDValue BPFTargetLowering::LowerATOMIC_FENCE(SDValue Op,
814 SelectionDAG &DAG) const {
815 SDLoc DL(Op);
816 SyncScope::ID FenceSSID =
817 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
818
819 if (FenceSSID == SyncScope::SingleThread)
820 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
821 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
822
823 report_fatal_error(reason: "Runtime fence is not supported at the moment");
824}
825
826static Function *createBPFUnreachable(Module *M) {
827 if (auto *Fn = M->getFunction(BPF_TRAP))
828 return Fn;
829
830 FunctionType *FT = FunctionType::get(Result: Type::getVoidTy(C&: M->getContext()), isVarArg: false);
831 Function *NewF =
832 Function::Create(Ty: FT, Linkage: GlobalValue::ExternalWeakLinkage, BPF_TRAP, M);
833 NewF->setDSOLocal(true);
834 NewF->setCallingConv(CallingConv::C);
835 NewF->setSection(".ksyms");
836
837 if (M->debug_compile_units().empty())
838 return NewF;
839
840 DIBuilder DBuilder(*M);
841 DITypeArray ParamTypes =
842 DBuilder.getOrCreateTypeArray(Elements: {nullptr /*void return*/});
843 DISubroutineType *FuncType = DBuilder.createSubroutineType(ParameterTypes: ParamTypes);
844 DICompileUnit *CU = *M->debug_compile_units_begin();
845 DISubprogram *SP =
846 DBuilder.createFunction(Scope: CU, BPF_TRAP, BPF_TRAP, File: nullptr, LineNo: 0, Ty: FuncType, ScopeLine: 0,
847 Flags: DINode::FlagZero, SPFlags: DISubprogram::SPFlagZero);
848 NewF->setSubprogram(SP);
849 return NewF;
850}
851
852SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const {
853 MachineFunction &MF = DAG.getMachineFunction();
854 TargetLowering::CallLoweringInfo CLI(DAG);
855 SmallVector<SDValue> InVals;
856 SDNode *N = Op.getNode();
857 SDLoc DL(N);
858
859 Function *Fn = createBPFUnreachable(M: MF.getFunction().getParent());
860 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
861 CLI.Callee = DAG.getTargetGlobalAddress(GV: Fn, DL, VT: PtrVT);
862 CLI.Chain = N->getOperand(Num: 0);
863 CLI.IsTailCall = false;
864 CLI.CallConv = CallingConv::C;
865 CLI.IsVarArg = false;
866 CLI.DL = std::move(DL);
867 CLI.NoMerge = false;
868 CLI.DoesNotReturn = true;
869 return LowerCall(CLI, InVals);
870}
871
872SDValue BPFTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
873 JumpTableSDNode *N = cast<JumpTableSDNode>(Val&: Op);
874 return getAddr(N, DAG);
875}
876
877static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
878 SelectionDAG &DAG, unsigned Flags) {
879 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
880 Offset: N->getOffset(), TargetFlags: Flags);
881}
882
883static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
884 SelectionDAG &DAG, unsigned Flags) {
885 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
886}
887
888template <class NodeTy>
889SDValue BPFTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
890 unsigned Flags) const {
891 SDLoc DL(N);
892
893 SDValue GA = getTargetNode(N, DL, MVT::i64, DAG, Flags);
894
895 return DAG.getNode(Opcode: BPFISD::Wrapper, DL, VT: MVT::i64, Operand: GA);
896}
897
898SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
899 SelectionDAG &DAG) const {
900 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
901 if (N->getOffset() != 0)
902 report_fatal_error(reason: "invalid offset for global address: " +
903 Twine(N->getOffset()));
904
905 const GlobalValue *GVal = N->getGlobal();
906 SDLoc DL(Op);
907
908 // Wrap it in a TargetGlobalAddress
909 SDValue Addr = DAG.getTargetGlobalAddress(GV: GVal, DL, VT: MVT::i64);
910
911 // Emit pseudo instruction
912 return SDValue(DAG.getMachineNode(Opcode: BPF::LDIMM64, dl: DL, VT: MVT::i64, Op1: Addr), 0);
913}
914
915SDValue BPFTargetLowering::LowerConstantPool(SDValue Op,
916 SelectionDAG &DAG) const {
917 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Val&: Op);
918
919 return getAddr(N, DAG);
920}
921
922SDValue BPFTargetLowering::LowerBlockAddress(SDValue Op,
923 SelectionDAG &DAG) const {
924 const BlockAddress *BA = cast<BlockAddressSDNode>(Val&: Op)->getBlockAddress();
925 SDLoc DL(Op);
926
927 // Wrap it in a TargetBlockAddress
928 SDValue Addr = DAG.getTargetBlockAddress(BA, VT: MVT::i64);
929
930 // Emit pseudo instruction
931 return SDValue(DAG.getMachineNode(Opcode: BPF::LDIMM64, dl: DL, VT: MVT::i64, Op1: Addr), 0);
932}
933
934unsigned
935BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
936 unsigned Reg, bool isSigned) const {
937 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
938 const TargetRegisterClass *RC = getRegClassFor(VT: MVT::i64);
939 int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
940 MachineFunction *F = BB->getParent();
941 DebugLoc DL = MI.getDebugLoc();
942
943 MachineRegisterInfo &RegInfo = F->getRegInfo();
944
945 if (!isSigned) {
946 Register PromotedReg0 = RegInfo.createVirtualRegister(RegClass: RC);
947 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::MOV_32_64), DestReg: PromotedReg0).addReg(RegNo: Reg);
948 return PromotedReg0;
949 }
950 Register PromotedReg0 = RegInfo.createVirtualRegister(RegClass: RC);
951 Register PromotedReg1 = RegInfo.createVirtualRegister(RegClass: RC);
952 Register PromotedReg2 = RegInfo.createVirtualRegister(RegClass: RC);
953 if (HasMovsx) {
954 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::MOVSX_rr_32), DestReg: PromotedReg0).addReg(RegNo: Reg);
955 } else {
956 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::MOV_32_64), DestReg: PromotedReg0).addReg(RegNo: Reg);
957 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: BPF::SLL_ri), DestReg: PromotedReg1)
958 .addReg(RegNo: PromotedReg0).addImm(Val: 32);
959 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: RShiftOp), DestReg: PromotedReg2)
960 .addReg(RegNo: PromotedReg1).addImm(Val: 32);
961 }
962
963 return PromotedReg2;
964}
965
966MachineBasicBlock *
967BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
968 MachineBasicBlock *BB)
969 const {
970 MachineFunction *MF = MI.getParent()->getParent();
971 MachineRegisterInfo &MRI = MF->getRegInfo();
972 MachineInstrBuilder MIB(*MF, MI);
973 unsigned ScratchReg;
974
975 // This function does custom insertion during lowering BPFISD::MEMCPY which
976 // only has two register operands from memcpy semantics, the copy source
977 // address and the copy destination address.
978 //
979 // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
980 // a third scratch register to serve as the destination register of load and
981 // source register of store.
982 //
983 // The scratch register here is with the Define | Dead | EarlyClobber flags.
984 // The EarlyClobber flag has the semantic property that the operand it is
985 // attached to is clobbered before the rest of the inputs are read. Hence it
986 // must be unique among the operands to the instruction. The Define flag is
987 // needed to coerce the machine verifier that an Undef value isn't a problem
988 // as we anyway is loading memory into it. The Dead flag is needed as the
989 // value in scratch isn't supposed to be used by any other instruction.
990 ScratchReg = MRI.createVirtualRegister(RegClass: &BPF::GPRRegClass);
991 MIB.addReg(RegNo: ScratchReg,
992 Flags: RegState::Define | RegState::Dead | RegState::EarlyClobber);
993
994 return BB;
995}
996
997MachineBasicBlock *BPFTargetLowering::EmitInstrWithCustomInserterLDimm64(
998 MachineInstr &MI, MachineBasicBlock *BB) const {
999 MachineFunction *MF = BB->getParent();
1000 const BPFInstrInfo *TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
1001 const TargetRegisterClass *RC = getRegClassFor(VT: MVT::i64);
1002 MachineRegisterInfo &RegInfo = MF->getRegInfo();
1003 DebugLoc DL = MI.getDebugLoc();
1004
1005 // Build address taken map for Global Varaibles and BlockAddresses
1006 DenseMap<const BasicBlock *, MachineBasicBlock *> AddressTakenBBs;
1007 for (MachineBasicBlock &MBB : *MF) {
1008 if (const BasicBlock *BB = MBB.getBasicBlock())
1009 if (BB->hasAddressTaken())
1010 AddressTakenBBs[BB] = &MBB;
1011 }
1012
1013 MachineOperand &MO = MI.getOperand(i: 1);
1014 assert(MO.isBlockAddress() || MO.isGlobal());
1015
1016 Register ResultReg = MI.getOperand(i: 0).getReg();
1017 Register TmpReg = RegInfo.createVirtualRegister(RegClass: RC);
1018
1019 std::vector<MachineBasicBlock *> Targets;
1020 unsigned JTI;
1021
1022 if (MO.isBlockAddress()) {
1023 auto *BA = MO.getBlockAddress();
1024 MachineBasicBlock *TgtMBB = AddressTakenBBs[BA->getBasicBlock()];
1025 assert(TgtMBB);
1026
1027 Targets.push_back(x: TgtMBB);
1028 JTI = MF->getOrCreateJumpTableInfo(JTEntryKind: getJumpTableEncoding())
1029 ->createJumpTableIndex(DestBBs: Targets);
1030
1031 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BPF::LD_imm64), DestReg: TmpReg)
1032 .addJumpTableIndex(Idx: JTI);
1033 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BPF::LDD), DestReg: ResultReg)
1034 .addReg(RegNo: TmpReg)
1035 .addImm(Val: 0);
1036 MI.eraseFromParent();
1037 return BB;
1038 }
1039
1040 // Helper: emit LD_imm64 with operand GlobalAddress or JumpTable
1041 auto emitLDImm64 = [&](const GlobalValue *GV = nullptr, unsigned JTI = -1) {
1042 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BPF::LD_imm64), DestReg: ResultReg);
1043 if (GV)
1044 MIB.addGlobalAddress(GV);
1045 else
1046 MIB.addJumpTableIndex(Idx: JTI);
1047 MI.eraseFromParent();
1048 return BB;
1049 };
1050
1051 // Must be a global at this point
1052 const GlobalValue *GVal = MO.getGlobal();
1053 const auto *GV = dyn_cast<GlobalVariable>(Val: GVal);
1054
1055 if (!GV || GV->getLinkage() != GlobalValue::PrivateLinkage ||
1056 !GV->isConstant() || !GV->hasInitializer())
1057 return emitLDImm64(GVal);
1058
1059 const auto *CA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
1060 if (!CA)
1061 return emitLDImm64(GVal);
1062
1063 for (const Use &Op : CA->operands()) {
1064 if (!isa<BlockAddress>(Val: Op))
1065 return emitLDImm64(GVal);
1066 auto *BA = cast<BlockAddress>(Val: Op);
1067 MachineBasicBlock *TgtMBB = AddressTakenBBs[BA->getBasicBlock()];
1068 assert(TgtMBB);
1069 Targets.push_back(x: TgtMBB);
1070 }
1071
1072 JTI = MF->getOrCreateJumpTableInfo(JTEntryKind: getJumpTableEncoding())
1073 ->createJumpTableIndex(DestBBs: Targets);
1074 return emitLDImm64(nullptr, JTI);
1075}
1076
1077MachineBasicBlock *
1078BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
1079 MachineBasicBlock *BB) const {
1080 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1081 DebugLoc DL = MI.getDebugLoc();
1082 unsigned Opc = MI.getOpcode();
1083 bool isSelectRROp = (Opc == BPF::Select ||
1084 Opc == BPF::Select_64_32 ||
1085 Opc == BPF::Select_32 ||
1086 Opc == BPF::Select_32_64);
1087
1088 bool isMemcpyOp = Opc == BPF::MEMCPY;
1089 bool isLDimm64Op = Opc == BPF::LDIMM64;
1090
1091#ifndef NDEBUG
1092 bool isSelectRIOp = (Opc == BPF::Select_Ri ||
1093 Opc == BPF::Select_Ri_64_32 ||
1094 Opc == BPF::Select_Ri_32 ||
1095 Opc == BPF::Select_Ri_32_64);
1096
1097 if (!(isSelectRROp || isSelectRIOp || isMemcpyOp || isLDimm64Op))
1098 report_fatal_error("unhandled instruction type: " + Twine(Opc));
1099#endif
1100
1101 if (isMemcpyOp)
1102 return EmitInstrWithCustomInserterMemcpy(MI, BB);
1103
1104 if (isLDimm64Op)
1105 return EmitInstrWithCustomInserterLDimm64(MI, BB);
1106
1107 bool is32BitCmp = (Opc == BPF::Select_32 ||
1108 Opc == BPF::Select_32_64 ||
1109 Opc == BPF::Select_Ri_32 ||
1110 Opc == BPF::Select_Ri_32_64);
1111
1112 // To "insert" a SELECT instruction, we actually have to insert the diamond
1113 // control-flow pattern. The incoming instruction knows the destination vreg
1114 // to set, the condition code register to branch on, the true/false values to
1115 // select between, and a branch opcode to use.
1116 const BasicBlock *LLVM_BB = BB->getBasicBlock();
1117 MachineFunction::iterator I = ++BB->getIterator();
1118
1119 // ThisMBB:
1120 // ...
1121 // TrueVal = ...
1122 // jmp_XX r1, r2 goto Copy1MBB
1123 // fallthrough --> Copy0MBB
1124 MachineBasicBlock *ThisMBB = BB;
1125 MachineFunction *F = BB->getParent();
1126 MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
1127 MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
1128
1129 F->insert(MBBI: I, MBB: Copy0MBB);
1130 F->insert(MBBI: I, MBB: Copy1MBB);
1131 // Update machine-CFG edges by transferring all successors of the current
1132 // block to the new block which will contain the Phi node for the select.
1133 Copy1MBB->splice(Where: Copy1MBB->begin(), Other: BB,
1134 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
1135 Copy1MBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
1136 // Next, add the true and fallthrough blocks as its successors.
1137 BB->addSuccessor(Succ: Copy0MBB);
1138 BB->addSuccessor(Succ: Copy1MBB);
1139
1140 // Insert Branch if Flag
1141 int CC = MI.getOperand(i: 3).getImm();
1142 int NewCC;
1143 switch (CC) {
1144#define SET_NEWCC(X, Y) \
1145 case ISD::X: \
1146 if (is32BitCmp && HasJmp32) \
1147 NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
1148 else \
1149 NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
1150 break
1151 SET_NEWCC(SETGT, JSGT);
1152 SET_NEWCC(SETUGT, JUGT);
1153 SET_NEWCC(SETGE, JSGE);
1154 SET_NEWCC(SETUGE, JUGE);
1155 SET_NEWCC(SETEQ, JEQ);
1156 SET_NEWCC(SETNE, JNE);
1157 SET_NEWCC(SETLT, JSLT);
1158 SET_NEWCC(SETULT, JULT);
1159 SET_NEWCC(SETLE, JSLE);
1160 SET_NEWCC(SETULE, JULE);
1161 default:
1162 report_fatal_error(reason: "unimplemented select CondCode " + Twine(CC));
1163 }
1164
1165 Register LHS = MI.getOperand(i: 1).getReg();
1166 bool isSignedCmp = (CC == ISD::SETGT ||
1167 CC == ISD::SETGE ||
1168 CC == ISD::SETLT ||
1169 CC == ISD::SETLE);
1170
1171 // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
1172 // to be promoted, however if the 32-bit comparison operands are destination
1173 // registers then they are implicitly zero-extended already, there is no
1174 // need of explicit zero-extend sequence for them.
1175 //
1176 // We simply do extension for all situations in this method, but we will
1177 // try to remove those unnecessary in BPFMIPeephole pass.
1178 if (is32BitCmp && !HasJmp32)
1179 LHS = EmitSubregExt(MI, BB, Reg: LHS, isSigned: isSignedCmp);
1180
1181 if (isSelectRROp) {
1182 Register RHS = MI.getOperand(i: 2).getReg();
1183
1184 if (is32BitCmp && !HasJmp32)
1185 RHS = EmitSubregExt(MI, BB, Reg: RHS, isSigned: isSignedCmp);
1186
1187 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: NewCC)).addReg(RegNo: LHS).addReg(RegNo: RHS).addMBB(MBB: Copy1MBB);
1188 } else {
1189 int64_t imm32 = MI.getOperand(i: 2).getImm();
1190 // Check before we build J*_ri instruction.
1191 if (!isInt<32>(x: imm32))
1192 report_fatal_error(reason: "immediate overflows 32 bits: " + Twine(imm32));
1193 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: NewCC))
1194 .addReg(RegNo: LHS).addImm(Val: imm32).addMBB(MBB: Copy1MBB);
1195 }
1196
1197 // Copy0MBB:
1198 // %FalseValue = ...
1199 // # fallthrough to Copy1MBB
1200 BB = Copy0MBB;
1201
1202 // Update machine-CFG edges
1203 BB->addSuccessor(Succ: Copy1MBB);
1204
1205 // Copy1MBB:
1206 // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
1207 // ...
1208 BB = Copy1MBB;
1209 BuildMI(BB&: *BB, I: BB->begin(), MIMD: DL, MCID: TII.get(Opcode: BPF::PHI), DestReg: MI.getOperand(i: 0).getReg())
1210 .addReg(RegNo: MI.getOperand(i: 5).getReg())
1211 .addMBB(MBB: Copy0MBB)
1212 .addReg(RegNo: MI.getOperand(i: 4).getReg())
1213 .addMBB(MBB: ThisMBB);
1214
1215 MI.eraseFromParent(); // The pseudo instruction is gone now.
1216 return BB;
1217}
1218
1219EVT BPFTargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
1220 EVT VT) const {
1221 return getHasAlu32() ? MVT::i32 : MVT::i64;
1222}
1223
1224MVT BPFTargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1225 EVT VT) const {
1226 return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
1227}
1228
1229bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1230 const AddrMode &AM, Type *Ty,
1231 unsigned AS,
1232 Instruction *I) const {
1233 // No global is ever allowed as a base.
1234 if (AM.BaseGV)
1235 return false;
1236
1237 switch (AM.Scale) {
1238 case 0: // "r+i" or just "i", depending on HasBaseReg.
1239 break;
1240 case 1:
1241 if (!AM.HasBaseReg) // allow "r+i".
1242 break;
1243 return false; // disallow "r+r" or "r+r+i".
1244 default:
1245 return false;
1246 }
1247
1248 return true;
1249}
1250
1251bool BPFTargetLowering::CanLowerReturn(
1252 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
1253 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
1254 const Type *RetTy) const {
1255 SmallVector<CCValAssign, 16> RVLocs;
1256 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
1257 return CCInfo.CheckReturn(Outs, Fn: getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
1258}
1259