1//===-- lib/CodeGen/GlobalISel/InlineAsmLowering.cpp ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering from LLVM IR inline asm to MIR INLINEASM
11///
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
15#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineOperand.h"
18#include "llvm/CodeGen/MachineRegisterInfo.h"
19#include "llvm/CodeGen/TargetLowering.h"
20#include "llvm/IR/DiagnosticInfo.h"
21#include "llvm/IR/Module.h"
22
23#define DEBUG_TYPE "inline-asm-lowering"
24
25using namespace llvm;
26
27void InlineAsmLowering::anchor() {}
28
29/// Emit an inline asm error diagnostic and materialize undef values for the
30/// call results so that the rest of the function remains well-formed.
31static void emitInlineAsmError(MachineIRBuilder &MIRBuilder,
32 const CallBase &Call, const Twine &Message,
33 ArrayRef<Register> ResRegs) {
34 Call.getContext().diagnose(DI: DiagnosticInfoInlineAsm(Call, Message));
35 for (Register Reg : ResRegs)
36 MIRBuilder.buildUndef(Res: Reg);
37}
38
39namespace {
40
41/// GISelAsmOperandInfo - This contains information for each constraint that we
42/// are lowering.
43class GISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
44public:
45 /// Regs - If this is a register or register class operand, this
46 /// contains the set of assigned registers corresponding to the operand.
47 SmallVector<Register, 1> Regs;
48
49 explicit GISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &Info)
50 : TargetLowering::AsmOperandInfo(Info) {}
51};
52
53using GISelAsmOperandInfoVector = SmallVector<GISelAsmOperandInfo, 16>;
54
55class ExtraFlags {
56 unsigned Flags = 0;
57
58public:
59 explicit ExtraFlags(const CallBase &CB) {
60 const InlineAsm *IA = cast<InlineAsm>(Val: CB.getCalledOperand());
61 if (IA->hasSideEffects())
62 Flags |= InlineAsm::Extra_HasSideEffects;
63 if (IA->isAlignStack())
64 Flags |= InlineAsm::Extra_IsAlignStack;
65 if (IA->canThrow())
66 Flags |= InlineAsm::Extra_MayUnwind;
67 if (CB.isConvergent())
68 Flags |= InlineAsm::Extra_IsConvergent;
69 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
70 }
71
72 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
73 // Ideally, we would only check against memory constraints. However, the
74 // meaning of an Other constraint can be target-specific and we can't easily
75 // reason about it. Therefore, be conservative and set MayLoad/MayStore
76 // for Other constraints as well.
77 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
78 OpInfo.ConstraintType == TargetLowering::C_Other) {
79 if (OpInfo.Type == InlineAsm::isInput)
80 Flags |= InlineAsm::Extra_MayLoad;
81 else if (OpInfo.Type == InlineAsm::isOutput)
82 Flags |= InlineAsm::Extra_MayStore;
83 else if (OpInfo.Type == InlineAsm::isClobber)
84 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
85 }
86 }
87
88 unsigned get() const { return Flags; }
89};
90
91} // namespace
92
93/// Assign virtual/physical registers for the specified register operand.
94static void getRegistersForValue(MachineFunction &MF,
95 MachineIRBuilder &MIRBuilder,
96 GISelAsmOperandInfo &OpInfo,
97 GISelAsmOperandInfo &RefOpInfo) {
98
99 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
100 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
101
102 // No work to do for memory operations.
103 if (OpInfo.ConstraintType == TargetLowering::C_Memory)
104 return;
105
106 // If this is a constraint for a single physreg, or a constraint for a
107 // register class, find it.
108 Register AssignedReg;
109 const TargetRegisterClass *RC;
110 std::tie(args&: AssignedReg, args&: RC) = TLI.getRegForInlineAsmConstraint(
111 TRI: &TRI, Constraint: RefOpInfo.ConstraintCode, VT: RefOpInfo.ConstraintVT);
112 // RC is unset only on failure. Return immediately.
113 if (!RC)
114 return;
115
116 // No need to allocate a matching input constraint since the constraint it's
117 // matching to has already been allocated.
118 if (OpInfo.isMatchingInputConstraint())
119 return;
120
121 // Initialize NumRegs.
122 unsigned NumRegs = 1;
123 if (OpInfo.ConstraintVT != MVT::Other)
124 NumRegs =
125 TLI.getNumRegisters(Context&: MF.getFunction().getContext(), VT: OpInfo.ConstraintVT);
126
127 // If this is a constraint for a specific physical register, but the type of
128 // the operand requires more than one register to be passed, we allocate the
129 // required amount of physical registers, starting from the selected physical
130 // register.
131 // For this, first retrieve a register iterator for the given register class
132 TargetRegisterClass::iterator I = RC->begin();
133 MachineRegisterInfo &RegInfo = MF.getRegInfo();
134
135 // Advance the iterator to the assigned register (if set)
136 if (AssignedReg) {
137 for (; *I != AssignedReg; ++I)
138 assert(I != RC->end() && "AssignedReg should be a member of provided RC");
139 }
140
141 // Finally, assign the registers. If the AssignedReg isn't set, create virtual
142 // registers with the provided register class
143 for (; NumRegs; --NumRegs, ++I) {
144 assert(I != RC->end() && "Ran out of registers to allocate!");
145 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RegClass: RC);
146 OpInfo.Regs.push_back(Elt: R);
147 }
148}
149
150static void computeConstraintToUse(const TargetLowering *TLI,
151 TargetLowering::AsmOperandInfo &OpInfo) {
152 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
153
154 // Single-letter constraints ('r') are very common.
155 if (OpInfo.Codes.size() == 1) {
156 OpInfo.ConstraintCode = OpInfo.Codes[0];
157 OpInfo.ConstraintType = TLI->getConstraintType(Constraint: OpInfo.ConstraintCode);
158 } else {
159 TargetLowering::ConstraintGroup G = TLI->getConstraintPreferences(OpInfo);
160 if (G.empty())
161 return;
162 // FIXME: prefer immediate constraints if the target allows it
163 unsigned BestIdx = 0;
164 for (const unsigned E = G.size();
165 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
166 G[BestIdx].second == TargetLowering::C_Immediate);
167 ++BestIdx)
168 ;
169 OpInfo.ConstraintCode = G[BestIdx].first;
170 OpInfo.ConstraintType = G[BestIdx].second;
171 }
172
173 // 'X' matches anything.
174 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
175 // Labels and constants are handled elsewhere ('X' is the only thing
176 // that matches labels). For Functions, the type here is the type of
177 // the result, which is not what we want to look at; leave them alone.
178 Value *Val = OpInfo.CallOperandVal;
179 if (isa<BasicBlock>(Val) || isa<ConstantInt>(Val) || isa<Function>(Val))
180 return;
181
182 // Otherwise, try to resolve it to something we know about by looking at
183 // the actual operand type.
184 if (const char *Repl = TLI->LowerXConstraint(ConstraintVT: OpInfo.ConstraintVT)) {
185 OpInfo.ConstraintCode = Repl;
186 OpInfo.ConstraintType = TLI->getConstraintType(Constraint: OpInfo.ConstraintCode);
187 }
188 }
189}
190
191static unsigned getNumOpRegs(const MachineInstr &I, unsigned OpIdx) {
192 const InlineAsm::Flag F(I.getOperand(i: OpIdx).getImm());
193 return F.getNumOperandRegisters();
194}
195
196static bool buildAnyextOrCopy(Register Dst, Register Src,
197 MachineIRBuilder &MIRBuilder) {
198 const TargetRegisterInfo *TRI =
199 MIRBuilder.getMF().getSubtarget().getRegisterInfo();
200 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
201
202 auto SrcTy = MRI->getType(Reg: Src);
203 if (!SrcTy.isValid()) {
204 LLVM_DEBUG(dbgs() << "Source type for copy is not valid\n");
205 return false;
206 }
207 unsigned SrcSize = TRI->getRegSizeInBits(Reg: Src, MRI: *MRI);
208 unsigned DstSize = TRI->getRegSizeInBits(Reg: Dst, MRI: *MRI);
209
210 if (DstSize < SrcSize) {
211 LLVM_DEBUG(dbgs() << "Input can't fit in destination reg class\n");
212 return false;
213 }
214
215 // Attempt to anyext small scalar sources.
216 if (DstSize > SrcSize) {
217 if (!SrcTy.isScalar()) {
218 LLVM_DEBUG(dbgs() << "Can't extend non-scalar input to size of"
219 "destination register class\n");
220 return false;
221 }
222 Src = MIRBuilder.buildAnyExt(Res: LLT::scalar(SizeInBits: DstSize), Op: Src).getReg(Idx: 0);
223 }
224
225 MIRBuilder.buildCopy(Res: Dst, Op: Src);
226 return true;
227}
228
229bool InlineAsmLowering::lowerInlineAsm(
230 MachineIRBuilder &MIRBuilder, const CallBase &Call,
231 std::function<ArrayRef<Register>(const Value &Val)> GetOrCreateVRegs)
232 const {
233 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
234
235 /// ConstraintOperands - Information about all of the constraints.
236 GISelAsmOperandInfoVector ConstraintOperands;
237
238 MachineFunction &MF = MIRBuilder.getMF();
239 const Function &F = MF.getFunction();
240 const DataLayout &DL = F.getDataLayout();
241 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
242
243 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
244
245 TargetLowering::AsmOperandInfoVector TargetConstraints =
246 TLI->ParseConstraints(DL, TRI, Call);
247
248 ExtraFlags ExtraInfo(Call);
249 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
250 unsigned ResNo = 0; // ResNo - The result number of the next output.
251 for (auto &T : TargetConstraints) {
252 ConstraintOperands.push_back(Elt: GISelAsmOperandInfo(T));
253 GISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
254
255 // Compute the value type for each operand.
256 if (OpInfo.hasArg()) {
257 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
258
259 if (isa<BasicBlock>(Val: OpInfo.CallOperandVal)) {
260 LLVM_DEBUG(dbgs() << "Basic block input operands not supported yet\n");
261 return false;
262 }
263
264 Type *OpTy = OpInfo.CallOperandVal->getType();
265
266 // If this is an indirect operand, the operand is a pointer to the
267 // accessed type.
268 if (OpInfo.isIndirect) {
269 OpTy = Call.getParamElementType(ArgNo);
270 assert(OpTy && "Indirect operand must have elementtype attribute");
271 }
272
273 // FIXME: Support aggregate input operands
274 if (!OpTy->isSingleValueType()) {
275 LLVM_DEBUG(
276 dbgs() << "Aggregate input operands are not supported yet\n");
277 return false;
278 }
279
280 OpInfo.ConstraintVT =
281 TLI->getAsmOperandValueType(DL, Ty: OpTy, AllowUnknown: true).getSimpleVT();
282 ++ArgNo;
283 } else if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
284 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
285 if (StructType *STy = dyn_cast<StructType>(Val: Call.getType())) {
286 OpInfo.ConstraintVT =
287 TLI->getSimpleValueType(DL, Ty: STy->getElementType(N: ResNo));
288 } else {
289 assert(ResNo == 0 && "Asm only has one result!");
290 OpInfo.ConstraintVT =
291 TLI->getAsmOperandValueType(DL, Ty: Call.getType()).getSimpleVT();
292 }
293 ++ResNo;
294 } else {
295 assert(OpInfo.Type != InlineAsm::isLabel &&
296 "GlobalISel currently doesn't support callbr");
297 OpInfo.ConstraintVT = MVT::Other;
298 }
299
300 if (OpInfo.ConstraintVT == MVT::i64x8)
301 return false;
302
303 // Compute the constraint code and ConstraintType to use.
304 computeConstraintToUse(TLI, OpInfo);
305
306 // The selected constraint type might expose new sideeffects
307 ExtraInfo.update(OpInfo);
308 }
309
310 // At this point, all operand types are decided.
311 // Create the MachineInstr, but don't insert it yet since input
312 // operands still need to insert instructions before this one
313 auto Inst = MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::INLINEASM)
314 .addExternalSymbol(FnName: IA->getAsmString().data())
315 .addImm(Val: ExtraInfo.get());
316
317 // Starting from this operand: flag followed by register(s) will be added as
318 // operands to Inst for each constraint. Used for matching input constraints.
319 unsigned StartIdx = Inst->getNumOperands();
320
321 // Collects the output operands for later processing
322 GISelAsmOperandInfoVector OutputOperands;
323
324 for (auto &OpInfo : ConstraintOperands) {
325 GISelAsmOperandInfo &RefOpInfo =
326 OpInfo.isMatchingInputConstraint()
327 ? ConstraintOperands[OpInfo.getMatchedOperand()]
328 : OpInfo;
329
330 // Assign registers for register operands
331 getRegistersForValue(MF, MIRBuilder, OpInfo, RefOpInfo);
332
333 switch (OpInfo.Type) {
334 case InlineAsm::isOutput:
335 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
336 const InlineAsm::ConstraintCode ConstraintID =
337 TLI->getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
338 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
339 "Failed to convert memory constraint code to constraint id.");
340
341 // Add information to the INLINEASM instruction to know about this
342 // output.
343 InlineAsm::Flag Flag(InlineAsm::Kind::Mem, 1);
344 Flag.setMemConstraint(ConstraintID);
345 Inst.addImm(Val: Flag);
346 ArrayRef<Register> SourceRegs =
347 GetOrCreateVRegs(*OpInfo.CallOperandVal);
348 assert(
349 SourceRegs.size() == 1 &&
350 "Expected the memory output to fit into a single virtual register");
351 Inst.addReg(RegNo: SourceRegs[0]);
352 } else {
353 // Otherwise, this outputs to a register (directly for C_Register /
354 // C_RegisterClass/C_Other.
355 assert(OpInfo.ConstraintType == TargetLowering::C_Register ||
356 OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
357 OpInfo.ConstraintType == TargetLowering::C_Other);
358
359 // Find a register that we can use.
360 if (OpInfo.Regs.empty()) {
361 emitInlineAsmError(MIRBuilder, Call,
362 Message: "could not allocate output register for "
363 "constraint '" +
364 Twine(OpInfo.ConstraintCode) + "'",
365 ResRegs: GetOrCreateVRegs(Call));
366 return true;
367 }
368
369 // Add information to the INLINEASM instruction to know that this
370 // register is set.
371 InlineAsm::Flag Flag(OpInfo.isEarlyClobber
372 ? InlineAsm::Kind::RegDefEarlyClobber
373 : InlineAsm::Kind::RegDef,
374 OpInfo.Regs.size());
375 if (OpInfo.Regs.front().isVirtual()) {
376 // Put the register class of the virtual registers in the flag word.
377 // That way, later passes can recompute register class constraints for
378 // inline assembly as well as normal instructions. Don't do this for
379 // tied operands that can use the regclass information from the def.
380 const TargetRegisterClass *RC = MRI->getRegClass(Reg: OpInfo.Regs.front());
381 Flag.setRegClass(RC->getID());
382 }
383
384 Inst.addImm(Val: Flag);
385
386 for (Register Reg : OpInfo.Regs) {
387 Inst.addReg(RegNo: Reg, Flags: RegState::Define |
388 getImplRegState(B: Reg.isPhysical()) |
389 getEarlyClobberRegState(B: OpInfo.isEarlyClobber));
390 }
391
392 // Remember this output operand for later processing
393 OutputOperands.push_back(Elt: OpInfo);
394 }
395
396 break;
397 case InlineAsm::isInput:
398 case InlineAsm::isLabel: {
399 if (OpInfo.isMatchingInputConstraint()) {
400 unsigned DefIdx = OpInfo.getMatchedOperand();
401 // Find operand with register def that corresponds to DefIdx.
402 unsigned InstFlagIdx = StartIdx;
403 for (unsigned i = 0; i < DefIdx; ++i)
404 InstFlagIdx += getNumOpRegs(I: *Inst, OpIdx: InstFlagIdx) + 1;
405 assert(getNumOpRegs(*Inst, InstFlagIdx) == 1 && "Wrong flag");
406
407 const InlineAsm::Flag MatchedOperandFlag(Inst->getOperand(i: InstFlagIdx).getImm());
408 if (MatchedOperandFlag.isMemKind()) {
409 LLVM_DEBUG(dbgs() << "Matching input constraint to mem operand not "
410 "supported. This should be target specific.\n");
411 return false;
412 }
413 if (!MatchedOperandFlag.isRegDefKind() && !MatchedOperandFlag.isRegDefEarlyClobberKind()) {
414 LLVM_DEBUG(dbgs() << "Unknown matching constraint\n");
415 return false;
416 }
417
418 // We want to tie input to register in next operand.
419 unsigned DefRegIdx = InstFlagIdx + 1;
420 Register Def = Inst->getOperand(i: DefRegIdx).getReg();
421
422 ArrayRef<Register> SrcRegs = GetOrCreateVRegs(*OpInfo.CallOperandVal);
423 assert(SrcRegs.size() == 1 && "Single register is expected here");
424
425 // We need the tied input to live in the same register class as the def.
426 //
427 // - if Def is a vreg, we can just use its regclass.
428 // - if Def is a physreg, create a vreg in the minimal regclass for that
429 // physreg.
430 //
431 // Otherwise RegBankSelect may leave it in the wrong bank (e.g. GPR even
432 // though it's tied to an FP physreg).
433 const TargetRegisterClass *RC = Def.isVirtual()
434 ? MRI->getRegClass(Reg: Def)
435 : TRI->getMinimalPhysRegClass(Reg: Def);
436
437 // Materialize `In` in a new vreg that has a register class that matches
438 // the register class of `Def`.
439 Register In = MRI->createVirtualRegister(RegClass: RC);
440 if (!buildAnyextOrCopy(Dst: In, Src: SrcRegs[0], MIRBuilder))
441 return false;
442
443 // Add Flag and input register operand (In) to Inst. Tie In to Def.
444 InlineAsm::Flag UseFlag(InlineAsm::Kind::RegUse, 1);
445 UseFlag.setMatchingOp(DefIdx);
446 Inst.addImm(Val: UseFlag);
447 Inst.addReg(RegNo: In);
448 Inst->tieOperands(DefIdx: DefRegIdx, UseIdx: Inst->getNumOperands() - 1);
449 break;
450 }
451
452 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
453 OpInfo.isIndirect) {
454 LLVM_DEBUG(dbgs() << "Indirect input operands with unknown constraint "
455 "not supported yet\n");
456 return false;
457 }
458
459 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
460 OpInfo.ConstraintType == TargetLowering::C_Other) {
461
462 std::vector<MachineOperand> Ops;
463 if (!lowerAsmOperandForConstraint(Val: OpInfo.CallOperandVal,
464 Constraint: OpInfo.ConstraintCode, Ops,
465 MIRBuilder)) {
466 LLVM_DEBUG(dbgs() << "Don't support constraint: "
467 << OpInfo.ConstraintCode << " yet\n");
468 return false;
469 }
470
471 assert(Ops.size() > 0 &&
472 "Expected constraint to be lowered to at least one operand");
473
474 // Add information to the INLINEASM node to know about this input.
475 const unsigned OpFlags =
476 InlineAsm::Flag(InlineAsm::Kind::Imm, Ops.size());
477 Inst.addImm(Val: OpFlags);
478 Inst.add(MOs: Ops);
479 break;
480 }
481
482 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
483 const InlineAsm::ConstraintCode ConstraintID =
484 TLI->getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
485 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
486 OpFlags.setMemConstraint(ConstraintID);
487 Inst.addImm(Val: OpFlags);
488
489 if (OpInfo.isIndirect) {
490 // already indirect
491 ArrayRef<Register> SourceRegs =
492 GetOrCreateVRegs(*OpInfo.CallOperandVal);
493 if (SourceRegs.size() != 1) {
494 LLVM_DEBUG(dbgs() << "Expected the memory input to fit into a "
495 "single virtual register "
496 "for constraint '"
497 << OpInfo.ConstraintCode << "'\n");
498 return false;
499 }
500 Inst.addReg(RegNo: SourceRegs[0]);
501 break;
502 }
503
504 // Needs to be made indirect. Store the value on the stack and use
505 // a pointer to it.
506 Value *OpVal = OpInfo.CallOperandVal;
507 TypeSize Bytes = DL.getTypeStoreSize(Ty: OpVal->getType());
508 Align Alignment = DL.getPrefTypeAlign(Ty: OpVal->getType());
509 int FrameIdx =
510 MF.getFrameInfo().CreateStackObject(Size: Bytes, Alignment, isSpillSlot: false);
511
512 unsigned AddrSpace = DL.getAllocaAddrSpace();
513 LLT FramePtrTy =
514 LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
515 auto Ptr = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx).getReg(Idx: 0);
516 ArrayRef<Register> SourceRegs =
517 GetOrCreateVRegs(*OpInfo.CallOperandVal);
518 if (SourceRegs.size() != 1) {
519 LLVM_DEBUG(dbgs() << "Expected the memory input to fit into a single "
520 "virtual register "
521 "for constraint '"
522 << OpInfo.ConstraintCode << "'\n");
523 return false;
524 }
525 MIRBuilder.buildStore(Val: SourceRegs[0], Addr: Ptr,
526 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdx),
527 Alignment);
528 Inst.addReg(RegNo: Ptr);
529 break;
530 }
531
532 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
533 OpInfo.ConstraintType == TargetLowering::C_Register) &&
534 "Unknown constraint type!");
535
536 if (OpInfo.isIndirect) {
537 LLVM_DEBUG(dbgs() << "Can't handle indirect register inputs yet "
538 "for constraint '"
539 << OpInfo.ConstraintCode << "'\n");
540 return false;
541 }
542
543 // Copy the input into the appropriate registers.
544 if (OpInfo.Regs.empty()) {
545 emitInlineAsmError(MIRBuilder, Call,
546 Message: "could not allocate input reg for constraint '" +
547 Twine(OpInfo.ConstraintCode) + "'",
548 ResRegs: GetOrCreateVRegs(Call));
549 return true;
550 }
551
552 unsigned NumRegs = OpInfo.Regs.size();
553 ArrayRef<Register> SourceRegs = GetOrCreateVRegs(*OpInfo.CallOperandVal);
554 assert(NumRegs == SourceRegs.size() &&
555 "Expected the number of input registers to match the number of "
556 "source registers");
557
558 if (NumRegs > 1) {
559 LLVM_DEBUG(dbgs() << "Input operands with multiple input registers are "
560 "not supported yet\n");
561 return false;
562 }
563
564 InlineAsm::Flag Flag(InlineAsm::Kind::RegUse, NumRegs);
565 if (OpInfo.Regs.front().isVirtual()) {
566 // Put the register class of the virtual registers in the flag word.
567 const TargetRegisterClass *RC = MRI->getRegClass(Reg: OpInfo.Regs.front());
568 Flag.setRegClass(RC->getID());
569 }
570 Inst.addImm(Val: Flag);
571 if (!buildAnyextOrCopy(Dst: OpInfo.Regs[0], Src: SourceRegs[0], MIRBuilder))
572 return false;
573 Inst.addReg(RegNo: OpInfo.Regs[0]);
574 break;
575 }
576
577 case InlineAsm::isClobber: {
578
579 const unsigned NumRegs = OpInfo.Regs.size();
580 if (NumRegs > 0) {
581 unsigned Flag = InlineAsm::Flag(InlineAsm::Kind::Clobber, NumRegs);
582 Inst.addImm(Val: Flag);
583
584 for (Register Reg : OpInfo.Regs) {
585 Inst.addReg(RegNo: Reg, Flags: RegState::Define | RegState::EarlyClobber |
586 getImplRegState(B: Reg.isPhysical()));
587 }
588 }
589 break;
590 }
591 }
592 }
593
594 if (auto Bundle = Call.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
595 auto *Token = Bundle->Inputs[0].get();
596 ArrayRef<Register> SourceRegs = GetOrCreateVRegs(*Token);
597 assert(SourceRegs.size() == 1 &&
598 "Expected the control token to fit into a single virtual register");
599 Inst.addUse(RegNo: SourceRegs[0], Flags: RegState::Implicit);
600 }
601
602 if (const MDNode *SrcLoc = Call.getMetadata(Kind: "srcloc"))
603 Inst.addMetadata(MD: SrcLoc);
604
605 // Add rounding control registers as implicit def for inline asm.
606 if (MF.getFunction().hasFnAttribute(Kind: Attribute::StrictFP)) {
607 ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
608 for (MCPhysReg Reg : RCRegs)
609 Inst.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
610 }
611
612 // All inputs are handled, insert the instruction now
613 MIRBuilder.insertInstr(MIB: Inst);
614
615 // Finally, copy the output operands into the output registers
616 ArrayRef<Register> ResRegs = GetOrCreateVRegs(Call);
617 if (ResRegs.size() != OutputOperands.size()) {
618 LLVM_DEBUG(dbgs() << "Expected the number of output registers to match the "
619 "number of destination registers\n");
620 return false;
621 }
622 for (unsigned int i = 0, e = ResRegs.size(); i < e; i++) {
623 GISelAsmOperandInfo &OpInfo = OutputOperands[i];
624
625 if (OpInfo.Regs.empty())
626 continue;
627
628 switch (OpInfo.ConstraintType) {
629 case TargetLowering::C_Register:
630 case TargetLowering::C_RegisterClass: {
631 if (OpInfo.Regs.size() > 1) {
632 LLVM_DEBUG(dbgs() << "Output operands with multiple defining "
633 "registers are not supported yet\n");
634 return false;
635 }
636
637 Register SrcReg = OpInfo.Regs[0];
638 unsigned SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
639 LLT ResTy = MRI->getType(Reg: ResRegs[i]);
640 if (ResTy.isScalar() && ResTy.getSizeInBits() < SrcSize) {
641 // First copy the non-typed virtual register into a generic virtual
642 // register
643 Register Tmp1Reg =
644 MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: SrcSize));
645 MIRBuilder.buildCopy(Res: Tmp1Reg, Op: SrcReg);
646 // Need to truncate the result of the register
647 MIRBuilder.buildTrunc(Res: ResRegs[i], Op: Tmp1Reg);
648 } else if (ResTy.getSizeInBits() == SrcSize) {
649 MIRBuilder.buildCopy(Res: ResRegs[i], Op: SrcReg);
650 } else {
651 LLVM_DEBUG(dbgs() << "Unhandled output operand with "
652 "mismatched register size\n");
653 return false;
654 }
655
656 break;
657 }
658 case TargetLowering::C_Immediate:
659 case TargetLowering::C_Other:
660 LLVM_DEBUG(
661 dbgs() << "Cannot lower target specific output constraints yet\n");
662 return false;
663 case TargetLowering::C_Memory:
664 break; // Already handled.
665 case TargetLowering::C_Address:
666 break; // Silence warning.
667 case TargetLowering::C_Unknown:
668 LLVM_DEBUG(dbgs() << "Unexpected unknown constraint\n");
669 return false;
670 }
671 }
672
673 return true;
674}
675
676bool InlineAsmLowering::lowerAsmOperandForConstraint(
677 Value *Val, StringRef Constraint, std::vector<MachineOperand> &Ops,
678 MachineIRBuilder &MIRBuilder) const {
679 if (Constraint.size() > 1)
680 return false;
681
682 char ConstraintLetter = Constraint[0];
683 switch (ConstraintLetter) {
684 default:
685 return false;
686 case 's': // Integer immediate not known at compile time
687 if (const auto *GV = dyn_cast<GlobalValue>(Val)) {
688 Ops.push_back(x: MachineOperand::CreateGA(GV, /*Offset=*/0));
689 return true;
690 }
691 return false;
692 case 'i': // Simple Integer or Relocatable Constant
693 if (const auto *GV = dyn_cast<GlobalValue>(Val)) {
694 Ops.push_back(x: MachineOperand::CreateGA(GV, /*Offset=*/0));
695 return true;
696 }
697 [[fallthrough]];
698 case 'n': // immediate integer with a known value.
699 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
700 assert(CI->getBitWidth() <= 64 &&
701 "expected immediate to fit into 64-bits");
702 // Boolean constants should be zero-extended, others are sign-extended
703 bool IsBool = CI->getBitWidth() == 1;
704 int64_t ExtVal = IsBool ? CI->getZExtValue() : CI->getSExtValue();
705 Ops.push_back(x: MachineOperand::CreateImm(Val: ExtVal));
706 return true;
707 }
708 return false;
709 }
710}
711