1//===-- lib/CodeGen/GlobalISel/InlineAsmLowering.cpp ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering from LLVM IR inline asm to MIR INLINEASM
11///
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
15#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineOperand.h"
18#include "llvm/CodeGen/MachineRegisterInfo.h"
19#include "llvm/CodeGen/TargetLowering.h"
20#include "llvm/IR/DiagnosticInfo.h"
21#include "llvm/IR/Module.h"
22
23#define DEBUG_TYPE "inline-asm-lowering"
24
25using namespace llvm;
26
27void InlineAsmLowering::anchor() {}
28
29/// Emit an inline asm error diagnostic and materialize undef values for the
30/// call results so that the rest of the function remains well-formed.
31static void emitInlineAsmError(MachineIRBuilder &MIRBuilder,
32 const CallBase &Call, const Twine &Message,
33 ArrayRef<Register> ResRegs) {
34 Call.getContext().diagnose(DI: DiagnosticInfoInlineAsm(Call, Message));
35 for (Register Reg : ResRegs)
36 MIRBuilder.buildUndef(Res: Reg);
37}
38
39namespace {
40
41/// GISelAsmOperandInfo - This contains information for each constraint that we
42/// are lowering.
43class GISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
44public:
45 /// Regs - If this is a register or register class operand, this
46 /// contains the set of assigned registers corresponding to the operand.
47 SmallVector<Register, 1> Regs;
48
49 /// The register class selected for this operand's constraint.
50 const TargetRegisterClass *RegClass = nullptr;
51
52 explicit GISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &Info)
53 : TargetLowering::AsmOperandInfo(Info) {}
54};
55
56using GISelAsmOperandInfoVector = SmallVector<GISelAsmOperandInfo, 16>;
57
58class ExtraFlags {
59 unsigned Flags = 0;
60
61public:
62 explicit ExtraFlags(const CallBase &CB) {
63 const InlineAsm *IA = cast<InlineAsm>(Val: CB.getCalledOperand());
64 if (IA->hasSideEffects())
65 Flags |= InlineAsm::Extra_HasSideEffects;
66 if (IA->isAlignStack())
67 Flags |= InlineAsm::Extra_IsAlignStack;
68 if (IA->canThrow())
69 Flags |= InlineAsm::Extra_MayUnwind;
70 if (CB.isConvergent())
71 Flags |= InlineAsm::Extra_IsConvergent;
72 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
73 }
74
75 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
76 // Ideally, we would only check against memory constraints. However, the
77 // meaning of an Other constraint can be target-specific and we can't easily
78 // reason about it. Therefore, be conservative and set MayLoad/MayStore
79 // for Other constraints as well.
80 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
81 OpInfo.ConstraintType == TargetLowering::C_Other) {
82 if (OpInfo.Type == InlineAsm::isInput)
83 Flags |= InlineAsm::Extra_MayLoad;
84 else if (OpInfo.Type == InlineAsm::isOutput)
85 Flags |= InlineAsm::Extra_MayStore;
86 else if (OpInfo.Type == InlineAsm::isClobber)
87 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
88 }
89 }
90
91 unsigned get() const { return Flags; }
92};
93
94} // namespace
95
96/// Assign virtual/physical registers for the specified register operand.
97static void getRegistersForValue(MachineFunction &MF,
98 MachineIRBuilder &MIRBuilder,
99 GISelAsmOperandInfo &OpInfo,
100 GISelAsmOperandInfo &RefOpInfo) {
101
102 const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
103 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
104
105 // No work to do for memory operations.
106 if (OpInfo.ConstraintType == TargetLowering::C_Memory)
107 return;
108
109 // If this is a constraint for a single physreg, or a constraint for a
110 // register class, find it.
111 Register AssignedReg;
112 const TargetRegisterClass *RC;
113 std::tie(args&: AssignedReg, args&: RC) = TLI.getRegForInlineAsmConstraint(
114 TRI: &TRI, Constraint: RefOpInfo.ConstraintCode, VT: RefOpInfo.ConstraintVT);
115 // RC is unset only on failure. Return immediately.
116 if (!RC)
117 return;
118 OpInfo.RegClass = RC;
119
120 // No need to allocate a matching input constraint since the constraint it's
121 // matching to has already been allocated.
122 if (OpInfo.isMatchingInputConstraint())
123 return;
124
125 // Initialize NumRegs.
126 unsigned NumRegs = 1;
127 if (OpInfo.ConstraintVT != MVT::Other)
128 NumRegs =
129 TLI.getNumRegisters(Context&: MF.getFunction().getContext(), VT: OpInfo.ConstraintVT);
130
131 // If this is a constraint for a specific physical register, but the type of
132 // the operand requires more than one register to be passed, we allocate the
133 // required amount of physical registers, starting from the selected physical
134 // register.
135 // For this, first retrieve a register iterator for the given register class
136 TargetRegisterClass::iterator I = RC->begin();
137 MachineRegisterInfo &RegInfo = MF.getRegInfo();
138
139 // Advance the iterator to the assigned register (if set)
140 if (AssignedReg) {
141 for (; *I != AssignedReg; ++I)
142 assert(I != RC->end() && "AssignedReg should be a member of provided RC");
143 }
144
145 // Finally, assign the registers. If the AssignedReg isn't set, create virtual
146 // registers with the provided register class
147 for (; NumRegs; --NumRegs, ++I) {
148 assert(I != RC->end() && "Ran out of registers to allocate!");
149 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RegClass: RC);
150 OpInfo.Regs.push_back(Elt: R);
151 }
152}
153
154static void computeConstraintToUse(const TargetLowering *TLI,
155 TargetLowering::AsmOperandInfo &OpInfo) {
156 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
157
158 // Single-letter constraints ('r') are very common.
159 if (OpInfo.Codes.size() == 1) {
160 OpInfo.ConstraintCode = OpInfo.Codes[0];
161 OpInfo.ConstraintType = TLI->getConstraintType(Constraint: OpInfo.ConstraintCode);
162 } else {
163 TargetLowering::ConstraintGroup G = TLI->getConstraintPreferences(OpInfo);
164 if (G.empty())
165 return;
166 // FIXME: prefer immediate constraints if the target allows it
167 unsigned BestIdx = 0;
168 for (const unsigned E = G.size();
169 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
170 G[BestIdx].second == TargetLowering::C_Immediate);
171 ++BestIdx)
172 ;
173 OpInfo.ConstraintCode = G[BestIdx].first;
174 OpInfo.ConstraintType = G[BestIdx].second;
175 }
176
177 // 'X' matches anything.
178 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
179 // Labels and constants are handled elsewhere ('X' is the only thing
180 // that matches labels). For Functions, the type here is the type of
181 // the result, which is not what we want to look at; leave them alone.
182 Value *Val = OpInfo.CallOperandVal;
183 if (isa<BasicBlock>(Val) || isa<ConstantInt>(Val) || isa<Function>(Val))
184 return;
185
186 // Otherwise, try to resolve it to something we know about by looking at
187 // the actual operand type.
188 if (const char *Repl = TLI->LowerXConstraint(ConstraintVT: OpInfo.ConstraintVT)) {
189 OpInfo.ConstraintCode = Repl;
190 OpInfo.ConstraintType = TLI->getConstraintType(Constraint: OpInfo.ConstraintCode);
191 }
192 }
193}
194
195static unsigned getNumOpRegs(const MachineInstr &I, unsigned OpIdx) {
196 const InlineAsm::Flag F(I.getOperand(i: OpIdx).getImm());
197 return F.getNumOperandRegisters();
198}
199
200static bool buildAnyextOrCopy(Register Dst, Register Src,
201 MachineIRBuilder &MIRBuilder) {
202 const TargetRegisterInfo *TRI =
203 MIRBuilder.getMF().getSubtarget().getRegisterInfo();
204 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
205
206 auto SrcTy = MRI->getType(Reg: Src);
207 if (!SrcTy.isValid()) {
208 LLVM_DEBUG(dbgs() << "Source type for copy is not valid\n");
209 return false;
210 }
211 unsigned SrcSize = TRI->getRegSizeInBits(Reg: Src, MRI: *MRI);
212 unsigned DstSize = TRI->getRegSizeInBits(Reg: Dst, MRI: *MRI);
213
214 if (DstSize < SrcSize) {
215 LLVM_DEBUG(dbgs() << "Input can't fit in destination reg class\n");
216 return false;
217 }
218
219 // Attempt to anyext small scalar sources.
220 if (DstSize > SrcSize) {
221 if (!SrcTy.isScalar()) {
222 LLVM_DEBUG(dbgs() << "Can't extend non-scalar input to size of"
223 "destination register class\n");
224 return false;
225 }
226 Src = MIRBuilder.buildAnyExt(Res: LLT::integer(SizeInBits: DstSize), Op: Src).getReg(Idx: 0);
227 }
228
229 MIRBuilder.buildCopy(Res: Dst, Op: Src);
230 return true;
231}
232
233bool InlineAsmLowering::lowerInlineAsm(
234 MachineIRBuilder &MIRBuilder, const CallBase &Call,
235 std::function<ArrayRef<Register>(const Value &Val)> GetOrCreateVRegs)
236 const {
237 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
238
239 /// ConstraintOperands - Information about all of the constraints.
240 GISelAsmOperandInfoVector ConstraintOperands;
241
242 MachineFunction &MF = MIRBuilder.getMF();
243 const Function &F = MF.getFunction();
244 const DataLayout &DL = F.getDataLayout();
245 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
246
247 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
248
249 TargetLowering::AsmOperandInfoVector TargetConstraints =
250 TLI->ParseConstraints(DL, TRI, Call);
251
252 ExtraFlags ExtraInfo(Call);
253 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
254 unsigned ResNo = 0; // ResNo - The result number of the next output.
255 for (auto &T : TargetConstraints) {
256 ConstraintOperands.push_back(Elt: GISelAsmOperandInfo(T));
257 GISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
258
259 // Compute the value type for each operand.
260 if (OpInfo.hasArg()) {
261 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
262
263 if (isa<BasicBlock>(Val: OpInfo.CallOperandVal)) {
264 LLVM_DEBUG(dbgs() << "Basic block input operands not supported yet\n");
265 return false;
266 }
267
268 Type *OpTy = OpInfo.CallOperandVal->getType();
269
270 // If this is an indirect operand, the operand is a pointer to the
271 // accessed type.
272 if (OpInfo.isIndirect) {
273 OpTy = Call.getParamElementType(ArgNo);
274 assert(OpTy && "Indirect operand must have elementtype attribute");
275 }
276
277 // FIXME: Support aggregate input operands
278 if (!OpTy->isSingleValueType()) {
279 LLVM_DEBUG(
280 dbgs() << "Aggregate input operands are not supported yet\n");
281 return false;
282 }
283
284 OpInfo.ConstraintVT =
285 TLI->getAsmOperandValueType(DL, Ty: OpTy, AllowUnknown: true).getSimpleVT();
286 ++ArgNo;
287 } else if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
288 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
289 if (StructType *STy = dyn_cast<StructType>(Val: Call.getType())) {
290 OpInfo.ConstraintVT =
291 TLI->getSimpleValueType(DL, Ty: STy->getElementType(N: ResNo));
292 } else {
293 assert(ResNo == 0 && "Asm only has one result!");
294 OpInfo.ConstraintVT =
295 TLI->getAsmOperandValueType(DL, Ty: Call.getType()).getSimpleVT();
296 }
297 ++ResNo;
298 } else {
299 assert(OpInfo.Type != InlineAsm::isLabel &&
300 "GlobalISel currently doesn't support callbr");
301 OpInfo.ConstraintVT = MVT::Other;
302 }
303
304 if (OpInfo.ConstraintVT == MVT::i64x8)
305 return false;
306
307 // Compute the constraint code and ConstraintType to use.
308 computeConstraintToUse(TLI, OpInfo);
309
310 // The selected constraint type might expose new sideeffects
311 ExtraInfo.update(OpInfo);
312 }
313
314 // At this point, all operand types are decided.
315 // Create the MachineInstr, but don't insert it yet since input
316 // operands still need to insert instructions before this one
317 auto Inst = MIRBuilder.buildInstrNoInsert(Opcode: TargetOpcode::INLINEASM)
318 .addExternalSymbol(FnName: IA->getAsmString().data())
319 .addImm(Val: ExtraInfo.get());
320
321 // Starting from this operand: flag followed by register(s) will be added as
322 // operands to Inst for each constraint. Used for matching input constraints.
323 unsigned StartIdx = Inst->getNumOperands();
324
325 // Collects the output operands for later processing
326 GISelAsmOperandInfoVector OutputOperands;
327
328 for (auto &OpInfo : ConstraintOperands) {
329 GISelAsmOperandInfo &RefOpInfo =
330 OpInfo.isMatchingInputConstraint()
331 ? ConstraintOperands[OpInfo.getMatchedOperand()]
332 : OpInfo;
333
334 // Assign registers for register operands
335 getRegistersForValue(MF, MIRBuilder, OpInfo, RefOpInfo);
336
337 switch (OpInfo.Type) {
338 case InlineAsm::isOutput:
339 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
340 const InlineAsm::ConstraintCode ConstraintID =
341 TLI->getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
342 assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
343 "Failed to convert memory constraint code to constraint id.");
344
345 // Add information to the INLINEASM instruction to know about this
346 // output.
347 InlineAsm::Flag Flag(InlineAsm::Kind::Mem, 1);
348 Flag.setMemConstraint(ConstraintID);
349 Inst.addImm(Val: Flag);
350 ArrayRef<Register> SourceRegs =
351 GetOrCreateVRegs(*OpInfo.CallOperandVal);
352 assert(
353 SourceRegs.size() == 1 &&
354 "Expected the memory output to fit into a single virtual register");
355 Inst.addReg(RegNo: SourceRegs[0]);
356 } else {
357 // Otherwise, this outputs to a register (directly for C_Register /
358 // C_RegisterClass/C_Other.
359 assert(OpInfo.ConstraintType == TargetLowering::C_Register ||
360 OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
361 OpInfo.ConstraintType == TargetLowering::C_Other);
362
363 // Find a register that we can use.
364 if (OpInfo.Regs.empty()) {
365 emitInlineAsmError(MIRBuilder, Call,
366 Message: "could not allocate output register for "
367 "constraint '" +
368 Twine(OpInfo.ConstraintCode) + "'",
369 ResRegs: GetOrCreateVRegs(Call));
370 return true;
371 }
372
373 // Add information to the INLINEASM instruction to know that this
374 // register is set.
375 InlineAsm::Flag Flag(OpInfo.isEarlyClobber
376 ? InlineAsm::Kind::RegDefEarlyClobber
377 : InlineAsm::Kind::RegDef,
378 OpInfo.Regs.size());
379 if (OpInfo.Regs.front().isVirtual()) {
380 // Put the register class of the virtual registers in the flag word.
381 // That way, later passes can recompute register class constraints for
382 // inline assembly as well as normal instructions. Don't do this for
383 // tied operands that can use the regclass information from the def.
384 const TargetRegisterClass *RC = MRI->getRegClass(Reg: OpInfo.Regs.front());
385 Flag.setRegClass(RC->getID());
386 }
387
388 Inst.addImm(Val: Flag);
389
390 for (Register Reg : OpInfo.Regs) {
391 Inst.addReg(RegNo: Reg, Flags: RegState::Define |
392 getImplRegState(B: Reg.isPhysical()) |
393 getEarlyClobberRegState(B: OpInfo.isEarlyClobber));
394 }
395
396 // Remember this output operand for later processing
397 OutputOperands.push_back(Elt: OpInfo);
398 }
399
400 break;
401 case InlineAsm::isInput:
402 case InlineAsm::isLabel: {
403 if (OpInfo.isMatchingInputConstraint()) {
404 unsigned DefIdx = OpInfo.getMatchedOperand();
405 // Find operand with register def that corresponds to DefIdx.
406 unsigned InstFlagIdx = StartIdx;
407 for (unsigned i = 0; i < DefIdx; ++i)
408 InstFlagIdx += getNumOpRegs(I: *Inst, OpIdx: InstFlagIdx) + 1;
409 assert(getNumOpRegs(*Inst, InstFlagIdx) == 1 && "Wrong flag");
410
411 const InlineAsm::Flag MatchedOperandFlag(Inst->getOperand(i: InstFlagIdx).getImm());
412 if (MatchedOperandFlag.isMemKind()) {
413 LLVM_DEBUG(dbgs() << "Matching input constraint to mem operand not "
414 "supported. This should be target specific.\n");
415 return false;
416 }
417 if (!MatchedOperandFlag.isRegDefKind() && !MatchedOperandFlag.isRegDefEarlyClobberKind()) {
418 LLVM_DEBUG(dbgs() << "Unknown matching constraint\n");
419 return false;
420 }
421
422 // We want to tie input to register in next operand.
423 unsigned DefRegIdx = InstFlagIdx + 1;
424 Register Def = Inst->getOperand(i: DefRegIdx).getReg();
425
426 ArrayRef<Register> SrcRegs = GetOrCreateVRegs(*OpInfo.CallOperandVal);
427 assert(SrcRegs.size() == 1 && "Single register is expected here");
428
429 // We need the tied input to live in the same register class as the def.
430 //
431 // - if Def is a vreg, we can just use its regclass.
432 // - if Def is a physreg, create a vreg in the regclass selected for its
433 // constraint.
434 //
435 // Otherwise RegBankSelect may leave it in the wrong bank (e.g. GPR even
436 // though it's tied to an FP physreg).
437 const TargetRegisterClass *RC =
438 Def.isVirtual() ? MRI->getRegClass(Reg: Def) : OpInfo.RegClass;
439 assert(RC && "Expected a register class for matching constraint");
440
441 // Materialize `In` in a new vreg that has a register class that matches
442 // the register class of `Def`.
443 Register In = MRI->createVirtualRegister(RegClass: RC);
444 if (!buildAnyextOrCopy(Dst: In, Src: SrcRegs[0], MIRBuilder))
445 return false;
446
447 // Add Flag and input register operand (In) to Inst. Tie In to Def.
448 InlineAsm::Flag UseFlag(InlineAsm::Kind::RegUse, 1);
449 UseFlag.setMatchingOp(DefIdx);
450 Inst.addImm(Val: UseFlag);
451 Inst.addReg(RegNo: In);
452 Inst->tieOperands(DefIdx: DefRegIdx, UseIdx: Inst->getNumOperands() - 1);
453 break;
454 }
455
456 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
457 OpInfo.isIndirect) {
458 LLVM_DEBUG(dbgs() << "Indirect input operands with unknown constraint "
459 "not supported yet\n");
460 return false;
461 }
462
463 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
464 OpInfo.ConstraintType == TargetLowering::C_Other) {
465
466 std::vector<MachineOperand> Ops;
467 if (!lowerAsmOperandForConstraint(Val: OpInfo.CallOperandVal,
468 Constraint: OpInfo.ConstraintCode, Ops,
469 MIRBuilder)) {
470 LLVM_DEBUG(dbgs() << "Don't support constraint: "
471 << OpInfo.ConstraintCode << " yet\n");
472 return false;
473 }
474
475 assert(Ops.size() > 0 &&
476 "Expected constraint to be lowered to at least one operand");
477
478 // Add information to the INLINEASM node to know about this input.
479 const unsigned OpFlags =
480 InlineAsm::Flag(InlineAsm::Kind::Imm, Ops.size());
481 Inst.addImm(Val: OpFlags);
482 Inst.add(MOs: Ops);
483 break;
484 }
485
486 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
487 const InlineAsm::ConstraintCode ConstraintID =
488 TLI->getInlineAsmMemConstraint(ConstraintCode: OpInfo.ConstraintCode);
489 InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
490 OpFlags.setMemConstraint(ConstraintID);
491 Inst.addImm(Val: OpFlags);
492
493 if (OpInfo.isIndirect) {
494 // already indirect
495 ArrayRef<Register> SourceRegs =
496 GetOrCreateVRegs(*OpInfo.CallOperandVal);
497 if (SourceRegs.size() != 1) {
498 LLVM_DEBUG(dbgs() << "Expected the memory input to fit into a "
499 "single virtual register "
500 "for constraint '"
501 << OpInfo.ConstraintCode << "'\n");
502 return false;
503 }
504 Inst.addReg(RegNo: SourceRegs[0]);
505 break;
506 }
507
508 // Needs to be made indirect. Store the value on the stack and use
509 // a pointer to it.
510 Value *OpVal = OpInfo.CallOperandVal;
511 TypeSize Bytes = DL.getTypeStoreSize(Ty: OpVal->getType());
512 Align Alignment = DL.getPrefTypeAlign(Ty: OpVal->getType());
513 int FrameIdx =
514 MF.getFrameInfo().CreateStackObject(Size: Bytes, Alignment, isSpillSlot: false);
515
516 unsigned AddrSpace = DL.getAllocaAddrSpace();
517 LLT FramePtrTy =
518 LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace));
519 auto Ptr = MIRBuilder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx).getReg(Idx: 0);
520 ArrayRef<Register> SourceRegs =
521 GetOrCreateVRegs(*OpInfo.CallOperandVal);
522 if (SourceRegs.size() != 1) {
523 LLVM_DEBUG(dbgs() << "Expected the memory input to fit into a single "
524 "virtual register "
525 "for constraint '"
526 << OpInfo.ConstraintCode << "'\n");
527 return false;
528 }
529 MIRBuilder.buildStore(Val: SourceRegs[0], Addr: Ptr,
530 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIdx),
531 Alignment);
532 Inst.addReg(RegNo: Ptr);
533 break;
534 }
535
536 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
537 OpInfo.ConstraintType == TargetLowering::C_Register) &&
538 "Unknown constraint type!");
539
540 if (OpInfo.isIndirect) {
541 LLVM_DEBUG(dbgs() << "Can't handle indirect register inputs yet "
542 "for constraint '"
543 << OpInfo.ConstraintCode << "'\n");
544 return false;
545 }
546
547 // Copy the input into the appropriate registers.
548 if (OpInfo.Regs.empty()) {
549 emitInlineAsmError(MIRBuilder, Call,
550 Message: "could not allocate input reg for constraint '" +
551 Twine(OpInfo.ConstraintCode) + "'",
552 ResRegs: GetOrCreateVRegs(Call));
553 return true;
554 }
555
556 unsigned NumRegs = OpInfo.Regs.size();
557 ArrayRef<Register> SourceRegs = GetOrCreateVRegs(*OpInfo.CallOperandVal);
558 if (NumRegs != 1 || SourceRegs.size() != 1) {
559 LLVM_DEBUG(dbgs() << "Input operands with multiple input registers are "
560 "not supported yet\n");
561 return false;
562 }
563
564 InlineAsm::Flag Flag(InlineAsm::Kind::RegUse, NumRegs);
565 if (OpInfo.Regs.front().isVirtual()) {
566 // Put the register class of the virtual registers in the flag word.
567 const TargetRegisterClass *RC = MRI->getRegClass(Reg: OpInfo.Regs.front());
568 Flag.setRegClass(RC->getID());
569 }
570 Inst.addImm(Val: Flag);
571 if (!buildAnyextOrCopy(Dst: OpInfo.Regs[0], Src: SourceRegs[0], MIRBuilder))
572 return false;
573 Inst.addReg(RegNo: OpInfo.Regs[0]);
574 break;
575 }
576
577 case InlineAsm::isClobber: {
578
579 const unsigned NumRegs = OpInfo.Regs.size();
580 if (NumRegs > 0) {
581 unsigned Flag = InlineAsm::Flag(InlineAsm::Kind::Clobber, NumRegs);
582 Inst.addImm(Val: Flag);
583
584 for (Register Reg : OpInfo.Regs) {
585 Inst.addReg(RegNo: Reg, Flags: RegState::Define | RegState::EarlyClobber |
586 getImplRegState(B: Reg.isPhysical()));
587 }
588 }
589 break;
590 }
591 }
592 }
593
594 if (auto Bundle = Call.getOperandBundle(ID: LLVMContext::OB_convergencectrl)) {
595 auto *Token = Bundle->Inputs[0].get();
596 ArrayRef<Register> SourceRegs = GetOrCreateVRegs(*Token);
597 assert(SourceRegs.size() == 1 &&
598 "Expected the control token to fit into a single virtual register");
599 Inst.addUse(RegNo: SourceRegs[0], Flags: RegState::Implicit);
600 }
601
602 if (const MDNode *SrcLoc = Call.getMetadata(Kind: "srcloc"))
603 Inst.addMetadata(MD: SrcLoc);
604
605 // Add rounding control registers as implicit def for inline asm.
606 if (MF.getFunction().hasFnAttribute(Kind: Attribute::StrictFP)) {
607 ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
608 for (MCPhysReg Reg : RCRegs)
609 Inst.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
610 }
611
612 // All inputs are handled, insert the instruction now
613 MIRBuilder.insertInstr(MIB: Inst);
614
615 // Finally, copy the output operands into the output registers
616 ArrayRef<Register> ResRegs = GetOrCreateVRegs(Call);
617 if (ResRegs.size() != OutputOperands.size()) {
618 LLVM_DEBUG(dbgs() << "Expected the number of output registers to match the "
619 "number of destination registers\n");
620 return false;
621 }
622 for (unsigned int i = 0, e = ResRegs.size(); i < e; i++) {
623 GISelAsmOperandInfo &OpInfo = OutputOperands[i];
624
625 if (OpInfo.Regs.empty())
626 continue;
627
628 switch (OpInfo.ConstraintType) {
629 case TargetLowering::C_Register:
630 case TargetLowering::C_RegisterClass: {
631 if (OpInfo.Regs.size() > 1) {
632 LLVM_DEBUG(dbgs() << "Output operands with multiple defining "
633 "registers are not supported yet\n");
634 return false;
635 }
636
637 Register SrcReg = OpInfo.Regs[0];
638 unsigned SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
639 LLT ResTy = MRI->getType(Reg: ResRegs[i]);
640 if (ResTy.isScalar() && ResTy.getSizeInBits() < SrcSize) {
641 // First copy the non-typed virtual register into a generic virtual
642 // register
643 auto Copy = MIRBuilder.buildCopy(Res: LLT::integer(SizeInBits: SrcSize), Op: SrcReg);
644 // Need to truncate the result of the register
645 MIRBuilder.buildTrunc(Res: ResRegs[i], Op: Copy);
646 } else if (ResTy.getSizeInBits() == SrcSize) {
647 MIRBuilder.buildCopy(Res: ResRegs[i], Op: SrcReg);
648 } else {
649 LLVM_DEBUG(dbgs() << "Unhandled output operand with "
650 "mismatched register size\n");
651 return false;
652 }
653
654 break;
655 }
656 case TargetLowering::C_Immediate:
657 case TargetLowering::C_Other:
658 LLVM_DEBUG(
659 dbgs() << "Cannot lower target specific output constraints yet\n");
660 return false;
661 case TargetLowering::C_Memory:
662 break; // Already handled.
663 case TargetLowering::C_Address:
664 break; // Silence warning.
665 case TargetLowering::C_Unknown:
666 LLVM_DEBUG(dbgs() << "Unexpected unknown constraint\n");
667 return false;
668 }
669 }
670
671 return true;
672}
673
674bool InlineAsmLowering::lowerAsmOperandForConstraint(
675 Value *Val, StringRef Constraint, std::vector<MachineOperand> &Ops,
676 MachineIRBuilder &MIRBuilder) const {
677 if (Constraint.size() > 1)
678 return false;
679
680 char ConstraintLetter = Constraint[0];
681 switch (ConstraintLetter) {
682 default:
683 return false;
684 case 's': // Integer immediate not known at compile time
685 if (const auto *GV = dyn_cast<GlobalValue>(Val)) {
686 Ops.push_back(x: MachineOperand::CreateGA(GV, /*Offset=*/0));
687 return true;
688 }
689 return false;
690 case 'i': // Simple Integer or Relocatable Constant
691 if (const auto *GV = dyn_cast<GlobalValue>(Val)) {
692 Ops.push_back(x: MachineOperand::CreateGA(GV, /*Offset=*/0));
693 return true;
694 }
695 [[fallthrough]];
696 case 'n': // immediate integer with a known value.
697 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
698 assert(CI->getBitWidth() <= 64 &&
699 "expected immediate to fit into 64-bits");
700 // Boolean constants should be zero-extended, others are sign-extended
701 bool IsBool = CI->getBitWidth() == 1;
702 int64_t ExtVal = IsBool ? CI->getZExtValue() : CI->getSExtValue();
703 Ops.push_back(x: MachineOperand::CreateImm(Val: ExtVal));
704 return true;
705 }
706 return false;
707 }
708}
709