1//===- X86InstructionSelector.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// X86.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86BaseInfo.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86RegisterBankInfo.h"
19#include "X86RegisterInfo.h"
20#include "X86Subtarget.h"
21#include "X86TargetMachine.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
25#include "llvm/CodeGen/GlobalISel/Utils.h"
26#include "llvm/CodeGen/MachineBasicBlock.h"
27#include "llvm/CodeGen/MachineConstantPool.h"
28#include "llvm/CodeGen/MachineFunction.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineInstrBuilder.h"
31#include "llvm/CodeGen/MachineMemOperand.h"
32#include "llvm/CodeGen/MachineOperand.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/RegisterBank.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/CodeGen/TargetRegisterInfo.h"
37#include "llvm/CodeGenTypes/LowLevelType.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/InstrTypes.h"
40#include "llvm/IR/IntrinsicsX86.h"
41#include "llvm/Support/CodeGen.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include <cassert>
47#include <cstdint>
48#include <tuple>
49
50#define DEBUG_TYPE "X86-isel"
51
52using namespace llvm;
53
54namespace {
55
56#define GET_GLOBALISEL_PREDICATE_BITSET
57#include "X86GenGlobalISel.inc"
58#undef GET_GLOBALISEL_PREDICATE_BITSET
59
60class X86InstructionSelector : public InstructionSelector {
61public:
62 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
63 const X86RegisterBankInfo &RBI);
64
65 bool select(MachineInstr &I) override;
66 static const char *getName() { return DEBUG_TYPE; }
67
68private:
69 /// tblgen-erated 'select' implementation, used as the initial selector for
70 /// the patterns that don't require complex C++.
71 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
72
73 // TODO: remove after supported by Tablegen-erated instruction selection.
74 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
75 Align Alignment) const;
76 // TODO: remove once p0<->i32/i64 matching is available
77 unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
78 unsigned Opc) const;
79
80 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
81 MachineFunction &MF) const;
82 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
83 MachineFunction &MF) const;
84 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
85 MachineFunction &MF) const;
86 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
87 MachineFunction &MF) const;
88 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
89 MachineFunction &MF) const;
90 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
91 MachineFunction &MF) const;
92 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
93 MachineFunction &MF) const;
94 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
95 MachineFunction &MF) const;
96 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
97 MachineFunction &MF) const;
98 bool selectUAddSub(MachineInstr &I, MachineRegisterInfo &MRI,
99 MachineFunction &MF) const;
100 bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI) const;
101 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
102 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
103 MachineFunction &MF);
104 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
105 MachineFunction &MF);
106 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
107 MachineFunction &MF) const;
108 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
109 MachineFunction &MF) const;
110 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
111 MachineFunction &MF) const;
112 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
113 const Register DstReg,
114 const TargetRegisterClass *DstRC,
115 const Register SrcReg,
116 const TargetRegisterClass *SrcRC) const;
117 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
118 MachineFunction &MF) const;
119 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
120 bool selectMulDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
121 MachineFunction &MF) const;
122 bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
123 MachineFunction &MF) const;
124
125 ComplexRendererFns selectAddr(MachineOperand &Root) const;
126
127 // emit insert subreg instruction and insert it before MachineInstr &I
128 bool emitInsertSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
129 MachineRegisterInfo &MRI, MachineFunction &MF) const;
130 // emit extract subreg instruction and insert it before MachineInstr &I
131 bool emitExtractSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
132 MachineRegisterInfo &MRI, MachineFunction &MF) const;
133
134 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
135 const TargetRegisterClass *getRegClass(LLT Ty, Register Reg,
136 MachineRegisterInfo &MRI) const;
137
138 const X86TargetMachine &TM;
139 const X86Subtarget &STI;
140 const X86InstrInfo &TII;
141 const X86RegisterInfo &TRI;
142 const X86RegisterBankInfo &RBI;
143
144#define GET_GLOBALISEL_PREDICATES_DECL
145#include "X86GenGlobalISel.inc"
146#undef GET_GLOBALISEL_PREDICATES_DECL
147
148#define GET_GLOBALISEL_TEMPORARIES_DECL
149#include "X86GenGlobalISel.inc"
150#undef GET_GLOBALISEL_TEMPORARIES_DECL
151};
152
153} // end anonymous namespace
154
155#define GET_GLOBALISEL_IMPL
156#include "X86GenGlobalISel.inc"
157#undef GET_GLOBALISEL_IMPL
158
159X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
160 const X86Subtarget &STI,
161 const X86RegisterBankInfo &RBI)
162 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
163 RBI(RBI),
164#define GET_GLOBALISEL_PREDICATES_INIT
165#include "X86GenGlobalISel.inc"
166#undef GET_GLOBALISEL_PREDICATES_INIT
167#define GET_GLOBALISEL_TEMPORARIES_INIT
168#include "X86GenGlobalISel.inc"
169#undef GET_GLOBALISEL_TEMPORARIES_INIT
170{
171}
172
173// FIXME: This should be target-independent, inferred from the types declared
174// for each class in the bank.
175const TargetRegisterClass *
176X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
177 if (RB.getID() == X86::GPRRegBankID) {
178 if (Ty.getSizeInBits() <= 8)
179 return &X86::GR8RegClass;
180 if (Ty.getSizeInBits() == 16)
181 return &X86::GR16RegClass;
182 if (Ty.getSizeInBits() == 32)
183 return &X86::GR32RegClass;
184 if (Ty.getSizeInBits() == 64)
185 return &X86::GR64RegClass;
186 }
187 if (RB.getID() == X86::VECRRegBankID) {
188 if (Ty.getSizeInBits() == 16)
189 return STI.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
190 if (Ty.getSizeInBits() == 32)
191 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
192 if (Ty.getSizeInBits() == 64)
193 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
194 if (Ty.getSizeInBits() == 128)
195 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
196 if (Ty.getSizeInBits() == 256)
197 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
198 if (Ty.getSizeInBits() == 512)
199 return &X86::VR512RegClass;
200 }
201
202 if (RB.getID() == X86::PSRRegBankID) {
203 if (Ty.getSizeInBits() == 80)
204 return &X86::RFP80RegClass;
205 if (Ty.getSizeInBits() == 64)
206 return &X86::RFP64RegClass;
207 if (Ty.getSizeInBits() == 32)
208 return &X86::RFP32RegClass;
209 }
210
211 llvm_unreachable("Unknown RegBank!");
212}
213
214const TargetRegisterClass *
215X86InstructionSelector::getRegClass(LLT Ty, Register Reg,
216 MachineRegisterInfo &MRI) const {
217 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
218 return getRegClass(Ty, RB: RegBank);
219}
220
221static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
222 unsigned SubIdx = X86::NoSubRegister;
223 if (RC == &X86::GR32RegClass) {
224 SubIdx = X86::sub_32bit;
225 } else if (RC == &X86::GR16RegClass) {
226 SubIdx = X86::sub_16bit;
227 } else if (RC == &X86::GR8RegClass) {
228 SubIdx = X86::sub_8bit;
229 }
230
231 return SubIdx;
232}
233
234static const TargetRegisterClass *getRegClassFromGRPhysReg(Register Reg) {
235 assert(Reg.isPhysical());
236 if (X86::GR64RegClass.contains(Reg))
237 return &X86::GR64RegClass;
238 if (X86::GR32RegClass.contains(Reg))
239 return &X86::GR32RegClass;
240 if (X86::GR16RegClass.contains(Reg))
241 return &X86::GR16RegClass;
242 if (X86::GR8RegClass.contains(Reg))
243 return &X86::GR8RegClass;
244
245 llvm_unreachable("Unknown RegClass for PhysReg!");
246}
247
248// FIXME: We need some sort of API in RBI/TRI to allow generic code to
249// constrain operands of simple instructions given a TargetRegisterClass
250// and LLT
251bool X86InstructionSelector::selectDebugInstr(MachineInstr &I,
252 MachineRegisterInfo &MRI) const {
253 for (MachineOperand &MO : I.operands()) {
254 if (!MO.isReg())
255 continue;
256 Register Reg = MO.getReg();
257 if (!Reg)
258 continue;
259 if (Reg.isPhysical())
260 continue;
261 LLT Ty = MRI.getType(Reg);
262 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
263 const TargetRegisterClass *RC =
264 dyn_cast_if_present<const TargetRegisterClass *>(Val: RegClassOrBank);
265 if (!RC) {
266 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
267 RC = getRegClass(Ty, RB);
268 if (!RC) {
269 LLVM_DEBUG(
270 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
271 break;
272 }
273 }
274 RBI.constrainGenericRegister(Reg, RC: *RC, MRI);
275 }
276
277 return true;
278}
279
280// Set X86 Opcode and constrain DestReg.
281bool X86InstructionSelector::selectCopy(MachineInstr &I,
282 MachineRegisterInfo &MRI) const {
283 Register DstReg = I.getOperand(i: 0).getReg();
284 const unsigned DstSize = RBI.getSizeInBits(Reg: DstReg, MRI, TRI);
285 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
286
287 Register SrcReg = I.getOperand(i: 1).getReg();
288 const unsigned SrcSize = RBI.getSizeInBits(Reg: SrcReg, MRI, TRI);
289 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
290
291 if (DstReg.isPhysical()) {
292 assert(I.isCopy() && "Generic operators do not allow physical registers");
293
294 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
295 DstRegBank.getID() == X86::GPRRegBankID) {
296
297 const TargetRegisterClass *SrcRC =
298 getRegClass(Ty: MRI.getType(Reg: SrcReg), RB: SrcRegBank);
299 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(Reg: DstReg);
300
301 if (SrcRC != DstRC) {
302 // This case can be generated by ABI lowering, performe anyext
303 Register ExtSrc = MRI.createVirtualRegister(RegClass: DstRC);
304 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
305 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
306 .addDef(RegNo: ExtSrc)
307 .addImm(Val: 0)
308 .addReg(RegNo: SrcReg)
309 .addImm(Val: getSubRegIndex(RC: SrcRC));
310
311 I.getOperand(i: 1).setReg(ExtSrc);
312 }
313 }
314
315 // Special case GPR16 -> XMM
316 if (SrcSize == 16 && SrcRegBank.getID() == X86::GPRRegBankID &&
317 (DstRegBank.getID() == X86::VECRRegBankID)) {
318
319 const DebugLoc &DL = I.getDebugLoc();
320
321 // Any extend GPR16 -> GPR32
322 Register ExtReg = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
323 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG),
324 DestReg: ExtReg)
325 .addImm(Val: 0)
326 .addReg(RegNo: SrcReg)
327 .addImm(Val: X86::sub_16bit);
328
329 // Copy GR32 -> XMM
330 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
331 .addReg(RegNo: ExtReg);
332
333 I.eraseFromParent();
334 }
335
336 // Special case XMM -> GR16
337 if (DstSize == 16 && DstRegBank.getID() == X86::GPRRegBankID &&
338 (SrcRegBank.getID() == X86::VECRRegBankID)) {
339
340 const DebugLoc &DL = I.getDebugLoc();
341
342 // Move XMM to GR32 register.
343 Register Temp32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
344 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Temp32)
345 .addReg(RegNo: SrcReg);
346
347 // Extract the lower 16 bits
348 if (Register Dst32 = TRI.getMatchingSuperReg(Reg: DstReg, SubIdx: X86::sub_16bit,
349 RC: &X86::GR32RegClass)) {
350 // Optimization for Physical Dst (e.g. AX): Copy to EAX directly.
351 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst32)
352 .addReg(RegNo: Temp32);
353 } else {
354 // Handle if there is no super.
355 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
356 .addReg(RegNo: Temp32, Flags: {}, SubReg: X86::sub_16bit);
357 }
358
359 I.eraseFromParent();
360 }
361
362 return true;
363 }
364
365 assert((!SrcReg.isPhysical() || I.isCopy()) &&
366 "No phys reg on generic operators");
367 assert((DstSize == SrcSize ||
368 // Copies are a mean to setup initial types, the number of
369 // bits may not exactly match.
370 (SrcReg.isPhysical() &&
371 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
372 "Copy with different width?!");
373
374 const TargetRegisterClass *DstRC =
375 getRegClass(Ty: MRI.getType(Reg: DstReg), RB: DstRegBank);
376
377 if (SrcRegBank.getID() == X86::GPRRegBankID &&
378 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
379 SrcReg.isPhysical()) {
380 // Change the physical register to performe truncate.
381
382 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(Reg: SrcReg);
383
384 if (DstRC != SrcRC) {
385 I.getOperand(i: 1).setSubReg(getSubRegIndex(RC: DstRC));
386 I.getOperand(i: 1).substPhysReg(Reg: SrcReg, TRI);
387 }
388 }
389
390 // No need to constrain SrcReg. It will get constrained when
391 // we hit another of its use or its defs.
392 // Copies do not have constraints.
393 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(Reg: DstReg);
394 if (!OldRC || !DstRC->hasSubClassEq(RC: OldRC)) {
395 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
396 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
397 << " operand\n");
398 return false;
399 }
400 }
401 I.setDesc(TII.get(Opcode: X86::COPY));
402 return true;
403}
404
405bool X86InstructionSelector::select(MachineInstr &I) {
406 assert(I.getParent() && "Instruction should be in a basic block!");
407 assert(I.getParent()->getParent() && "Instruction should be in a function!");
408
409 MachineBasicBlock &MBB = *I.getParent();
410 MachineFunction &MF = *MBB.getParent();
411 MachineRegisterInfo &MRI = MF.getRegInfo();
412
413 unsigned Opcode = I.getOpcode();
414 if (!isPreISelGenericOpcode(Opcode) && !I.isPreISelOpcode()) {
415 // Certain non-generic instructions also need some special handling.
416
417 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
418 return false;
419
420 if (I.isCopy())
421 return selectCopy(I, MRI);
422
423 if (I.isDebugInstr())
424 return selectDebugInstr(I, MRI);
425
426 return true;
427 }
428
429 assert(I.getNumOperands() == I.getNumExplicitOperands() &&
430 "Generic instruction has unexpected implicit operands\n");
431
432 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
433 return true;
434
435 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
436
437 // TODO: This should be implemented by tblgen.
438 switch (I.getOpcode()) {
439 default:
440 return false;
441 case TargetOpcode::G_STORE:
442 case TargetOpcode::G_LOAD:
443 return selectLoadStoreOp(I, MRI, MF);
444 case TargetOpcode::G_PTR_ADD:
445 case TargetOpcode::G_FRAME_INDEX:
446 return selectFrameIndexOrGep(I, MRI, MF);
447 case TargetOpcode::G_GLOBAL_VALUE:
448 return selectGlobalValue(I, MRI, MF);
449 case TargetOpcode::G_CONSTANT:
450 return selectConstant(I, MRI, MF);
451 case TargetOpcode::G_FCONSTANT:
452 return materializeFP(I, MRI, MF);
453 case TargetOpcode::G_PTRTOINT:
454 case TargetOpcode::G_TRUNC:
455 return selectTruncOrPtrToInt(I, MRI, MF);
456 case TargetOpcode::G_INTTOPTR:
457 case TargetOpcode::G_FREEZE:
458 return selectCopy(I, MRI);
459 case TargetOpcode::G_ZEXT:
460 return selectZext(I, MRI, MF);
461 case TargetOpcode::G_ANYEXT:
462 return selectAnyext(I, MRI, MF);
463 case TargetOpcode::G_ICMP:
464 return selectCmp(I, MRI, MF);
465 case TargetOpcode::G_FCMP:
466 return selectFCmp(I, MRI, MF);
467 case TargetOpcode::G_UADDE:
468 case TargetOpcode::G_UADDO:
469 case TargetOpcode::G_USUBE:
470 case TargetOpcode::G_USUBO:
471 return selectUAddSub(I, MRI, MF);
472 case TargetOpcode::G_UNMERGE_VALUES:
473 return selectUnmergeValues(I, MRI, MF);
474 case TargetOpcode::G_MERGE_VALUES:
475 case TargetOpcode::G_CONCAT_VECTORS:
476 return selectMergeValues(I, MRI, MF);
477 case TargetOpcode::G_EXTRACT:
478 return selectExtract(I, MRI, MF);
479 case TargetOpcode::G_INSERT:
480 return selectInsert(I, MRI, MF);
481 case TargetOpcode::G_BRCOND:
482 return selectCondBranch(I, MRI, MF);
483 case TargetOpcode::G_IMPLICIT_DEF:
484 case TargetOpcode::G_PHI:
485 return selectImplicitDefOrPHI(I, MRI);
486 case TargetOpcode::G_MUL:
487 case TargetOpcode::G_SMULH:
488 case TargetOpcode::G_UMULH:
489 case TargetOpcode::G_SDIV:
490 case TargetOpcode::G_UDIV:
491 case TargetOpcode::G_SREM:
492 case TargetOpcode::G_UREM:
493 return selectMulDivRem(I, MRI, MF);
494 case TargetOpcode::G_SELECT:
495 return selectSelect(I, MRI, MF);
496 }
497
498 return false;
499}
500
501unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
502 const RegisterBank &RB,
503 unsigned Opc) const {
504 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
505 "Only G_STORE and G_LOAD are expected for selection");
506 if (Ty.isPointer() && X86::GPRRegBankID == RB.getID()) {
507 bool IsLoad = (Opc == TargetOpcode::G_LOAD);
508 switch (Ty.getSizeInBits()) {
509 default:
510 break;
511 case 32:
512 return IsLoad ? X86::MOV32rm : X86::MOV32mr;
513 case 64:
514 return IsLoad ? X86::MOV64rm : X86::MOV64mr;
515 }
516 }
517 return Opc;
518}
519
520unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
521 const RegisterBank &RB,
522 unsigned Opc,
523 Align Alignment) const {
524 bool Isload = (Opc == TargetOpcode::G_LOAD);
525 bool HasAVX = STI.hasAVX();
526 bool HasAVX512 = STI.hasAVX512();
527 bool HasVLX = STI.hasVLX();
528
529 if (Ty == LLT::scalar(SizeInBits: 8)) {
530 if (X86::GPRRegBankID == RB.getID())
531 return Isload ? X86::MOV8rm : X86::MOV8mr;
532 } else if (Ty == LLT::scalar(SizeInBits: 16)) {
533 if (X86::GPRRegBankID == RB.getID())
534 return Isload ? X86::MOV16rm : X86::MOV16mr;
535 } else if (Ty == LLT::scalar(SizeInBits: 32)) {
536 if (X86::GPRRegBankID == RB.getID())
537 return Isload ? X86::MOV32rm : X86::MOV32mr;
538 if (X86::VECRRegBankID == RB.getID())
539 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
540 HasAVX ? X86::VMOVSSrm_alt :
541 X86::MOVSSrm_alt)
542 : (HasAVX512 ? X86::VMOVSSZmr :
543 HasAVX ? X86::VMOVSSmr :
544 X86::MOVSSmr);
545 if (X86::PSRRegBankID == RB.getID())
546 return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
547 } else if (Ty == LLT::scalar(SizeInBits: 64)) {
548 if (X86::GPRRegBankID == RB.getID())
549 return Isload ? X86::MOV64rm : X86::MOV64mr;
550 if (X86::VECRRegBankID == RB.getID())
551 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
552 HasAVX ? X86::VMOVSDrm_alt :
553 X86::MOVSDrm_alt)
554 : (HasAVX512 ? X86::VMOVSDZmr :
555 HasAVX ? X86::VMOVSDmr :
556 X86::MOVSDmr);
557 if (X86::PSRRegBankID == RB.getID())
558 return Isload ? X86::LD_Fp64m : X86::ST_Fp64m;
559 } else if (Ty == LLT::scalar(SizeInBits: 80)) {
560 return Isload ? X86::LD_Fp80m : X86::ST_FpP80m;
561 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
562 if (Alignment >= Align(16))
563 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
564 : HasAVX512
565 ? X86::VMOVAPSZ128rm_NOVLX
566 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
567 : (HasVLX ? X86::VMOVAPSZ128mr
568 : HasAVX512
569 ? X86::VMOVAPSZ128mr_NOVLX
570 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
571 else
572 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
573 : HasAVX512
574 ? X86::VMOVUPSZ128rm_NOVLX
575 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
576 : (HasVLX ? X86::VMOVUPSZ128mr
577 : HasAVX512
578 ? X86::VMOVUPSZ128mr_NOVLX
579 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
580 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
581 if (Alignment >= Align(32))
582 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
583 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
584 : X86::VMOVAPSYrm)
585 : (HasVLX ? X86::VMOVAPSZ256mr
586 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
587 : X86::VMOVAPSYmr);
588 else
589 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
590 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
591 : X86::VMOVUPSYrm)
592 : (HasVLX ? X86::VMOVUPSZ256mr
593 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
594 : X86::VMOVUPSYmr);
595 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
596 if (Alignment >= Align(64))
597 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
598 else
599 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
600 }
601 return Opc;
602}
603
604// Fill in an address from the given instruction.
605static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
606 const MachineRegisterInfo &MRI,
607 const X86Subtarget &STI, X86AddressMode &AM) {
608 assert(I.getOperand(0).isReg() && "unsupported operand.");
609 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
610 "unsupported type.");
611
612 switch (I.getOpcode()) {
613 default:
614 break;
615 case TargetOpcode::G_FRAME_INDEX:
616 AM.Base.FrameIndex = I.getOperand(i: 1).getIndex();
617 AM.BaseType = X86AddressMode::FrameIndexBase;
618 return true;
619 case TargetOpcode::G_PTR_ADD: {
620 if (auto COff = getIConstantVRegSExtVal(VReg: I.getOperand(i: 2).getReg(), MRI)) {
621 int64_t Imm = *COff;
622 if (isInt<32>(x: Imm)) { // Check for displacement overflow.
623 AM.Disp = static_cast<int32_t>(Imm);
624 AM.Base.Reg = I.getOperand(i: 1).getReg();
625 return true;
626 }
627 }
628 break;
629 }
630 case TargetOpcode::G_GLOBAL_VALUE: {
631 auto GV = I.getOperand(i: 1).getGlobal();
632 if (GV->isThreadLocal()) {
633 return false; // TODO: we don't support TLS yet.
634 }
635 // Can't handle alternate code models yet.
636 if (TM.getCodeModel() != CodeModel::Small)
637 return false;
638 AM.GV = GV;
639 AM.GVOpFlags = STI.classifyGlobalReference(GV);
640
641 // TODO: The ABI requires an extra load. not supported yet.
642 if (isGlobalStubReference(TargetFlag: AM.GVOpFlags))
643 return false;
644
645 // TODO: This reference is relative to the pic base. not supported yet.
646 if (isGlobalRelativeToPICBase(TargetFlag: AM.GVOpFlags))
647 return false;
648
649 if (STI.isPICStyleRIPRel()) {
650 // Use rip-relative addressing.
651 assert(AM.Base.Reg == 0 && AM.IndexReg == 0 &&
652 "RIP-relative addresses can't have additional register operands");
653 AM.Base.Reg = X86::RIP;
654 }
655 return true;
656 }
657 case TargetOpcode::G_CONSTANT_POOL: {
658 // TODO: Need a separate move for Large model
659 if (TM.getCodeModel() == CodeModel::Large)
660 return false;
661
662 AM.GVOpFlags = STI.classifyLocalReference(GV: nullptr);
663 if (AM.GVOpFlags == X86II::MO_GOTOFF)
664 AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(MF: I.getMF());
665 else if (STI.is64Bit())
666 AM.Base.Reg = X86::RIP;
667 AM.CP = true;
668 AM.Disp = I.getOperand(i: 1).getIndex();
669 return true;
670 }
671 }
672 // Default behavior.
673 AM.Base.Reg = I.getOperand(i: 0).getReg();
674 return true;
675}
676
677bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
678 MachineRegisterInfo &MRI,
679 MachineFunction &MF) const {
680 unsigned Opc = I.getOpcode();
681
682 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
683 "Only G_STORE and G_LOAD are expected for selection");
684
685 const Register DefReg = I.getOperand(i: 0).getReg();
686 LLT Ty = MRI.getType(Reg: DefReg);
687 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
688
689 assert(I.hasOneMemOperand());
690 auto &MemOp = **I.memoperands_begin();
691 if (MemOp.isAtomic()) {
692 // Note: for unordered operations, we rely on the fact the appropriate MMO
693 // is already on the instruction we're mutating, and thus we don't need to
694 // make any changes. So long as we select an opcode which is capable of
695 // loading or storing the appropriate size atomically, the rest of the
696 // backend is required to respect the MMO state.
697 if (!MemOp.isUnordered()) {
698 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
699 return false;
700 }
701 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
702 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
703 return false;
704 }
705 }
706
707 unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
708 if (NewOpc == Opc)
709 return false;
710
711 I.setDesc(TII.get(Opcode: NewOpc));
712 MachineInstrBuilder MIB(MF, I);
713 MachineInstr *Ptr = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
714
715 X86AddressMode AM;
716 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
717 return false;
718
719 if (Opc == TargetOpcode::G_LOAD) {
720 I.removeOperand(OpNo: 1);
721 addFullAddress(MIB, AM);
722 } else {
723 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
724 I.removeOperand(OpNo: 1);
725 I.removeOperand(OpNo: 0);
726 addFullAddress(MIB, AM).addUse(RegNo: DefReg);
727 }
728 bool Constrained = constrainSelectedInstRegOperands(I, TII, TRI, RBI);
729 I.addImplicitDefUseOperands(MF);
730 return Constrained;
731}
732
733static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
734 if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 64))
735 return X86::LEA64r;
736 else if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 32))
737 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
738 else
739 llvm_unreachable("Can't get LEA opcode. Unsupported type.");
740}
741
742bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
743 MachineRegisterInfo &MRI,
744 MachineFunction &MF) const {
745 unsigned Opc = I.getOpcode();
746
747 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
748 "unexpected instruction");
749
750 const Register DefReg = I.getOperand(i: 0).getReg();
751 LLT Ty = MRI.getType(Reg: DefReg);
752
753 // Use LEA to calculate frame index and GEP
754 unsigned NewOpc = getLeaOP(Ty, STI);
755 I.setDesc(TII.get(Opcode: NewOpc));
756 MachineInstrBuilder MIB(MF, I);
757
758 if (Opc == TargetOpcode::G_FRAME_INDEX) {
759 addOffset(MIB, Offset: 0);
760 } else {
761 MachineOperand &InxOp = I.getOperand(i: 2);
762 I.addOperand(Op: InxOp); // set IndexReg
763 InxOp.ChangeToImmediate(ImmVal: 1); // set Scale
764 MIB.addImm(Val: 0).addReg(RegNo: 0);
765 }
766
767 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
768}
769
770bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
771 MachineRegisterInfo &MRI,
772 MachineFunction &MF) const {
773 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
774 "unexpected instruction");
775
776 X86AddressMode AM;
777 if (!X86SelectAddress(I, TM, MRI, STI, AM))
778 return false;
779
780 const Register DefReg = I.getOperand(i: 0).getReg();
781 LLT Ty = MRI.getType(Reg: DefReg);
782 unsigned NewOpc = getLeaOP(Ty, STI);
783
784 I.setDesc(TII.get(Opcode: NewOpc));
785 MachineInstrBuilder MIB(MF, I);
786
787 I.removeOperand(OpNo: 1);
788 addFullAddress(MIB, AM);
789
790 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
791}
792
793bool X86InstructionSelector::selectConstant(MachineInstr &I,
794 MachineRegisterInfo &MRI,
795 MachineFunction &MF) const {
796 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
797 "unexpected instruction");
798
799 const Register DefReg = I.getOperand(i: 0).getReg();
800 LLT Ty = MRI.getType(Reg: DefReg);
801
802 if (RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
803 return false;
804
805 uint64_t Val = 0;
806 if (I.getOperand(i: 1).isCImm()) {
807 Val = I.getOperand(i: 1).getCImm()->getZExtValue();
808 I.getOperand(i: 1).ChangeToImmediate(ImmVal: Val);
809 } else if (I.getOperand(i: 1).isImm()) {
810 Val = I.getOperand(i: 1).getImm();
811 } else
812 llvm_unreachable("Unsupported operand type.");
813
814 unsigned NewOpc;
815 switch (Ty.getSizeInBits()) {
816 case 8:
817 NewOpc = X86::MOV8ri;
818 break;
819 case 16:
820 NewOpc = X86::MOV16ri;
821 break;
822 case 32:
823 NewOpc = X86::MOV32ri;
824 break;
825 case 64:
826 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
827 if (isInt<32>(x: Val))
828 NewOpc = X86::MOV64ri32;
829 else
830 NewOpc = X86::MOV64ri;
831 break;
832 default:
833 llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
834 }
835
836 I.setDesc(TII.get(Opcode: NewOpc));
837 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
838}
839
840// Helper function for selectTruncOrPtrToInt and selectAnyext.
841// Returns true if DstRC lives on a floating register class and
842// SrcRC lives on a 128-bit vector class.
843static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
844 const TargetRegisterClass *SrcRC) {
845 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
846 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
847 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
848}
849
850bool X86InstructionSelector::selectTurnIntoCOPY(
851 MachineInstr &I, MachineRegisterInfo &MRI, const Register DstReg,
852 const TargetRegisterClass *DstRC, const Register SrcReg,
853 const TargetRegisterClass *SrcRC) const {
854
855 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
856 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
857 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
858 << " operand\n");
859 return false;
860 }
861 I.setDesc(TII.get(Opcode: X86::COPY));
862 return true;
863}
864
865bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
866 MachineRegisterInfo &MRI,
867 MachineFunction &MF) const {
868 assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
869 I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
870 "unexpected instruction");
871
872 const Register DstReg = I.getOperand(i: 0).getReg();
873 const Register SrcReg = I.getOperand(i: 1).getReg();
874
875 const LLT DstTy = MRI.getType(Reg: DstReg);
876 const LLT SrcTy = MRI.getType(Reg: SrcReg);
877
878 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
879 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
880
881 if (DstRB.getID() != SrcRB.getID()) {
882 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
883 << " input/output on different banks\n");
884 return false;
885 }
886
887 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
888 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
889
890 if (!DstRC || !SrcRC)
891 return false;
892
893 // If that's truncation of the value that lives on the vector class and goes
894 // into the floating class, just replace it with copy, as we are able to
895 // select it as a regular move.
896 if (canTurnIntoCOPY(DstRC, SrcRC))
897 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
898
899 if (DstRB.getID() != X86::GPRRegBankID)
900 return false;
901
902 unsigned SubIdx;
903 if (DstRC == SrcRC) {
904 // Nothing to be done
905 SubIdx = X86::NoSubRegister;
906 } else if (DstRC == &X86::GR32RegClass) {
907 SubIdx = X86::sub_32bit;
908 } else if (DstRC == &X86::GR16RegClass) {
909 SubIdx = X86::sub_16bit;
910 } else if (DstRC == &X86::GR8RegClass) {
911 SubIdx = X86::sub_8bit;
912 } else {
913 return false;
914 }
915
916 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
917
918 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
919 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
920 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
921 << "\n");
922 return false;
923 }
924
925 I.getOperand(i: 1).setSubReg(SubIdx);
926
927 I.setDesc(TII.get(Opcode: X86::COPY));
928 return true;
929}
930
931bool X86InstructionSelector::selectZext(MachineInstr &I,
932 MachineRegisterInfo &MRI,
933 MachineFunction &MF) const {
934 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
935
936 const Register DstReg = I.getOperand(i: 0).getReg();
937 const Register SrcReg = I.getOperand(i: 1).getReg();
938
939 const LLT DstTy = MRI.getType(Reg: DstReg);
940 const LLT SrcTy = MRI.getType(Reg: SrcReg);
941
942 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) &&
943 "8=>16 Zext is handled by tablegen");
944 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
945 "8=>32 Zext is handled by tablegen");
946 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
947 "16=>32 Zext is handled by tablegen");
948 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) &&
949 "8=>64 Zext is handled by tablegen");
950 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) &&
951 "16=>64 Zext is handled by tablegen");
952 assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) &&
953 "32=>64 Zext is handled by tablegen");
954
955 if (SrcTy != LLT::scalar(SizeInBits: 1))
956 return false;
957
958 unsigned AndOpc;
959 if (DstTy == LLT::scalar(SizeInBits: 8))
960 AndOpc = X86::AND8ri;
961 else if (DstTy == LLT::scalar(SizeInBits: 16))
962 AndOpc = X86::AND16ri;
963 else if (DstTy == LLT::scalar(SizeInBits: 32))
964 AndOpc = X86::AND32ri;
965 else if (DstTy == LLT::scalar(SizeInBits: 64))
966 AndOpc = X86::AND64ri32;
967 else
968 return false;
969
970 Register DefReg = SrcReg;
971 if (DstTy != LLT::scalar(SizeInBits: 8)) {
972 Register ImpDefReg =
973 MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
974 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
975 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImpDefReg);
976
977 DefReg = MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
978 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
979 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DefReg)
980 .addReg(RegNo: ImpDefReg)
981 .addReg(RegNo: SrcReg)
982 .addImm(Val: X86::sub_8bit);
983 }
984
985 MachineInstr &AndInst =
986 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AndOpc), DestReg: DstReg)
987 .addReg(RegNo: DefReg)
988 .addImm(Val: 1);
989
990 constrainSelectedInstRegOperands(I&: AndInst, TII, TRI, RBI);
991
992 I.eraseFromParent();
993 return true;
994}
995
996bool X86InstructionSelector::selectAnyext(MachineInstr &I,
997 MachineRegisterInfo &MRI,
998 MachineFunction &MF) const {
999 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
1000
1001 const Register DstReg = I.getOperand(i: 0).getReg();
1002 const Register SrcReg = I.getOperand(i: 1).getReg();
1003
1004 const LLT DstTy = MRI.getType(Reg: DstReg);
1005 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1006
1007 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1008 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
1009
1010 assert(DstRB.getID() == SrcRB.getID() &&
1011 "G_ANYEXT input/output on different banks\n");
1012
1013 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
1014 "G_ANYEXT incorrect operand size");
1015
1016 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
1017 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
1018
1019 // If that's ANY_EXT of the value that lives on the floating class and goes
1020 // into the vector class, just replace it with copy, as we are able to select
1021 // it as a regular move.
1022 if (canTurnIntoCOPY(DstRC: SrcRC, SrcRC: DstRC))
1023 return selectTurnIntoCOPY(I, MRI, DstReg: SrcReg, DstRC: SrcRC, SrcReg: DstReg, SrcRC: DstRC);
1024
1025 if (DstRB.getID() != X86::GPRRegBankID)
1026 return false;
1027
1028 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1029 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1030 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1031 << " operand\n");
1032 return false;
1033 }
1034
1035 if (SrcRC == DstRC) {
1036 I.setDesc(TII.get(Opcode: X86::COPY));
1037 return true;
1038 }
1039
1040 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1041 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
1042 .addDef(RegNo: DstReg)
1043 .addImm(Val: 0)
1044 .addReg(RegNo: SrcReg)
1045 .addImm(Val: getSubRegIndex(RC: SrcRC));
1046
1047 I.eraseFromParent();
1048 return true;
1049}
1050
1051bool X86InstructionSelector::selectCmp(MachineInstr &I,
1052 MachineRegisterInfo &MRI,
1053 MachineFunction &MF) const {
1054 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
1055
1056 X86::CondCode CC;
1057 bool SwapArgs;
1058 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(
1059 Predicate: (CmpInst::Predicate)I.getOperand(i: 1).getPredicate());
1060
1061 Register LHS = I.getOperand(i: 2).getReg();
1062 Register RHS = I.getOperand(i: 3).getReg();
1063
1064 if (SwapArgs)
1065 std::swap(a&: LHS, b&: RHS);
1066
1067 unsigned OpCmp;
1068 LLT Ty = MRI.getType(Reg: LHS);
1069
1070 switch (Ty.getSizeInBits()) {
1071 default:
1072 return false;
1073 case 8:
1074 OpCmp = X86::CMP8rr;
1075 break;
1076 case 16:
1077 OpCmp = X86::CMP16rr;
1078 break;
1079 case 32:
1080 OpCmp = X86::CMP32rr;
1081 break;
1082 case 64:
1083 OpCmp = X86::CMP64rr;
1084 break;
1085 }
1086
1087 MachineInstr &CmpInst =
1088 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1089 .addReg(RegNo: LHS)
1090 .addReg(RegNo: RHS);
1091
1092 MachineInstr &SetInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1093 MCID: TII.get(Opcode: X86::SETCCr), DestReg: I.getOperand(i: 0).getReg()).addImm(Val: CC);
1094
1095 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1096 constrainSelectedInstRegOperands(I&: SetInst, TII, TRI, RBI);
1097
1098 I.eraseFromParent();
1099 return true;
1100}
1101
1102bool X86InstructionSelector::selectFCmp(MachineInstr &I,
1103 MachineRegisterInfo &MRI,
1104 MachineFunction &MF) const {
1105 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
1106
1107 Register LhsReg = I.getOperand(i: 2).getReg();
1108 Register RhsReg = I.getOperand(i: 3).getReg();
1109 CmpInst::Predicate Predicate =
1110 (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1111
1112 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1113 static const uint16_t SETFOpcTable[2][3] = {
1114 {X86::COND_E, X86::COND_NP, X86::AND8rr},
1115 {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1116 const uint16_t *SETFOpc = nullptr;
1117 switch (Predicate) {
1118 default:
1119 break;
1120 case CmpInst::FCMP_OEQ:
1121 SETFOpc = &SETFOpcTable[0][0];
1122 break;
1123 case CmpInst::FCMP_UNE:
1124 SETFOpc = &SETFOpcTable[1][0];
1125 break;
1126 }
1127
1128 assert((LhsReg.isVirtual() && RhsReg.isVirtual()) &&
1129 "Both arguments of FCMP need to be virtual!");
1130 auto *LhsBank = RBI.getRegBank(Reg: LhsReg, MRI, TRI);
1131 [[maybe_unused]] auto *RhsBank = RBI.getRegBank(Reg: RhsReg, MRI, TRI);
1132 assert((LhsBank == RhsBank) &&
1133 "Both banks assigned to FCMP arguments need to be same!");
1134
1135 // Compute the opcode for the CMP instruction.
1136 unsigned OpCmp;
1137 LLT Ty = MRI.getType(Reg: LhsReg);
1138 switch (Ty.getSizeInBits()) {
1139 default:
1140 return false;
1141 case 32:
1142 OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr32
1143 : X86::UCOMISSrr;
1144 break;
1145 case 64:
1146 OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr64
1147 : X86::UCOMISDrr;
1148 break;
1149 case 80:
1150 OpCmp = X86::UCOM_FpIr80;
1151 break;
1152 }
1153
1154 Register ResultReg = I.getOperand(i: 0).getReg();
1155 RBI.constrainGenericRegister(
1156 Reg: ResultReg,
1157 RC: *getRegClass(Ty: LLT::scalar(SizeInBits: 8), RB: *RBI.getRegBank(Reg: ResultReg, MRI, TRI)), MRI);
1158 if (SETFOpc) {
1159 MachineInstr &CmpInst =
1160 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1161 .addReg(RegNo: LhsReg)
1162 .addReg(RegNo: RhsReg);
1163
1164 Register FlagReg1 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1165 Register FlagReg2 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1166 MachineInstr &Set1 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1167 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg1).addImm(Val: SETFOpc[0]);
1168 MachineInstr &Set2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1169 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg2).addImm(Val: SETFOpc[1]);
1170 MachineInstr &Set3 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1171 MCID: TII.get(Opcode: SETFOpc[2]), DestReg: ResultReg)
1172 .addReg(RegNo: FlagReg1)
1173 .addReg(RegNo: FlagReg2);
1174 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1175 constrainSelectedInstRegOperands(I&: Set1, TII, TRI, RBI);
1176 constrainSelectedInstRegOperands(I&: Set2, TII, TRI, RBI);
1177 constrainSelectedInstRegOperands(I&: Set3, TII, TRI, RBI);
1178
1179 I.eraseFromParent();
1180 return true;
1181 }
1182
1183 X86::CondCode CC;
1184 bool SwapArgs;
1185 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1186 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1187
1188 if (SwapArgs)
1189 std::swap(a&: LhsReg, b&: RhsReg);
1190
1191 // Emit a compare of LHS/RHS.
1192 MachineInstr &CmpInst =
1193 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1194 .addReg(RegNo: LhsReg)
1195 .addReg(RegNo: RhsReg);
1196
1197 MachineInstr &Set =
1198 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: ResultReg).addImm(Val: CC);
1199 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1200 constrainSelectedInstRegOperands(I&: Set, TII, TRI, RBI);
1201 I.eraseFromParent();
1202 return true;
1203}
1204
1205bool X86InstructionSelector::selectUAddSub(MachineInstr &I,
1206 MachineRegisterInfo &MRI,
1207 MachineFunction &MF) const {
1208 assert((I.getOpcode() == TargetOpcode::G_UADDE ||
1209 I.getOpcode() == TargetOpcode::G_UADDO ||
1210 I.getOpcode() == TargetOpcode::G_USUBE ||
1211 I.getOpcode() == TargetOpcode::G_USUBO) &&
1212 "unexpected instruction");
1213
1214 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
1215
1216 const Register DstReg = CarryMI.getDstReg();
1217 const Register CarryOutReg = CarryMI.getCarryOutReg();
1218 const Register Op0Reg = CarryMI.getLHSReg();
1219 const Register Op1Reg = CarryMI.getRHSReg();
1220 bool IsSub = CarryMI.isSub();
1221
1222 const LLT DstTy = MRI.getType(Reg: DstReg);
1223 assert(DstTy.isScalar() && "selectUAddSub only supported for scalar types");
1224
1225 // TODO: Handle immediate argument variants?
1226 unsigned OpADC, OpADD, OpSBB, OpSUB;
1227 switch (DstTy.getSizeInBits()) {
1228 case 8:
1229 OpADC = X86::ADC8rr;
1230 OpADD = X86::ADD8rr;
1231 OpSBB = X86::SBB8rr;
1232 OpSUB = X86::SUB8rr;
1233 break;
1234 case 16:
1235 OpADC = X86::ADC16rr;
1236 OpADD = X86::ADD16rr;
1237 OpSBB = X86::SBB16rr;
1238 OpSUB = X86::SUB16rr;
1239 break;
1240 case 32:
1241 OpADC = X86::ADC32rr;
1242 OpADD = X86::ADD32rr;
1243 OpSBB = X86::SBB32rr;
1244 OpSUB = X86::SUB32rr;
1245 break;
1246 case 64:
1247 OpADC = X86::ADC64rr;
1248 OpADD = X86::ADD64rr;
1249 OpSBB = X86::SBB64rr;
1250 OpSUB = X86::SUB64rr;
1251 break;
1252 default:
1253 llvm_unreachable("selectUAddSub unsupported type.");
1254 }
1255
1256 const RegisterBank &CarryRB = *RBI.getRegBank(Reg: CarryOutReg, MRI, TRI);
1257 const TargetRegisterClass *CarryRC =
1258 getRegClass(Ty: MRI.getType(Reg: CarryOutReg), RB: CarryRB);
1259
1260 unsigned Opcode = IsSub ? OpSUB : OpADD;
1261
1262 // G_UADDE/G_USUBE - find CarryIn def instruction.
1263 if (auto CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
1264 Register CarryInReg = CarryInMI->getCarryInReg();
1265 MachineInstr *Def = MRI.getVRegDef(Reg: CarryInReg);
1266 while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1267 CarryInReg = Def->getOperand(i: 1).getReg();
1268 Def = MRI.getVRegDef(Reg: CarryInReg);
1269 }
1270
1271 // TODO - handle more CF generating instructions
1272 if (Def->getOpcode() == TargetOpcode::G_UADDE ||
1273 Def->getOpcode() == TargetOpcode::G_UADDO ||
1274 Def->getOpcode() == TargetOpcode::G_USUBE ||
1275 Def->getOpcode() == TargetOpcode::G_USUBO) {
1276 // carry set by prev ADD/SUB.
1277
1278 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::CMP8ri))
1279 .addReg(RegNo: CarryInReg)
1280 .addImm(Val: 1);
1281
1282 if (!RBI.constrainGenericRegister(Reg: CarryInReg, RC: *CarryRC, MRI))
1283 return false;
1284
1285 Opcode = IsSub ? OpSBB : OpADC;
1286 } else if (auto val = getIConstantVRegVal(VReg: CarryInReg, MRI)) {
1287 // carry is constant, support only 0.
1288 if (*val != 0)
1289 return false;
1290
1291 Opcode = IsSub ? OpSUB : OpADD;
1292 } else
1293 return false;
1294 }
1295
1296 MachineInstr &Inst =
1297 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1298 .addReg(RegNo: Op0Reg)
1299 .addReg(RegNo: Op1Reg);
1300
1301 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: CarryOutReg)
1302 .addImm(Val: X86::COND_B);
1303
1304 if (!constrainSelectedInstRegOperands(I&: Inst, TII, TRI, RBI) ||
1305 !RBI.constrainGenericRegister(Reg: CarryOutReg, RC: *CarryRC, MRI))
1306 return false;
1307
1308 I.eraseFromParent();
1309 return true;
1310}
1311
1312bool X86InstructionSelector::selectExtract(MachineInstr &I,
1313 MachineRegisterInfo &MRI,
1314 MachineFunction &MF) const {
1315 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1316 "unexpected instruction");
1317
1318 const Register DstReg = I.getOperand(i: 0).getReg();
1319 const Register SrcReg = I.getOperand(i: 1).getReg();
1320 int64_t Index = I.getOperand(i: 2).getImm();
1321
1322 const LLT DstTy = MRI.getType(Reg: DstReg);
1323 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1324
1325 // Meanwile handle vector type only.
1326 if (!DstTy.isVector())
1327 return false;
1328
1329 if (Index % DstTy.getSizeInBits() != 0)
1330 return false; // Not extract subvector.
1331
1332 if (Index == 0) {
1333 // Replace by extract subreg copy.
1334 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1335 return false;
1336
1337 I.eraseFromParent();
1338 return true;
1339 }
1340
1341 bool HasAVX = STI.hasAVX();
1342 bool HasAVX512 = STI.hasAVX512();
1343 bool HasVLX = STI.hasVLX();
1344
1345 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1346 if (HasVLX)
1347 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Z256rri));
1348 else if (HasAVX)
1349 I.setDesc(TII.get(Opcode: X86::VEXTRACTF128rri));
1350 else
1351 return false;
1352 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1353 if (DstTy.getSizeInBits() == 128)
1354 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Zrri));
1355 else if (DstTy.getSizeInBits() == 256)
1356 I.setDesc(TII.get(Opcode: X86::VEXTRACTF64X4Zrri));
1357 else
1358 return false;
1359 } else
1360 return false;
1361
1362 // Convert to X86 VEXTRACT immediate.
1363 Index = Index / DstTy.getSizeInBits();
1364 I.getOperand(i: 2).setImm(Index);
1365
1366 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1367}
1368
1369bool X86InstructionSelector::emitExtractSubreg(Register DstReg, Register SrcReg,
1370 MachineInstr &I,
1371 MachineRegisterInfo &MRI,
1372 MachineFunction &MF) const {
1373 const LLT DstTy = MRI.getType(Reg: DstReg);
1374 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1375 unsigned SubIdx = X86::NoSubRegister;
1376
1377 if (!DstTy.isVector() || !SrcTy.isVector())
1378 return false;
1379
1380 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1381 "Incorrect Src/Dst register size");
1382
1383 if (DstTy.getSizeInBits() == 128)
1384 SubIdx = X86::sub_xmm;
1385 else if (DstTy.getSizeInBits() == 256)
1386 SubIdx = X86::sub_ymm;
1387 else
1388 return false;
1389
1390 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1391 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1392
1393 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
1394
1395 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1396 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1397 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1398 return false;
1399 }
1400
1401 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::COPY), DestReg: DstReg)
1402 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubIdx);
1403
1404 return true;
1405}
1406
1407bool X86InstructionSelector::emitInsertSubreg(Register DstReg, Register SrcReg,
1408 MachineInstr &I,
1409 MachineRegisterInfo &MRI,
1410 MachineFunction &MF) const {
1411 const LLT DstTy = MRI.getType(Reg: DstReg);
1412 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1413 unsigned SubIdx = X86::NoSubRegister;
1414
1415 // TODO: support scalar types
1416 if (!DstTy.isVector() || !SrcTy.isVector())
1417 return false;
1418
1419 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1420 "Incorrect Src/Dst register size");
1421
1422 if (SrcTy.getSizeInBits() == 128)
1423 SubIdx = X86::sub_xmm;
1424 else if (SrcTy.getSizeInBits() == 256)
1425 SubIdx = X86::sub_ymm;
1426 else
1427 return false;
1428
1429 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1430 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1431
1432 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1433 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1434 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1435 return false;
1436 }
1437
1438 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::COPY))
1439 .addReg(RegNo: DstReg, Flags: RegState::DefineNoRead, SubReg: SubIdx)
1440 .addReg(RegNo: SrcReg);
1441
1442 return true;
1443}
1444
1445bool X86InstructionSelector::selectInsert(MachineInstr &I,
1446 MachineRegisterInfo &MRI,
1447 MachineFunction &MF) const {
1448 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1449
1450 const Register DstReg = I.getOperand(i: 0).getReg();
1451 const Register SrcReg = I.getOperand(i: 1).getReg();
1452 const Register InsertReg = I.getOperand(i: 2).getReg();
1453 int64_t Index = I.getOperand(i: 3).getImm();
1454
1455 const LLT DstTy = MRI.getType(Reg: DstReg);
1456 const LLT InsertRegTy = MRI.getType(Reg: InsertReg);
1457
1458 // Meanwile handle vector type only.
1459 if (!DstTy.isVector())
1460 return false;
1461
1462 if (Index % InsertRegTy.getSizeInBits() != 0)
1463 return false; // Not insert subvector.
1464
1465 if (Index == 0 && MRI.getVRegDef(Reg: SrcReg)->isImplicitDef()) {
1466 // Replace by subreg copy.
1467 if (!emitInsertSubreg(DstReg, SrcReg: InsertReg, I, MRI, MF))
1468 return false;
1469
1470 I.eraseFromParent();
1471 return true;
1472 }
1473
1474 bool HasAVX = STI.hasAVX();
1475 bool HasAVX512 = STI.hasAVX512();
1476 bool HasVLX = STI.hasVLX();
1477
1478 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1479 if (HasVLX)
1480 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Z256rri));
1481 else if (HasAVX)
1482 I.setDesc(TII.get(Opcode: X86::VINSERTF128rri));
1483 else
1484 return false;
1485 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1486 if (InsertRegTy.getSizeInBits() == 128)
1487 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Zrri));
1488 else if (InsertRegTy.getSizeInBits() == 256)
1489 I.setDesc(TII.get(Opcode: X86::VINSERTF64X4Zrri));
1490 else
1491 return false;
1492 } else
1493 return false;
1494
1495 // Convert to X86 VINSERT immediate.
1496 Index = Index / InsertRegTy.getSizeInBits();
1497
1498 I.getOperand(i: 3).setImm(Index);
1499
1500 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1501}
1502
1503bool X86InstructionSelector::selectUnmergeValues(
1504 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1505 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1506 "unexpected instruction");
1507
1508 // Split to extracts.
1509 unsigned NumDefs = I.getNumOperands() - 1;
1510 Register SrcReg = I.getOperand(i: NumDefs).getReg();
1511 unsigned DefSize = MRI.getType(Reg: I.getOperand(i: 0).getReg()).getSizeInBits();
1512
1513 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1514 MachineInstr &ExtrInst =
1515 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1516 MCID: TII.get(Opcode: TargetOpcode::G_EXTRACT), DestReg: I.getOperand(i: Idx).getReg())
1517 .addReg(RegNo: SrcReg)
1518 .addImm(Val: Idx * DefSize);
1519
1520 if (!select(I&: ExtrInst))
1521 return false;
1522 }
1523
1524 I.eraseFromParent();
1525 return true;
1526}
1527
1528bool X86InstructionSelector::selectMergeValues(
1529 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1530 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1531 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1532 "unexpected instruction");
1533
1534 // Split to inserts.
1535 Register DstReg = I.getOperand(i: 0).getReg();
1536 Register SrcReg0 = I.getOperand(i: 1).getReg();
1537
1538 const LLT DstTy = MRI.getType(Reg: DstReg);
1539 const LLT SrcTy = MRI.getType(Reg: SrcReg0);
1540 unsigned SrcSize = SrcTy.getSizeInBits();
1541
1542 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1543
1544 // For the first src use insertSubReg.
1545 Register DefReg = MRI.createGenericVirtualRegister(Ty: DstTy);
1546 MRI.setRegBank(Reg: DefReg, RegBank);
1547 if (!emitInsertSubreg(DstReg: DefReg, SrcReg: I.getOperand(i: 1).getReg(), I, MRI, MF))
1548 return false;
1549
1550 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1551 Register Tmp = MRI.createGenericVirtualRegister(Ty: DstTy);
1552 MRI.setRegBank(Reg: Tmp, RegBank);
1553
1554 MachineInstr &InsertInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1555 MCID: TII.get(Opcode: TargetOpcode::G_INSERT), DestReg: Tmp)
1556 .addReg(RegNo: DefReg)
1557 .addReg(RegNo: I.getOperand(i: Idx).getReg())
1558 .addImm(Val: (Idx - 1) * SrcSize);
1559
1560 DefReg = Tmp;
1561
1562 if (!select(I&: InsertInst))
1563 return false;
1564 }
1565
1566 MachineInstr &CopyInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1567 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
1568 .addReg(RegNo: DefReg);
1569
1570 if (!select(I&: CopyInst))
1571 return false;
1572
1573 I.eraseFromParent();
1574 return true;
1575}
1576
1577bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1578 MachineRegisterInfo &MRI,
1579 MachineFunction &MF) const {
1580 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1581
1582 const Register CondReg = I.getOperand(i: 0).getReg();
1583 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1584
1585 MachineInstr &TestInst =
1586 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST8ri))
1587 .addReg(RegNo: CondReg)
1588 .addImm(Val: 1);
1589 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::JCC_1))
1590 .addMBB(MBB: DestMBB).addImm(Val: X86::COND_NE);
1591
1592 constrainSelectedInstRegOperands(I&: TestInst, TII, TRI, RBI);
1593
1594 I.eraseFromParent();
1595 return true;
1596}
1597
1598bool X86InstructionSelector::materializeFP(MachineInstr &I,
1599 MachineRegisterInfo &MRI,
1600 MachineFunction &MF) const {
1601 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1602 "unexpected instruction");
1603
1604 // Can't handle alternate code models yet.
1605 CodeModel::Model CM = TM.getCodeModel();
1606 if (CM != CodeModel::Small && CM != CodeModel::Large)
1607 return false;
1608
1609 const Register DstReg = I.getOperand(i: 0).getReg();
1610 const LLT DstTy = MRI.getType(Reg: DstReg);
1611 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1612 // Create the load from the constant pool.
1613 const ConstantFP *CFP = I.getOperand(i: 1).getFPImm();
1614 const auto &DL = MF.getDataLayout();
1615 Align Alignment = DL.getPrefTypeAlign(Ty: CFP->getType());
1616 const DebugLoc &DbgLoc = I.getDebugLoc();
1617
1618 unsigned Opc =
1619 getLoadStoreOp(Ty: DstTy, RB: RegBank, Opc: TargetOpcode::G_LOAD, Alignment);
1620
1621 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(C: CFP, Alignment);
1622 MachineInstr *LoadInst = nullptr;
1623 unsigned char OpFlag = STI.classifyLocalReference(GV: nullptr);
1624
1625 if (CM == CodeModel::Large && STI.is64Bit()) {
1626 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1627 // they cannot be folded into immediate fields.
1628
1629 Register AddrReg = MRI.createVirtualRegister(RegClass: &X86::GR64RegClass);
1630 BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: AddrReg)
1631 .addConstantPoolIndex(Idx: CPI, Offset: 0, TargetFlags: OpFlag);
1632
1633 MachineMemOperand *MMO = MF.getMachineMemOperand(
1634 PtrInfo: MachinePointerInfo::getConstantPool(MF), f: MachineMemOperand::MOLoad,
1635 MemTy: LLT::pointer(AddressSpace: 0, SizeInBits: DL.getPointerSizeInBits()), base_alignment: Alignment);
1636
1637 LoadInst =
1638 addDirectMem(MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg),
1639 Reg: AddrReg)
1640 .addMemOperand(MMO);
1641
1642 } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1643 // Handle the case when globals fit in our immediate field.
1644 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1645
1646 // x86-32 PIC requires a PIC base register for constant pools.
1647 unsigned PICBase = 0;
1648 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1649 // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1650 // In DAGISEL the code that initialize it generated by the CGBR pass.
1651 return false; // TODO support the mode.
1652 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1653 PICBase = X86::RIP;
1654
1655 LoadInst = addConstantPoolReference(
1656 MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg), CPI, GlobalBaseReg: PICBase,
1657 OpFlags: OpFlag);
1658 } else
1659 return false;
1660
1661 constrainSelectedInstRegOperands(I&: *LoadInst, TII, TRI, RBI);
1662 I.eraseFromParent();
1663 return true;
1664}
1665
1666bool X86InstructionSelector::selectImplicitDefOrPHI(
1667 MachineInstr &I, MachineRegisterInfo &MRI) const {
1668 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1669 I.getOpcode() == TargetOpcode::G_PHI) &&
1670 "unexpected instruction");
1671
1672 Register DstReg = I.getOperand(i: 0).getReg();
1673
1674 if (!MRI.getRegClassOrNull(Reg: DstReg)) {
1675 const LLT DstTy = MRI.getType(Reg: DstReg);
1676 const TargetRegisterClass *RC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1677
1678 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI)) {
1679 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1680 << " operand\n");
1681 return false;
1682 }
1683 }
1684
1685 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1686 I.setDesc(TII.get(Opcode: X86::IMPLICIT_DEF));
1687 else
1688 I.setDesc(TII.get(Opcode: X86::PHI));
1689
1690 return true;
1691}
1692
1693bool X86InstructionSelector::selectMulDivRem(MachineInstr &I,
1694 MachineRegisterInfo &MRI,
1695 MachineFunction &MF) const {
1696 // The implementation of this function is adapted from X86FastISel.
1697 assert((I.getOpcode() == TargetOpcode::G_MUL ||
1698 I.getOpcode() == TargetOpcode::G_SMULH ||
1699 I.getOpcode() == TargetOpcode::G_UMULH ||
1700 I.getOpcode() == TargetOpcode::G_SDIV ||
1701 I.getOpcode() == TargetOpcode::G_SREM ||
1702 I.getOpcode() == TargetOpcode::G_UDIV ||
1703 I.getOpcode() == TargetOpcode::G_UREM) &&
1704 "unexpected instruction");
1705
1706 const Register DstReg = I.getOperand(i: 0).getReg();
1707 const Register Op1Reg = I.getOperand(i: 1).getReg();
1708 const Register Op2Reg = I.getOperand(i: 2).getReg();
1709
1710 const LLT RegTy = MRI.getType(Reg: DstReg);
1711 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1712 "Arguments and return value types must match");
1713
1714 const RegisterBank *RegRB = RBI.getRegBank(Reg: DstReg, MRI, TRI);
1715 if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1716 return false;
1717
1718 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1719 const static unsigned NumOps = 7; // SDiv/SRem/UDiv/URem/Mul/SMulH/UMulh
1720 const static bool S = true; // IsSigned
1721 const static bool U = false; // !IsSigned
1722 const static unsigned Copy = TargetOpcode::COPY;
1723
1724 // For the X86 IDIV instruction, in most cases the dividend
1725 // (numerator) must be in a specific register pair highreg:lowreg,
1726 // producing the quotient in lowreg and the remainder in highreg.
1727 // For most data types, to set up the instruction, the dividend is
1728 // copied into lowreg, and lowreg is sign-extended into highreg. The
1729 // exception is i8, where the dividend is defined as a single register rather
1730 // than a register pair, and we therefore directly sign-extend the dividend
1731 // into lowreg, instead of copying, and ignore the highreg.
1732 const static struct MulDivRemEntry {
1733 // The following portion depends only on the data type.
1734 unsigned SizeInBits;
1735 unsigned LowInReg; // low part of the register pair
1736 unsigned HighInReg; // high part of the register pair
1737 // The following portion depends on both the data type and the operation.
1738 struct MulDivRemResult {
1739 unsigned OpMulDivRem; // The specific MUL/DIV opcode to use.
1740 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1741 // highreg, or copying a zero into highreg.
1742 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1743 // zero/sign-extending into lowreg for i8.
1744 unsigned ResultReg; // Register containing the desired result.
1745 bool IsOpSigned; // Whether to use signed or unsigned form.
1746 } ResultTable[NumOps];
1747 } OpTable[NumTypes] = {
1748 {.SizeInBits: 8,
1749 .LowInReg: X86::AX,
1750 .HighInReg: 0,
1751 .ResultTable: {
1752 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // SDiv
1753 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SRem
1754 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AL, .IsOpSigned: U}, // UDiv
1755 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // URem
1756 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // Mul
1757 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SMulH
1758 {.OpMulDivRem: X86::MUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // UMulH
1759 }}, // i8
1760 {.SizeInBits: 16,
1761 .LowInReg: X86::AX,
1762 .HighInReg: X86::DX,
1763 .ResultTable: {
1764 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // SDiv
1765 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SRem
1766 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: U}, // UDiv
1767 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // URem
1768 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // Mul
1769 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SMulH
1770 {.OpMulDivRem: X86::MUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // UMulH
1771 }}, // i16
1772 {.SizeInBits: 32,
1773 .LowInReg: X86::EAX,
1774 .HighInReg: X86::EDX,
1775 .ResultTable: {
1776 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // SDiv
1777 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SRem
1778 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: U}, // UDiv
1779 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // URem
1780 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // Mul
1781 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SMulH
1782 {.OpMulDivRem: X86::MUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // UMulH
1783 }}, // i32
1784 {.SizeInBits: 64,
1785 .LowInReg: X86::RAX,
1786 .HighInReg: X86::RDX,
1787 .ResultTable: {
1788 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // SDiv
1789 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SRem
1790 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: U}, // UDiv
1791 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // URem
1792 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // Mul
1793 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SMulH
1794 {.OpMulDivRem: X86::MUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // UMulH
1795 }}, // i64
1796 };
1797
1798 auto OpEntryIt = llvm::find_if(Range: OpTable, P: [RegTy](const MulDivRemEntry &El) {
1799 return El.SizeInBits == RegTy.getSizeInBits();
1800 });
1801 if (OpEntryIt == std::end(arr: OpTable))
1802 return false;
1803
1804 unsigned OpIndex;
1805 switch (I.getOpcode()) {
1806 default:
1807 llvm_unreachable("Unexpected mul/div/rem opcode");
1808 case TargetOpcode::G_SDIV:
1809 OpIndex = 0;
1810 break;
1811 case TargetOpcode::G_SREM:
1812 OpIndex = 1;
1813 break;
1814 case TargetOpcode::G_UDIV:
1815 OpIndex = 2;
1816 break;
1817 case TargetOpcode::G_UREM:
1818 OpIndex = 3;
1819 break;
1820 case TargetOpcode::G_MUL:
1821 OpIndex = 4;
1822 break;
1823 case TargetOpcode::G_SMULH:
1824 OpIndex = 5;
1825 break;
1826 case TargetOpcode::G_UMULH:
1827 OpIndex = 6;
1828 break;
1829 }
1830
1831 const MulDivRemEntry &TypeEntry = *OpEntryIt;
1832 const MulDivRemEntry::MulDivRemResult &OpEntry =
1833 TypeEntry.ResultTable[OpIndex];
1834
1835 const TargetRegisterClass *RegRC = getRegClass(Ty: RegTy, RB: *RegRB);
1836 if (!RBI.constrainGenericRegister(Reg: Op1Reg, RC: *RegRC, MRI) ||
1837 !RBI.constrainGenericRegister(Reg: Op2Reg, RC: *RegRC, MRI) ||
1838 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RegRC, MRI)) {
1839 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1840 << " operand\n");
1841 return false;
1842 }
1843
1844 // Move op1 into low-order input register.
1845 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpCopy),
1846 DestReg: TypeEntry.LowInReg)
1847 .addReg(RegNo: Op1Reg);
1848
1849 // Zero-extend or sign-extend into high-order input register.
1850 if (OpEntry.OpSignExtend) {
1851 if (OpEntry.IsOpSigned)
1852 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1853 MCID: TII.get(Opcode: OpEntry.OpSignExtend));
1854 else {
1855 Register Zero32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
1856 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::MOV32r0),
1857 DestReg: Zero32);
1858
1859 // Copy the zero into the appropriate sub/super/identical physical
1860 // register. Unfortunately the operations needed are not uniform enough
1861 // to fit neatly into the table above.
1862 if (RegTy.getSizeInBits() == 16) {
1863 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1864 DestReg: TypeEntry.HighInReg)
1865 .addReg(RegNo: Zero32, Flags: {}, SubReg: X86::sub_16bit);
1866 } else if (RegTy.getSizeInBits() == 32) {
1867 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1868 DestReg: TypeEntry.HighInReg)
1869 .addReg(RegNo: Zero32);
1870 } else if (RegTy.getSizeInBits() == 64) {
1871 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1872 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: TypeEntry.HighInReg)
1873 .addImm(Val: 0)
1874 .addReg(RegNo: Zero32)
1875 .addImm(Val: X86::sub_32bit);
1876 }
1877 }
1878 }
1879
1880 // Generate the DIV/IDIV/MUL/IMUL instruction.
1881 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpMulDivRem))
1882 .addReg(RegNo: Op2Reg);
1883
1884 // For i8 remainder, we can't reference ah directly, as we'll end
1885 // up with bogus copies like %r9b = COPY %ah. Reference ax
1886 // instead to prevent ah references in a rex instruction.
1887 //
1888 // The current assumption of the fast register allocator is that isel
1889 // won't generate explicit references to the GR8_NOREX registers. If
1890 // the allocator and/or the backend get enhanced to be more robust in
1891 // that regard, this can be, and should be, removed.
1892 if (OpEntry.ResultReg == X86::AH && STI.is64Bit()) {
1893 Register SourceSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1894 Register ResultSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1895 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy), DestReg: SourceSuperReg)
1896 .addReg(RegNo: X86::AX);
1897
1898 // Shift AX right by 8 bits instead of using AH.
1899 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SHR16ri),
1900 DestReg: ResultSuperReg)
1901 .addReg(RegNo: SourceSuperReg)
1902 .addImm(Val: 8);
1903
1904 // Now reference the 8-bit subreg of the result.
1905 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1906 DestReg: DstReg)
1907 .addReg(RegNo: ResultSuperReg, Flags: {}, SubReg: X86::sub_8bit);
1908 } else {
1909 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1910 DestReg: DstReg)
1911 .addReg(RegNo: OpEntry.ResultReg);
1912 }
1913 I.eraseFromParent();
1914
1915 return true;
1916}
1917
1918bool X86InstructionSelector::selectSelect(MachineInstr &I,
1919 MachineRegisterInfo &MRI,
1920 MachineFunction &MF) const {
1921 GSelect &Sel = cast<GSelect>(Val&: I);
1922 Register DstReg = Sel.getReg(Idx: 0);
1923 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST32rr))
1924 .addReg(RegNo: Sel.getCondReg())
1925 .addReg(RegNo: Sel.getCondReg());
1926
1927 unsigned OpCmp;
1928 LLT Ty = MRI.getType(Reg: DstReg);
1929 if (Ty.getSizeInBits() == 80) {
1930 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::CMOVE_Fp80),
1931 DestReg: DstReg)
1932 .addReg(RegNo: Sel.getTrueReg())
1933 .addReg(RegNo: Sel.getFalseReg());
1934 } else {
1935 switch (Ty.getSizeInBits()) {
1936 default:
1937 return false;
1938 case 8:
1939 OpCmp = X86::CMOV_GR8;
1940 break;
1941 case 16:
1942 OpCmp = STI.canUseCMOV() ? X86::CMOV16rr : X86::CMOV_GR16;
1943 break;
1944 case 32:
1945 OpCmp = STI.canUseCMOV() ? X86::CMOV32rr : X86::CMOV_GR32;
1946 break;
1947 case 64:
1948 assert(STI.is64Bit() && STI.canUseCMOV());
1949 OpCmp = X86::CMOV64rr;
1950 break;
1951 }
1952 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: OpCmp), DestReg: DstReg)
1953 .addReg(RegNo: Sel.getTrueReg())
1954 .addReg(RegNo: Sel.getFalseReg())
1955 .addImm(Val: X86::COND_E);
1956 }
1957 const TargetRegisterClass *DstRC = getRegClass(Ty, Reg: DstReg, MRI);
1958 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1959 LLVM_DEBUG(dbgs() << "Failed to constrain CMOV\n");
1960 return false;
1961 }
1962
1963 Sel.eraseFromParent();
1964 return true;
1965}
1966
1967InstructionSelector::ComplexRendererFns
1968X86InstructionSelector::selectAddr(MachineOperand &Root) const {
1969 MachineInstr *MI = Root.getParent();
1970 MachineIRBuilder MIRBuilder(*MI);
1971
1972 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
1973 MachineInstr *Ptr = MRI.getVRegDef(Reg: Root.getReg());
1974 X86AddressMode AM;
1975 X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM);
1976
1977 if (AM.IndexReg)
1978 return std::nullopt;
1979
1980 return {// Base
1981 {[=](MachineInstrBuilder &MIB) {
1982 if (AM.BaseType == X86AddressMode::RegBase)
1983 MIB.addUse(RegNo: AM.Base.Reg);
1984 else {
1985 assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
1986 "Unknown type of address base");
1987 MIB.addFrameIndex(Idx: AM.Base.FrameIndex);
1988 }
1989 },
1990 // Scale
1991 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: AM.Scale); },
1992 // Index
1993 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); },
1994 // Disp
1995 [=](MachineInstrBuilder &MIB) {
1996 if (AM.GV)
1997 MIB.addGlobalAddress(GV: AM.GV, Offset: AM.Disp, TargetFlags: AM.GVOpFlags);
1998 else if (AM.CP)
1999 MIB.addConstantPoolIndex(Idx: AM.Disp, Offset: 0, TargetFlags: AM.GVOpFlags);
2000 else
2001 MIB.addImm(Val: AM.Disp);
2002 },
2003 // Segment
2004 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); }}};
2005}
2006
2007InstructionSelector *
2008llvm::createX86InstructionSelector(const X86TargetMachine &TM,
2009 const X86Subtarget &Subtarget,
2010 const X86RegisterBankInfo &RBI) {
2011 return new X86InstructionSelector(TM, Subtarget, RBI);
2012}
2013