1//===- X86InstructionSelector.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// X86.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86BaseInfo.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86RegisterBankInfo.h"
19#include "X86RegisterInfo.h"
20#include "X86Subtarget.h"
21#include "X86TargetMachine.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/Utils.h"
27#include "llvm/CodeGen/MachineBasicBlock.h"
28#include "llvm/CodeGen/MachineConstantPool.h"
29#include "llvm/CodeGen/MachineFunction.h"
30#include "llvm/CodeGen/MachineInstr.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/MachineOperand.h"
34#include "llvm/CodeGen/MachineRegisterInfo.h"
35#include "llvm/CodeGen/RegisterBank.h"
36#include "llvm/CodeGen/TargetOpcodes.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/CodeGenTypes/LowLevelType.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/InstrTypes.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/Support/CodeGen.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MathExtras.h"
46#include "llvm/Support/raw_ostream.h"
47#include <cassert>
48#include <cstdint>
49#include <tuple>
50
51#define DEBUG_TYPE "X86-isel"
52
53using namespace llvm;
54using namespace MIPatternMatch;
55
56namespace {
57
58#define GET_GLOBALISEL_PREDICATE_BITSET
59#include "X86GenGlobalISel.inc"
60#undef GET_GLOBALISEL_PREDICATE_BITSET
61
62class X86InstructionSelector : public InstructionSelector {
63public:
64 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
65 const X86RegisterBankInfo &RBI);
66
67 bool select(MachineInstr &I) override;
68 static const char *getName() { return DEBUG_TYPE; }
69
70private:
71 /// tblgen-erated 'select' implementation, used as the initial selector for
72 /// the patterns that don't require complex C++.
73 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
74
75 // TODO: remove after supported by Tablegen-erated instruction selection.
76 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
77 Align Alignment) const;
78 // TODO: remove once p0<->i32/i64 matching is available
79 unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
80 unsigned Opc) const;
81
82 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
83 MachineFunction &MF) const;
84 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
85 MachineFunction &MF) const;
86 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
87 MachineFunction &MF) const;
88 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
89 MachineFunction &MF) const;
90 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
91 MachineFunction &MF) const;
92 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
93 MachineFunction &MF) const;
94 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
95 MachineFunction &MF) const;
96 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
97 MachineFunction &MF) const;
98 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
99 MachineFunction &MF) const;
100 bool selectUAddSub(MachineInstr &I, MachineRegisterInfo &MRI,
101 MachineFunction &MF) const;
102 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
103 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
104 MachineFunction &MF);
105 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
106 MachineFunction &MF);
107 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
108 MachineFunction &MF) const;
109 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
110 MachineFunction &MF) const;
111 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
112 MachineFunction &MF) const;
113 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
114 const Register DstReg,
115 const TargetRegisterClass *DstRC,
116 const Register SrcReg,
117 const TargetRegisterClass *SrcRC) const;
118 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
119 MachineFunction &MF) const;
120 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
121 bool selectMulDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
122 MachineFunction &MF) const;
123 bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
124 MachineFunction &MF) const;
125
126 ComplexRendererFns selectAddr(MachineOperand &Root) const;
127
128 // emit insert subreg instruction and insert it before MachineInstr &I
129 bool emitInsertSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
130 MachineRegisterInfo &MRI, MachineFunction &MF) const;
131 // emit extract subreg instruction and insert it before MachineInstr &I
132 bool emitExtractSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
133 MachineRegisterInfo &MRI, MachineFunction &MF) const;
134
135 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
136 const TargetRegisterClass *getRegClass(LLT Ty, Register Reg,
137 MachineRegisterInfo &MRI) const;
138
139 const X86TargetMachine &TM;
140 const X86Subtarget &STI;
141 const X86InstrInfo &TII;
142 const X86RegisterInfo &TRI;
143 const X86RegisterBankInfo &RBI;
144
145#define GET_GLOBALISEL_PREDICATES_DECL
146#include "X86GenGlobalISel.inc"
147#undef GET_GLOBALISEL_PREDICATES_DECL
148
149#define GET_GLOBALISEL_TEMPORARIES_DECL
150#include "X86GenGlobalISel.inc"
151#undef GET_GLOBALISEL_TEMPORARIES_DECL
152};
153
154} // end anonymous namespace
155
156#define GET_GLOBALISEL_IMPL
157#include "X86GenGlobalISel.inc"
158#undef GET_GLOBALISEL_IMPL
159
160X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
161 const X86Subtarget &STI,
162 const X86RegisterBankInfo &RBI)
163 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
164 RBI(RBI),
165#define GET_GLOBALISEL_PREDICATES_INIT
166#include "X86GenGlobalISel.inc"
167#undef GET_GLOBALISEL_PREDICATES_INIT
168#define GET_GLOBALISEL_TEMPORARIES_INIT
169#include "X86GenGlobalISel.inc"
170#undef GET_GLOBALISEL_TEMPORARIES_INIT
171{
172}
173
174// FIXME: This should be target-independent, inferred from the types declared
175// for each class in the bank.
176const TargetRegisterClass *
177X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
178 if (RB.getID() == X86::GPRRegBankID) {
179 if (Ty.getSizeInBits() <= 8)
180 return &X86::GR8RegClass;
181 if (Ty.getSizeInBits() == 16)
182 return &X86::GR16RegClass;
183 if (Ty.getSizeInBits() == 32)
184 return &X86::GR32RegClass;
185 if (Ty.getSizeInBits() == 64)
186 return &X86::GR64RegClass;
187 }
188 if (RB.getID() == X86::VECRRegBankID) {
189 if (Ty.getSizeInBits() == 16)
190 return STI.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
191 if (Ty.getSizeInBits() == 32)
192 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
193 if (Ty.getSizeInBits() == 64)
194 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
195 if (Ty.getSizeInBits() == 128)
196 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
197 if (Ty.getSizeInBits() == 256)
198 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
199 if (Ty.getSizeInBits() == 512)
200 return &X86::VR512RegClass;
201 }
202
203 if (RB.getID() == X86::PSRRegBankID) {
204 if (Ty.getSizeInBits() == 80)
205 return &X86::RFP80RegClass;
206 if (Ty.getSizeInBits() == 64)
207 return &X86::RFP64RegClass;
208 if (Ty.getSizeInBits() == 32)
209 return &X86::RFP32RegClass;
210 }
211
212 llvm_unreachable("Unknown RegBank!");
213}
214
215const TargetRegisterClass *
216X86InstructionSelector::getRegClass(LLT Ty, Register Reg,
217 MachineRegisterInfo &MRI) const {
218 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
219 return getRegClass(Ty, RB: RegBank);
220}
221
222static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
223 unsigned SubIdx = X86::NoSubRegister;
224 if (RC == &X86::GR32RegClass) {
225 SubIdx = X86::sub_32bit;
226 } else if (RC == &X86::GR16RegClass) {
227 SubIdx = X86::sub_16bit;
228 } else if (RC == &X86::GR8RegClass) {
229 SubIdx = X86::sub_8bit;
230 }
231
232 return SubIdx;
233}
234
235static const TargetRegisterClass *getRegClassFromGRPhysReg(Register Reg) {
236 assert(Reg.isPhysical());
237 if (X86::GR64RegClass.contains(Reg))
238 return &X86::GR64RegClass;
239 if (X86::GR32RegClass.contains(Reg))
240 return &X86::GR32RegClass;
241 if (X86::GR16RegClass.contains(Reg))
242 return &X86::GR16RegClass;
243 if (X86::GR8RegClass.contains(Reg))
244 return &X86::GR8RegClass;
245
246 llvm_unreachable("Unknown RegClass for PhysReg!");
247}
248
249// Set X86 Opcode and constrain DestReg.
250bool X86InstructionSelector::selectCopy(MachineInstr &I,
251 MachineRegisterInfo &MRI) const {
252 Register DstReg = I.getOperand(i: 0).getReg();
253 const unsigned DstSize = TRI.getRegSizeInBits(Reg: DstReg, MRI);
254 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
255
256 Register SrcReg = I.getOperand(i: 1).getReg();
257 const unsigned SrcSize = TRI.getRegSizeInBits(Reg: SrcReg, MRI);
258 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
259
260 if (DstReg.isPhysical()) {
261 assert(I.isCopy() && "Generic operators do not allow physical registers");
262
263 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
264 DstRegBank.getID() == X86::GPRRegBankID) {
265
266 const TargetRegisterClass *SrcRC =
267 getRegClass(Ty: MRI.getType(Reg: SrcReg), RB: SrcRegBank);
268 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(Reg: DstReg);
269
270 if (SrcRC != DstRC) {
271 // This case can be generated by ABI lowering, performe anyext
272 Register ExtSrc = MRI.createVirtualRegister(RegClass: DstRC);
273 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
274 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
275 .addDef(RegNo: ExtSrc)
276 .addReg(RegNo: SrcReg)
277 .addImm(Val: getSubRegIndex(RC: SrcRC));
278
279 I.getOperand(i: 1).setReg(ExtSrc);
280 }
281 }
282
283 // Special case GPR16 -> XMM
284 if (SrcSize == 16 && SrcRegBank.getID() == X86::GPRRegBankID &&
285 (DstRegBank.getID() == X86::VECRRegBankID)) {
286
287 const DebugLoc &DL = I.getDebugLoc();
288
289 // Any extend GPR16 -> GPR32
290 Register ExtReg = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
291 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG),
292 DestReg: ExtReg)
293 .addReg(RegNo: SrcReg)
294 .addImm(Val: X86::sub_16bit);
295
296 // Copy GR32 -> XMM
297 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
298 .addReg(RegNo: ExtReg);
299
300 I.eraseFromParent();
301 }
302
303 // Special case XMM -> GR16
304 if (DstSize == 16 && DstRegBank.getID() == X86::GPRRegBankID &&
305 (SrcRegBank.getID() == X86::VECRRegBankID)) {
306
307 const DebugLoc &DL = I.getDebugLoc();
308
309 // Move XMM to GR32 register.
310 Register Temp32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
311 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Temp32)
312 .addReg(RegNo: SrcReg);
313
314 // Extract the lower 16 bits
315 if (Register Dst32 = TRI.getMatchingSuperReg(Reg: DstReg, SubIdx: X86::sub_16bit,
316 RC: &X86::GR32RegClass)) {
317 // Optimization for Physical Dst (e.g. AX): Copy to EAX directly.
318 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst32)
319 .addReg(RegNo: Temp32);
320 } else {
321 // Handle if there is no super.
322 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
323 .addReg(RegNo: Temp32, Flags: {}, SubReg: X86::sub_16bit);
324 }
325
326 I.eraseFromParent();
327 }
328
329 return true;
330 }
331
332 assert((!SrcReg.isPhysical() || I.isCopy()) &&
333 "No phys reg on generic operators");
334 assert(
335 (DstSize == SrcSize ||
336 // Copies are a mean to setup initial types, the number of
337 // bits may not exactly match.
338 (SrcReg.isPhysical() && DstSize <= TRI.getRegSizeInBits(SrcReg, MRI))) &&
339 "Copy with different width?!");
340
341 const TargetRegisterClass *DstRC =
342 getRegClass(Ty: MRI.getType(Reg: DstReg), RB: DstRegBank);
343
344 if (SrcRegBank.getID() == X86::GPRRegBankID &&
345 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
346 SrcReg.isPhysical()) {
347 // Change the physical register to performe truncate.
348
349 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(Reg: SrcReg);
350
351 if (DstRC != SrcRC) {
352 I.getOperand(i: 1).setSubReg(getSubRegIndex(RC: DstRC));
353 I.getOperand(i: 1).substPhysReg(Reg: SrcReg, TRI);
354 }
355 }
356
357 // No need to constrain SrcReg. It will get constrained when
358 // we hit another of its use or its defs.
359 // Copies do not have constraints.
360 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(Reg: DstReg);
361 if (!OldRC || !DstRC->hasSubClassEq(RC: OldRC)) {
362 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
363 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
364 << " operand\n");
365 return false;
366 }
367 }
368 I.setDesc(TII.get(Opcode: X86::COPY));
369 return true;
370}
371
372bool X86InstructionSelector::select(MachineInstr &I) {
373 assert(I.getParent() && "Instruction should be in a basic block!");
374 assert(I.getParent()->getParent() && "Instruction should be in a function!");
375
376 MachineBasicBlock &MBB = *I.getParent();
377 MachineFunction &MF = *MBB.getParent();
378 MachineRegisterInfo &MRI = MF.getRegInfo();
379
380 unsigned Opcode = I.getOpcode();
381 if (!isPreISelGenericOpcode(Opcode) && !I.isPreISelOpcode()) {
382 // Certain non-generic instructions also need some special handling.
383
384 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
385 return false;
386
387 if (I.isCopy())
388 return selectCopy(I, MRI);
389
390 return true;
391 }
392
393 assert(I.getNumOperands() == I.getNumExplicitOperands() &&
394 "Generic instruction has unexpected implicit operands\n");
395
396 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
397 return true;
398
399 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
400
401 // TODO: This should be implemented by tblgen.
402 switch (I.getOpcode()) {
403 default:
404 return false;
405 case TargetOpcode::G_STORE:
406 case TargetOpcode::G_LOAD:
407 return selectLoadStoreOp(I, MRI, MF);
408 case TargetOpcode::G_PTR_ADD:
409 case TargetOpcode::G_FRAME_INDEX:
410 return selectFrameIndexOrGep(I, MRI, MF);
411 case TargetOpcode::G_GLOBAL_VALUE:
412 return selectGlobalValue(I, MRI, MF);
413 case TargetOpcode::G_CONSTANT:
414 return selectConstant(I, MRI, MF);
415 case TargetOpcode::G_FCONSTANT:
416 return materializeFP(I, MRI, MF);
417 case TargetOpcode::G_PTRTOINT:
418 case TargetOpcode::G_TRUNC:
419 return selectTruncOrPtrToInt(I, MRI, MF);
420 case TargetOpcode::G_INTTOPTR:
421 case TargetOpcode::G_FREEZE:
422 return selectCopy(I, MRI);
423 case TargetOpcode::G_ZEXT:
424 return selectZext(I, MRI, MF);
425 case TargetOpcode::G_ANYEXT:
426 return selectAnyext(I, MRI, MF);
427 case TargetOpcode::G_ICMP:
428 return selectCmp(I, MRI, MF);
429 case TargetOpcode::G_FCMP:
430 return selectFCmp(I, MRI, MF);
431 case TargetOpcode::G_UADDE:
432 case TargetOpcode::G_UADDO:
433 case TargetOpcode::G_USUBE:
434 case TargetOpcode::G_USUBO:
435 return selectUAddSub(I, MRI, MF);
436 case TargetOpcode::G_UNMERGE_VALUES:
437 return selectUnmergeValues(I, MRI, MF);
438 case TargetOpcode::G_MERGE_VALUES:
439 case TargetOpcode::G_CONCAT_VECTORS:
440 return selectMergeValues(I, MRI, MF);
441 case TargetOpcode::G_EXTRACT:
442 return selectExtract(I, MRI, MF);
443 case TargetOpcode::G_INSERT:
444 return selectInsert(I, MRI, MF);
445 case TargetOpcode::G_BRCOND:
446 return selectCondBranch(I, MRI, MF);
447 case TargetOpcode::G_IMPLICIT_DEF:
448 case TargetOpcode::G_PHI:
449 return selectImplicitDefOrPHI(I, MRI);
450 case TargetOpcode::G_MUL:
451 case TargetOpcode::G_SMULH:
452 case TargetOpcode::G_UMULH:
453 case TargetOpcode::G_SDIV:
454 case TargetOpcode::G_UDIV:
455 case TargetOpcode::G_SREM:
456 case TargetOpcode::G_UREM:
457 return selectMulDivRem(I, MRI, MF);
458 case TargetOpcode::G_SELECT:
459 return selectSelect(I, MRI, MF);
460 }
461
462 return false;
463}
464
465unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
466 const RegisterBank &RB,
467 unsigned Opc) const {
468 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
469 "Only G_STORE and G_LOAD are expected for selection");
470 if (Ty.isPointer() && X86::GPRRegBankID == RB.getID()) {
471 bool IsLoad = (Opc == TargetOpcode::G_LOAD);
472 switch (Ty.getSizeInBits()) {
473 default:
474 break;
475 case 32:
476 return IsLoad ? X86::MOV32rm : X86::MOV32mr;
477 case 64:
478 return IsLoad ? X86::MOV64rm : X86::MOV64mr;
479 }
480 }
481 return Opc;
482}
483
484unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
485 const RegisterBank &RB,
486 unsigned Opc,
487 Align Alignment) const {
488 bool Isload = (Opc == TargetOpcode::G_LOAD);
489 bool HasAVX = STI.hasAVX();
490 bool HasAVX512 = STI.hasAVX512();
491 bool HasVLX = STI.hasVLX();
492
493 if (Ty == LLT::scalar(SizeInBits: 8)) {
494 if (X86::GPRRegBankID == RB.getID())
495 return Isload ? X86::MOV8rm : X86::MOV8mr;
496 } else if (Ty == LLT::scalar(SizeInBits: 16)) {
497 if (X86::GPRRegBankID == RB.getID())
498 return Isload ? X86::MOV16rm : X86::MOV16mr;
499 } else if (Ty == LLT::scalar(SizeInBits: 32)) {
500 if (X86::GPRRegBankID == RB.getID())
501 return Isload ? X86::MOV32rm : X86::MOV32mr;
502 if (X86::VECRRegBankID == RB.getID())
503 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
504 HasAVX ? X86::VMOVSSrm_alt :
505 X86::MOVSSrm_alt)
506 : (HasAVX512 ? X86::VMOVSSZmr :
507 HasAVX ? X86::VMOVSSmr :
508 X86::MOVSSmr);
509 if (X86::PSRRegBankID == RB.getID())
510 return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
511 } else if (Ty == LLT::scalar(SizeInBits: 64)) {
512 if (X86::GPRRegBankID == RB.getID())
513 return Isload ? X86::MOV64rm : X86::MOV64mr;
514 if (X86::VECRRegBankID == RB.getID())
515 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
516 HasAVX ? X86::VMOVSDrm_alt :
517 X86::MOVSDrm_alt)
518 : (HasAVX512 ? X86::VMOVSDZmr :
519 HasAVX ? X86::VMOVSDmr :
520 X86::MOVSDmr);
521 if (X86::PSRRegBankID == RB.getID())
522 return Isload ? X86::LD_Fp64m : X86::ST_Fp64m;
523 } else if (Ty == LLT::scalar(SizeInBits: 80)) {
524 return Isload ? X86::LD_Fp80m : X86::ST_FpP80m;
525 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
526 if (Alignment >= Align(16))
527 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
528 : HasAVX512
529 ? X86::VMOVAPSZ128rm_NOVLX
530 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
531 : (HasVLX ? X86::VMOVAPSZ128mr
532 : HasAVX512
533 ? X86::VMOVAPSZ128mr_NOVLX
534 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
535 else
536 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
537 : HasAVX512
538 ? X86::VMOVUPSZ128rm_NOVLX
539 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
540 : (HasVLX ? X86::VMOVUPSZ128mr
541 : HasAVX512
542 ? X86::VMOVUPSZ128mr_NOVLX
543 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
544 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
545 if (Alignment >= Align(32))
546 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
547 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
548 : X86::VMOVAPSYrm)
549 : (HasVLX ? X86::VMOVAPSZ256mr
550 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
551 : X86::VMOVAPSYmr);
552 else
553 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
554 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
555 : X86::VMOVUPSYrm)
556 : (HasVLX ? X86::VMOVUPSZ256mr
557 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
558 : X86::VMOVUPSYmr);
559 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
560 if (Alignment >= Align(64))
561 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
562 else
563 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
564 }
565 return Opc;
566}
567
568// Fill in an address from the given instruction.
569static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
570 const MachineRegisterInfo &MRI,
571 const X86Subtarget &STI, X86AddressMode &AM) {
572 assert(I.getOperand(0).isReg() && "unsupported operand.");
573 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
574 "unsupported type.");
575
576 switch (I.getOpcode()) {
577 default:
578 break;
579 case TargetOpcode::G_FRAME_INDEX:
580 AM.Base.FrameIndex = I.getOperand(i: 1).getIndex();
581 AM.BaseType = X86AddressMode::FrameIndexBase;
582 return true;
583 case TargetOpcode::G_PTR_ADD: {
584 if (auto COff = getIConstantVRegSExtVal(VReg: I.getOperand(i: 2).getReg(), MRI)) {
585 int64_t Imm = *COff;
586 if (isInt<32>(x: Imm)) { // Check for displacement overflow.
587 AM.Disp = static_cast<int32_t>(Imm);
588 AM.Base.Reg = I.getOperand(i: 1).getReg();
589 return true;
590 }
591 }
592 break;
593 }
594 case TargetOpcode::G_GLOBAL_VALUE:
595 case X86::G_WRAPPER_RIP: {
596 auto GV = I.getOperand(i: 1).getGlobal();
597 if (GV->isThreadLocal()) {
598 return false; // TODO: we don't support TLS yet.
599 }
600 // Can't handle alternate code models yet.
601 if (TM.getCodeModel() != CodeModel::Small)
602 return false;
603 AM.GV = GV;
604 AM.GVOpFlags = STI.classifyGlobalReference(GV);
605
606 // TODO: This reference is relative to the pic base. not supported yet.
607 if (isGlobalRelativeToPICBase(TargetFlag: AM.GVOpFlags))
608 return false;
609
610 if (STI.isPICStyleRIPRel() || AM.GVOpFlags == X86II::MO_GOTPCREL ||
611 AM.GVOpFlags == X86II::MO_GOTPCREL_NORELAX) {
612 // Use rip-relative addressing.
613 assert(AM.Base.Reg == 0 && AM.IndexReg == 0 &&
614 "RIP-relative addresses can't have additional register operands");
615 AM.Base.Reg = X86::RIP;
616 }
617 return true;
618 }
619 case TargetOpcode::G_CONSTANT_POOL: {
620 // TODO: Need a separate move for Large model
621 if (TM.getCodeModel() == CodeModel::Large)
622 return false;
623
624 AM.GVOpFlags = STI.classifyLocalReference(GV: nullptr);
625 if (AM.GVOpFlags == X86II::MO_GOTOFF)
626 AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(MF: I.getMF());
627 else if (STI.is64Bit())
628 AM.Base.Reg = X86::RIP;
629 AM.CP = true;
630 AM.Disp = I.getOperand(i: 1).getIndex();
631 return true;
632 }
633 }
634 // Default behavior.
635 AM.Base.Reg = I.getOperand(i: 0).getReg();
636 return true;
637}
638
639bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
640 MachineRegisterInfo &MRI,
641 MachineFunction &MF) const {
642 unsigned Opc = I.getOpcode();
643
644 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
645 "Only G_STORE and G_LOAD are expected for selection");
646
647 const Register DefReg = I.getOperand(i: 0).getReg();
648 LLT Ty = MRI.getType(Reg: DefReg);
649 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
650
651 assert(I.hasOneMemOperand());
652 auto &MemOp = **I.memoperands_begin();
653 if (MemOp.isAtomic()) {
654 // Note: for unordered operations, we rely on the fact the appropriate MMO
655 // is already on the instruction we're mutating, and thus we don't need to
656 // make any changes. So long as we select an opcode which is capable of
657 // loading or storing the appropriate size atomically, the rest of the
658 // backend is required to respect the MMO state.
659 if (!MemOp.isUnordered()) {
660 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
661 return false;
662 }
663 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
664 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
665 return false;
666 }
667 }
668
669 unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
670 if (NewOpc == Opc)
671 return false;
672
673 I.setDesc(TII.get(Opcode: NewOpc));
674 MachineInstrBuilder MIB(MF, I);
675 MachineInstr *Ptr = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
676
677 X86AddressMode AM;
678 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
679 return false;
680
681 if (Opc == TargetOpcode::G_LOAD) {
682 I.removeOperand(OpNo: 1);
683 addFullAddress(MIB, AM);
684 } else {
685 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
686 I.removeOperand(OpNo: 1);
687 I.removeOperand(OpNo: 0);
688 addFullAddress(MIB, AM).addUse(RegNo: DefReg);
689 }
690 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
691 I.addImplicitDefUseOperands(MF);
692 return true;
693}
694
695static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
696 if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 64))
697 return X86::LEA64r;
698 else if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 32))
699 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
700 else
701 llvm_unreachable("Can't get LEA opcode. Unsupported type.");
702}
703
704bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
705 MachineRegisterInfo &MRI,
706 MachineFunction &MF) const {
707 unsigned Opc = I.getOpcode();
708
709 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
710 "unexpected instruction");
711
712 const Register DefReg = I.getOperand(i: 0).getReg();
713 LLT Ty = MRI.getType(Reg: DefReg);
714
715 // Use LEA to calculate frame index and GEP
716 unsigned NewOpc = getLeaOP(Ty, STI);
717 I.setDesc(TII.get(Opcode: NewOpc));
718 MachineInstrBuilder MIB(MF, I);
719
720 if (Opc == TargetOpcode::G_FRAME_INDEX) {
721 addOffset(MIB, Offset: 0);
722 } else {
723 MachineOperand &InxOp = I.getOperand(i: 2);
724 I.addOperand(Op: InxOp); // set IndexReg
725 InxOp.ChangeToImmediate(ImmVal: 1); // set Scale
726 MIB.addImm(Val: 0).addReg(RegNo: 0);
727 }
728
729 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
730 return true;
731}
732
733bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
734 MachineRegisterInfo &MRI,
735 MachineFunction &MF) const {
736 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
737 "unexpected instruction");
738
739 X86AddressMode AM;
740 if (!X86SelectAddress(I, TM, MRI, STI, AM))
741 return false;
742
743 const Register DefReg = I.getOperand(i: 0).getReg();
744 LLT Ty = MRI.getType(Reg: DefReg);
745 unsigned NewOpc = getLeaOP(Ty, STI);
746
747 I.setDesc(TII.get(Opcode: NewOpc));
748 MachineInstrBuilder MIB(MF, I);
749
750 I.removeOperand(OpNo: 1);
751 addFullAddress(MIB, AM);
752
753 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
754 return true;
755}
756
757bool X86InstructionSelector::selectConstant(MachineInstr &I,
758 MachineRegisterInfo &MRI,
759 MachineFunction &MF) const {
760 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
761 "unexpected instruction");
762
763 const Register DefReg = I.getOperand(i: 0).getReg();
764 LLT Ty = MRI.getType(Reg: DefReg);
765
766 if (RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
767 return false;
768
769 uint64_t Val = 0;
770 if (I.getOperand(i: 1).isCImm()) {
771 Val = I.getOperand(i: 1).getCImm()->getZExtValue();
772 I.getOperand(i: 1).ChangeToImmediate(ImmVal: Val);
773 } else if (I.getOperand(i: 1).isImm()) {
774 Val = I.getOperand(i: 1).getImm();
775 } else
776 llvm_unreachable("Unsupported operand type.");
777
778 unsigned NewOpc;
779 switch (Ty.getSizeInBits()) {
780 case 8:
781 NewOpc = X86::MOV8ri;
782 break;
783 case 16:
784 NewOpc = X86::MOV16ri;
785 break;
786 case 32:
787 NewOpc = X86::MOV32ri;
788 break;
789 case 64:
790 NewOpc = X86::getMOVriOpcode(/*Use64BitReg=*/true, Imm: Val);
791 break;
792 default:
793 llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
794 }
795
796 I.setDesc(TII.get(Opcode: NewOpc));
797 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
798 return true;
799}
800
801// Helper function for selectTruncOrPtrToInt and selectAnyext.
802// Returns true if DstRC lives on a floating register class and
803// SrcRC lives on a 128-bit vector class.
804static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
805 const TargetRegisterClass *SrcRC) {
806 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
807 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
808 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
809}
810
811bool X86InstructionSelector::selectTurnIntoCOPY(
812 MachineInstr &I, MachineRegisterInfo &MRI, const Register DstReg,
813 const TargetRegisterClass *DstRC, const Register SrcReg,
814 const TargetRegisterClass *SrcRC) const {
815
816 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
817 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
818 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
819 << " operand\n");
820 return false;
821 }
822 I.setDesc(TII.get(Opcode: X86::COPY));
823 return true;
824}
825
826bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
827 MachineRegisterInfo &MRI,
828 MachineFunction &MF) const {
829 assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
830 I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
831 "unexpected instruction");
832
833 const Register DstReg = I.getOperand(i: 0).getReg();
834 const Register SrcReg = I.getOperand(i: 1).getReg();
835
836 const LLT DstTy = MRI.getType(Reg: DstReg);
837 const LLT SrcTy = MRI.getType(Reg: SrcReg);
838
839 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
840 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
841
842 if (DstRB.getID() != SrcRB.getID()) {
843 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
844 << " input/output on different banks\n");
845 return false;
846 }
847
848 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
849 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
850
851 if (!DstRC || !SrcRC)
852 return false;
853
854 // If that's truncation of the value that lives on the vector class and goes
855 // into the floating class, just replace it with copy, as we are able to
856 // select it as a regular move.
857 if (canTurnIntoCOPY(DstRC, SrcRC))
858 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
859
860 if (DstRB.getID() != X86::GPRRegBankID)
861 return false;
862
863 unsigned SubIdx;
864 if (DstRC == SrcRC) {
865 // Nothing to be done
866 SubIdx = X86::NoSubRegister;
867 } else if (DstRC == &X86::GR32RegClass) {
868 SubIdx = X86::sub_32bit;
869 } else if (DstRC == &X86::GR16RegClass) {
870 SubIdx = X86::sub_16bit;
871 } else if (DstRC == &X86::GR8RegClass) {
872 SubIdx = X86::sub_8bit;
873 } else {
874 return false;
875 }
876
877 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
878
879 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
880 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
881 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
882 << "\n");
883 return false;
884 }
885
886 I.getOperand(i: 1).setSubReg(SubIdx);
887
888 I.setDesc(TII.get(Opcode: X86::COPY));
889 return true;
890}
891
892bool X86InstructionSelector::selectZext(MachineInstr &I,
893 MachineRegisterInfo &MRI,
894 MachineFunction &MF) const {
895 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
896
897 const Register DstReg = I.getOperand(i: 0).getReg();
898 const Register SrcReg = I.getOperand(i: 1).getReg();
899
900 const LLT DstTy = MRI.getType(Reg: DstReg);
901 const LLT SrcTy = MRI.getType(Reg: SrcReg);
902
903 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) &&
904 "8=>16 Zext is handled by tablegen");
905 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
906 "8=>32 Zext is handled by tablegen");
907 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
908 "16=>32 Zext is handled by tablegen");
909 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) &&
910 "8=>64 Zext is handled by tablegen");
911 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) &&
912 "16=>64 Zext is handled by tablegen");
913 assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) &&
914 "32=>64 Zext is handled by tablegen");
915
916 if (SrcTy != LLT::scalar(SizeInBits: 1))
917 return false;
918
919 unsigned AndOpc;
920 if (DstTy == LLT::scalar(SizeInBits: 8))
921 AndOpc = X86::AND8ri;
922 else if (DstTy == LLT::scalar(SizeInBits: 16))
923 AndOpc = X86::AND16ri;
924 else if (DstTy == LLT::scalar(SizeInBits: 32))
925 AndOpc = X86::AND32ri;
926 else if (DstTy == LLT::scalar(SizeInBits: 64))
927 AndOpc = X86::AND64ri32;
928 else
929 return false;
930
931 Register DefReg = SrcReg;
932 if (DstTy != LLT::scalar(SizeInBits: 8)) {
933 Register ImpDefReg =
934 MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
935 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
936 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImpDefReg);
937
938 DefReg = MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
939 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
940 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DefReg)
941 .addReg(RegNo: ImpDefReg)
942 .addReg(RegNo: SrcReg)
943 .addImm(Val: X86::sub_8bit);
944 }
945
946 MachineInstr &AndInst =
947 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AndOpc), DestReg: DstReg)
948 .addReg(RegNo: DefReg)
949 .addImm(Val: 1);
950
951 constrainSelectedInstRegOperands(I&: AndInst, TII, TRI, RBI);
952
953 I.eraseFromParent();
954 return true;
955}
956
957bool X86InstructionSelector::selectAnyext(MachineInstr &I,
958 MachineRegisterInfo &MRI,
959 MachineFunction &MF) const {
960 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
961
962 const Register DstReg = I.getOperand(i: 0).getReg();
963 const Register SrcReg = I.getOperand(i: 1).getReg();
964
965 const LLT DstTy = MRI.getType(Reg: DstReg);
966 const LLT SrcTy = MRI.getType(Reg: SrcReg);
967
968 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
969 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
970
971 assert(DstRB.getID() == SrcRB.getID() &&
972 "G_ANYEXT input/output on different banks\n");
973
974 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
975 "G_ANYEXT incorrect operand size");
976
977 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
978 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
979
980 // If that's ANY_EXT of the value that lives on the floating class and goes
981 // into the vector class, just replace it with copy, as we are able to select
982 // it as a regular move.
983 if (canTurnIntoCOPY(DstRC: SrcRC, SrcRC: DstRC))
984 return selectTurnIntoCOPY(I, MRI, DstReg: SrcReg, DstRC: SrcRC, SrcReg: DstReg, SrcRC: DstRC);
985
986 if (DstRB.getID() != X86::GPRRegBankID)
987 return false;
988
989 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
990 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
991 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
992 << " operand\n");
993 return false;
994 }
995
996 if (SrcRC == DstRC) {
997 I.setDesc(TII.get(Opcode: X86::COPY));
998 return true;
999 }
1000
1001 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1002 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
1003 .addDef(RegNo: DstReg)
1004 .addReg(RegNo: SrcReg)
1005 .addImm(Val: getSubRegIndex(RC: SrcRC));
1006
1007 I.eraseFromParent();
1008 return true;
1009}
1010
1011bool X86InstructionSelector::selectCmp(MachineInstr &I,
1012 MachineRegisterInfo &MRI,
1013 MachineFunction &MF) const {
1014 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
1015
1016 X86::CondCode CC;
1017 bool SwapArgs;
1018 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(
1019 Predicate: (CmpInst::Predicate)I.getOperand(i: 1).getPredicate());
1020
1021 Register LHS = I.getOperand(i: 2).getReg();
1022 Register RHS = I.getOperand(i: 3).getReg();
1023
1024 if (SwapArgs)
1025 std::swap(a&: LHS, b&: RHS);
1026
1027 unsigned OpCmp;
1028 LLT Ty = MRI.getType(Reg: LHS);
1029
1030 switch (Ty.getSizeInBits()) {
1031 default:
1032 return false;
1033 case 8:
1034 OpCmp = X86::CMP8rr;
1035 break;
1036 case 16:
1037 OpCmp = X86::CMP16rr;
1038 break;
1039 case 32:
1040 OpCmp = X86::CMP32rr;
1041 break;
1042 case 64:
1043 OpCmp = X86::CMP64rr;
1044 break;
1045 }
1046
1047 MachineInstr &CmpInst =
1048 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1049 .addReg(RegNo: LHS)
1050 .addReg(RegNo: RHS);
1051
1052 MachineInstr &SetInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1053 MCID: TII.get(Opcode: X86::SETCCr), DestReg: I.getOperand(i: 0).getReg()).addImm(Val: CC);
1054
1055 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1056 constrainSelectedInstRegOperands(I&: SetInst, TII, TRI, RBI);
1057
1058 I.eraseFromParent();
1059 return true;
1060}
1061
1062bool X86InstructionSelector::selectFCmp(MachineInstr &I,
1063 MachineRegisterInfo &MRI,
1064 MachineFunction &MF) const {
1065 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
1066
1067 Register LhsReg = I.getOperand(i: 2).getReg();
1068 Register RhsReg = I.getOperand(i: 3).getReg();
1069 CmpInst::Predicate Predicate =
1070 (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1071
1072 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1073 static const uint16_t SETFOpcTable[2][3] = {
1074 {X86::COND_E, X86::COND_NP, X86::AND8rr},
1075 {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1076 const uint16_t *SETFOpc = nullptr;
1077 switch (Predicate) {
1078 default:
1079 break;
1080 case CmpInst::FCMP_OEQ:
1081 SETFOpc = &SETFOpcTable[0][0];
1082 break;
1083 case CmpInst::FCMP_UNE:
1084 SETFOpc = &SETFOpcTable[1][0];
1085 break;
1086 }
1087
1088 assert((LhsReg.isVirtual() && RhsReg.isVirtual()) &&
1089 "Both arguments of FCMP need to be virtual!");
1090 auto *LhsBank = RBI.getRegBank(Reg: LhsReg, MRI, TRI);
1091 [[maybe_unused]] auto *RhsBank = RBI.getRegBank(Reg: RhsReg, MRI, TRI);
1092 assert((LhsBank == RhsBank) &&
1093 "Both banks assigned to FCMP arguments need to be same!");
1094
1095 bool IsX87 = LhsBank->getID() == X86::PSRRegBankID;
1096
1097 // Compute the opcode for the CMP instruction.
1098 unsigned OpCmp;
1099 LLT Ty = MRI.getType(Reg: LhsReg);
1100 switch (Ty.getSizeInBits()) {
1101 default:
1102 return false;
1103 case 32:
1104 OpCmp = IsX87 ? X86::UCOM_FpIr32 : X86::UCOMISSrr;
1105 break;
1106 case 64:
1107 OpCmp = IsX87 ? X86::UCOM_FpIr64 : X86::UCOMISDrr;
1108 break;
1109 case 80:
1110 OpCmp = X86::UCOM_FpIr80;
1111 break;
1112 }
1113
1114 Register ResultReg = I.getOperand(i: 0).getReg();
1115 RBI.constrainGenericRegister(
1116 Reg: ResultReg,
1117 RC: *getRegClass(Ty: LLT::scalar(SizeInBits: 8), RB: *RBI.getRegBank(Reg: ResultReg, MRI, TRI)), MRI);
1118 if (SETFOpc) {
1119 MachineInstr &CmpInst =
1120 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1121 .addReg(RegNo: LhsReg)
1122 .addReg(RegNo: RhsReg);
1123 if (IsX87)
1124 CmpInst.getOperand(i: 3).setIsDead(); // Mark unused FPSW def dead.
1125
1126 Register FlagReg1 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1127 Register FlagReg2 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1128 MachineInstr &Set1 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1129 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg1).addImm(Val: SETFOpc[0]);
1130 MachineInstr &Set2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1131 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg2).addImm(Val: SETFOpc[1]);
1132 MachineInstr &Set3 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1133 MCID: TII.get(Opcode: SETFOpc[2]), DestReg: ResultReg)
1134 .addReg(RegNo: FlagReg1)
1135 .addReg(RegNo: FlagReg2)
1136 .setOperandDead(3);
1137 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1138 constrainSelectedInstRegOperands(I&: Set1, TII, TRI, RBI);
1139 constrainSelectedInstRegOperands(I&: Set2, TII, TRI, RBI);
1140 constrainSelectedInstRegOperands(I&: Set3, TII, TRI, RBI);
1141
1142 I.eraseFromParent();
1143 return true;
1144 }
1145
1146 X86::CondCode CC;
1147 bool SwapArgs;
1148 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1149 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1150
1151 if (SwapArgs)
1152 std::swap(a&: LhsReg, b&: RhsReg);
1153
1154 // Emit a compare of LHS/RHS.
1155 MachineInstr &CmpInst =
1156 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1157 .addReg(RegNo: LhsReg)
1158 .addReg(RegNo: RhsReg);
1159 if (IsX87)
1160 CmpInst.getOperand(i: 3).setIsDead(); // Mark unused FPSW def dead.
1161
1162 MachineInstr &Set =
1163 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: ResultReg).addImm(Val: CC);
1164 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1165 constrainSelectedInstRegOperands(I&: Set, TII, TRI, RBI);
1166 I.eraseFromParent();
1167 return true;
1168}
1169
1170bool X86InstructionSelector::selectUAddSub(MachineInstr &I,
1171 MachineRegisterInfo &MRI,
1172 MachineFunction &MF) const {
1173 assert((I.getOpcode() == TargetOpcode::G_UADDE ||
1174 I.getOpcode() == TargetOpcode::G_UADDO ||
1175 I.getOpcode() == TargetOpcode::G_USUBE ||
1176 I.getOpcode() == TargetOpcode::G_USUBO) &&
1177 "unexpected instruction");
1178
1179 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
1180
1181 const Register DstReg = CarryMI.getDstReg();
1182 const Register CarryOutReg = CarryMI.getCarryOutReg();
1183 const Register Op0Reg = CarryMI.getLHSReg();
1184 const Register Op1Reg = CarryMI.getRHSReg();
1185 bool IsSub = CarryMI.isSub();
1186
1187 const LLT DstTy = MRI.getType(Reg: DstReg);
1188 assert(DstTy.isScalar() && "selectUAddSub only supported for scalar types");
1189
1190 // TODO: Handle immediate argument variants?
1191 unsigned OpADC, OpADD, OpSBB, OpSUB;
1192 switch (DstTy.getSizeInBits()) {
1193 case 8:
1194 OpADC = X86::ADC8rr;
1195 OpADD = X86::ADD8rr;
1196 OpSBB = X86::SBB8rr;
1197 OpSUB = X86::SUB8rr;
1198 break;
1199 case 16:
1200 OpADC = X86::ADC16rr;
1201 OpADD = X86::ADD16rr;
1202 OpSBB = X86::SBB16rr;
1203 OpSUB = X86::SUB16rr;
1204 break;
1205 case 32:
1206 OpADC = X86::ADC32rr;
1207 OpADD = X86::ADD32rr;
1208 OpSBB = X86::SBB32rr;
1209 OpSUB = X86::SUB32rr;
1210 break;
1211 case 64:
1212 OpADC = X86::ADC64rr;
1213 OpADD = X86::ADD64rr;
1214 OpSBB = X86::SBB64rr;
1215 OpSUB = X86::SUB64rr;
1216 break;
1217 default:
1218 llvm_unreachable("selectUAddSub unsupported type.");
1219 }
1220
1221 const RegisterBank &CarryRB = *RBI.getRegBank(Reg: CarryOutReg, MRI, TRI);
1222 const TargetRegisterClass *CarryRC =
1223 getRegClass(Ty: MRI.getType(Reg: CarryOutReg), RB: CarryRB);
1224
1225 unsigned Opcode = IsSub ? OpSUB : OpADD;
1226
1227 // G_UADDE/G_USUBE - find CarryIn def instruction.
1228 if (auto CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
1229 Register CarryInReg = CarryInMI->getCarryInReg();
1230 MachineInstr *Def = MRI.getVRegDef(Reg: CarryInReg);
1231 while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1232 CarryInReg = Def->getOperand(i: 1).getReg();
1233 Def = MRI.getVRegDef(Reg: CarryInReg);
1234 }
1235
1236 // TODO - handle more CF generating instructions
1237 if (Def->getOpcode() == TargetOpcode::G_UADDE ||
1238 Def->getOpcode() == TargetOpcode::G_UADDO ||
1239 Def->getOpcode() == TargetOpcode::G_USUBE ||
1240 Def->getOpcode() == TargetOpcode::G_USUBO) {
1241 // The carry-in is a SETB byte (0 or 1) from a chained add/sub.
1242 // Materialize EFLAGS.CF from that byte for the following ADC/SBB
1243 // by emitting NEG, which sets CF iff its operand is non-zero.
1244 if (!RBI.constrainGenericRegister(Reg: CarryInReg, RC: *CarryRC, MRI))
1245 return false;
1246
1247 Register NegDef = MRI.createVirtualRegister(RegClass: CarryRC);
1248 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::NEG8r), DestReg: NegDef)
1249 .addReg(RegNo: CarryInReg);
1250
1251 Opcode = IsSub ? OpSBB : OpADC;
1252 } else if (auto val = getIConstantVRegVal(VReg: CarryInReg, MRI)) {
1253 // carry is constant, support only 0.
1254 if (*val != 0)
1255 return false;
1256
1257 Opcode = IsSub ? OpSUB : OpADD;
1258 } else
1259 return false;
1260 }
1261
1262 MachineInstr &Inst =
1263 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1264 .addReg(RegNo: Op0Reg)
1265 .addReg(RegNo: Op1Reg);
1266
1267 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: CarryOutReg)
1268 .addImm(Val: X86::COND_B);
1269
1270 constrainSelectedInstRegOperands(I&: Inst, TII, TRI, RBI);
1271 if (!RBI.constrainGenericRegister(Reg: CarryOutReg, RC: *CarryRC, MRI))
1272 return false;
1273
1274 I.eraseFromParent();
1275 return true;
1276}
1277
1278bool X86InstructionSelector::selectExtract(MachineInstr &I,
1279 MachineRegisterInfo &MRI,
1280 MachineFunction &MF) const {
1281 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1282 "unexpected instruction");
1283
1284 const Register DstReg = I.getOperand(i: 0).getReg();
1285 const Register SrcReg = I.getOperand(i: 1).getReg();
1286 int64_t Index = I.getOperand(i: 2).getImm();
1287
1288 const LLT DstTy = MRI.getType(Reg: DstReg);
1289 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1290
1291 // Meanwile handle vector type only.
1292 if (!DstTy.isVector())
1293 return false;
1294
1295 if (Index % DstTy.getSizeInBits() != 0)
1296 return false; // Not extract subvector.
1297
1298 if (Index == 0) {
1299 // Replace by extract subreg copy.
1300 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1301 return false;
1302
1303 I.eraseFromParent();
1304 return true;
1305 }
1306
1307 bool HasAVX = STI.hasAVX();
1308 bool HasAVX512 = STI.hasAVX512();
1309 bool HasVLX = STI.hasVLX();
1310
1311 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1312 if (HasVLX)
1313 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Z256rri));
1314 else if (HasAVX)
1315 I.setDesc(TII.get(Opcode: X86::VEXTRACTF128rri));
1316 else
1317 return false;
1318 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1319 if (DstTy.getSizeInBits() == 128)
1320 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Zrri));
1321 else if (DstTy.getSizeInBits() == 256)
1322 I.setDesc(TII.get(Opcode: X86::VEXTRACTF64X4Zrri));
1323 else
1324 return false;
1325 } else
1326 return false;
1327
1328 // Convert to X86 VEXTRACT immediate.
1329 Index = Index / DstTy.getSizeInBits();
1330 I.getOperand(i: 2).setImm(Index);
1331
1332 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1333 return true;
1334}
1335
1336bool X86InstructionSelector::emitExtractSubreg(Register DstReg, Register SrcReg,
1337 MachineInstr &I,
1338 MachineRegisterInfo &MRI,
1339 MachineFunction &MF) const {
1340 const LLT DstTy = MRI.getType(Reg: DstReg);
1341 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1342 unsigned SubIdx = X86::NoSubRegister;
1343
1344 if (!DstTy.isVector() || !SrcTy.isVector())
1345 return false;
1346
1347 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1348 "Incorrect Src/Dst register size");
1349
1350 if (DstTy.getSizeInBits() == 128)
1351 SubIdx = X86::sub_xmm;
1352 else if (DstTy.getSizeInBits() == 256)
1353 SubIdx = X86::sub_ymm;
1354 else
1355 return false;
1356
1357 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1358 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1359
1360 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
1361
1362 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1363 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1364 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1365 return false;
1366 }
1367
1368 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::COPY), DestReg: DstReg)
1369 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubIdx);
1370
1371 return true;
1372}
1373
1374bool X86InstructionSelector::emitInsertSubreg(Register DstReg, Register SrcReg,
1375 MachineInstr &I,
1376 MachineRegisterInfo &MRI,
1377 MachineFunction &MF) const {
1378 const LLT DstTy = MRI.getType(Reg: DstReg);
1379 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1380 unsigned SubIdx = X86::NoSubRegister;
1381
1382 // TODO: support scalar types
1383 if (!DstTy.isVector() || !SrcTy.isVector())
1384 return false;
1385
1386 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1387 "Incorrect Src/Dst register size");
1388
1389 if (SrcTy.getSizeInBits() == 128)
1390 SubIdx = X86::sub_xmm;
1391 else if (SrcTy.getSizeInBits() == 256)
1392 SubIdx = X86::sub_ymm;
1393 else
1394 return false;
1395
1396 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1397 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1398
1399 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1400 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1401 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1402 return false;
1403 }
1404
1405 Register ImpDefReg = MRI.createVirtualRegister(RegClass: DstRC);
1406 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::IMPLICIT_DEF),
1407 DestReg: ImpDefReg);
1408
1409 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::INSERT_SUBREG),
1410 DestReg: DstReg)
1411 .addReg(RegNo: ImpDefReg)
1412 .addReg(RegNo: SrcReg)
1413 .addImm(Val: SubIdx);
1414
1415 return true;
1416}
1417
1418bool X86InstructionSelector::selectInsert(MachineInstr &I,
1419 MachineRegisterInfo &MRI,
1420 MachineFunction &MF) const {
1421 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1422
1423 const Register DstReg = I.getOperand(i: 0).getReg();
1424 const Register SrcReg = I.getOperand(i: 1).getReg();
1425 const Register InsertReg = I.getOperand(i: 2).getReg();
1426 int64_t Index = I.getOperand(i: 3).getImm();
1427
1428 const LLT DstTy = MRI.getType(Reg: DstReg);
1429 const LLT InsertRegTy = MRI.getType(Reg: InsertReg);
1430
1431 // Meanwile handle vector type only.
1432 if (!DstTy.isVector())
1433 return false;
1434
1435 if (Index % InsertRegTy.getSizeInBits() != 0)
1436 return false; // Not insert subvector.
1437
1438 if (Index == 0 && mi_match(R: SrcReg, MRI, P: m_GImplicitDef())) {
1439 // Replace by subreg copy.
1440 if (!emitInsertSubreg(DstReg, SrcReg: InsertReg, I, MRI, MF))
1441 return false;
1442
1443 I.eraseFromParent();
1444 return true;
1445 }
1446
1447 bool HasAVX = STI.hasAVX();
1448 bool HasAVX512 = STI.hasAVX512();
1449 bool HasVLX = STI.hasVLX();
1450
1451 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1452 if (HasVLX)
1453 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Z256rri));
1454 else if (HasAVX)
1455 I.setDesc(TII.get(Opcode: X86::VINSERTF128rri));
1456 else
1457 return false;
1458 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1459 if (InsertRegTy.getSizeInBits() == 128)
1460 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Zrri));
1461 else if (InsertRegTy.getSizeInBits() == 256)
1462 I.setDesc(TII.get(Opcode: X86::VINSERTF64X4Zrri));
1463 else
1464 return false;
1465 } else
1466 return false;
1467
1468 // Convert to X86 VINSERT immediate.
1469 Index = Index / InsertRegTy.getSizeInBits();
1470
1471 I.getOperand(i: 3).setImm(Index);
1472
1473 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1474 return true;
1475}
1476
1477bool X86InstructionSelector::selectUnmergeValues(
1478 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1479 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1480 "unexpected instruction");
1481
1482 // Split to extracts.
1483 unsigned NumDefs = I.getNumOperands() - 1;
1484 Register SrcReg = I.getOperand(i: NumDefs).getReg();
1485 unsigned DefSize = MRI.getType(Reg: I.getOperand(i: 0).getReg()).getSizeInBits();
1486
1487 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1488 MachineInstr &ExtrInst =
1489 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1490 MCID: TII.get(Opcode: TargetOpcode::G_EXTRACT), DestReg: I.getOperand(i: Idx).getReg())
1491 .addReg(RegNo: SrcReg)
1492 .addImm(Val: Idx * DefSize);
1493
1494 if (!select(I&: ExtrInst))
1495 return false;
1496 }
1497
1498 I.eraseFromParent();
1499 return true;
1500}
1501
1502bool X86InstructionSelector::selectMergeValues(
1503 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1504 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1505 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1506 "unexpected instruction");
1507
1508 // Split to inserts.
1509 Register DstReg = I.getOperand(i: 0).getReg();
1510 Register SrcReg0 = I.getOperand(i: 1).getReg();
1511
1512 const LLT DstTy = MRI.getType(Reg: DstReg);
1513 const LLT SrcTy = MRI.getType(Reg: SrcReg0);
1514 unsigned SrcSize = SrcTy.getSizeInBits();
1515
1516 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1517
1518 // For the first src use insertSubReg.
1519 Register DefReg = MRI.createGenericVirtualRegister(Ty: DstTy);
1520 MRI.setRegBank(Reg: DefReg, RegBank);
1521 if (!emitInsertSubreg(DstReg: DefReg, SrcReg: I.getOperand(i: 1).getReg(), I, MRI, MF))
1522 return false;
1523
1524 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1525 Register Tmp = MRI.createGenericVirtualRegister(Ty: DstTy);
1526 MRI.setRegBank(Reg: Tmp, RegBank);
1527
1528 MachineInstr &InsertInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1529 MCID: TII.get(Opcode: TargetOpcode::G_INSERT), DestReg: Tmp)
1530 .addReg(RegNo: DefReg)
1531 .addReg(RegNo: I.getOperand(i: Idx).getReg())
1532 .addImm(Val: (Idx - 1) * SrcSize);
1533
1534 DefReg = Tmp;
1535
1536 if (!select(I&: InsertInst))
1537 return false;
1538 }
1539
1540 MachineInstr &CopyInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1541 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
1542 .addReg(RegNo: DefReg);
1543
1544 if (!select(I&: CopyInst))
1545 return false;
1546
1547 I.eraseFromParent();
1548 return true;
1549}
1550
1551bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1552 MachineRegisterInfo &MRI,
1553 MachineFunction &MF) const {
1554 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1555
1556 const Register CondReg = I.getOperand(i: 0).getReg();
1557 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1558
1559 MachineInstr &TestInst =
1560 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST8ri))
1561 .addReg(RegNo: CondReg)
1562 .addImm(Val: 1);
1563 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::JCC_1))
1564 .addMBB(MBB: DestMBB).addImm(Val: X86::COND_NE);
1565
1566 constrainSelectedInstRegOperands(I&: TestInst, TII, TRI, RBI);
1567
1568 I.eraseFromParent();
1569 return true;
1570}
1571
1572bool X86InstructionSelector::materializeFP(MachineInstr &I,
1573 MachineRegisterInfo &MRI,
1574 MachineFunction &MF) const {
1575 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1576 "unexpected instruction");
1577
1578 // Can't handle alternate code models yet.
1579 CodeModel::Model CM = TM.getCodeModel();
1580 if (CM != CodeModel::Small && CM != CodeModel::Large)
1581 return false;
1582
1583 const Register DstReg = I.getOperand(i: 0).getReg();
1584 const LLT DstTy = MRI.getType(Reg: DstReg);
1585 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1586 // Create the load from the constant pool.
1587 const ConstantFP *CFP = I.getOperand(i: 1).getFPImm();
1588 const auto &DL = MF.getDataLayout();
1589 Align Alignment = DL.getPrefTypeAlign(Ty: CFP->getType());
1590 const DebugLoc &DbgLoc = I.getDebugLoc();
1591
1592 unsigned Opc =
1593 getLoadStoreOp(Ty: DstTy, RB: RegBank, Opc: TargetOpcode::G_LOAD, Alignment);
1594
1595 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(C: CFP, Alignment);
1596 MachineInstr *LoadInst = nullptr;
1597 unsigned char OpFlag = STI.classifyLocalReference(GV: nullptr);
1598
1599 if (CM == CodeModel::Large && STI.is64Bit()) {
1600 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1601 // they cannot be folded into immediate fields.
1602
1603 Register AddrReg = MRI.createVirtualRegister(RegClass: &X86::GR64RegClass);
1604 BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: AddrReg)
1605 .addConstantPoolIndex(Idx: CPI, Offset: 0, TargetFlags: OpFlag);
1606
1607 MachineMemOperand *MMO = MF.getMachineMemOperand(
1608 PtrInfo: MachinePointerInfo::getConstantPool(MF), F: MachineMemOperand::MOLoad,
1609 MemTy: LLT::pointer(AddressSpace: 0, SizeInBits: DL.getPointerSizeInBits()), BaseAlignment: Alignment);
1610
1611 LoadInst =
1612 addDirectMem(MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg),
1613 Reg: AddrReg)
1614 .addMemOperand(MMO);
1615
1616 } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1617 // Handle the case when globals fit in our immediate field.
1618 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1619
1620 // x86-32 PIC requires a PIC base register for constant pools.
1621 unsigned PICBase = 0;
1622 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1623 // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1624 // In DAGISEL the code that initialize it generated by the CGBR pass.
1625 return false; // TODO support the mode.
1626 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1627 PICBase = X86::RIP;
1628
1629 LoadInst = addConstantPoolReference(
1630 MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg), CPI, GlobalBaseReg: PICBase,
1631 OpFlags: OpFlag);
1632 } else
1633 return false;
1634
1635 constrainSelectedInstRegOperands(I&: *LoadInst, TII, TRI, RBI);
1636 I.eraseFromParent();
1637 return true;
1638}
1639
1640bool X86InstructionSelector::selectImplicitDefOrPHI(
1641 MachineInstr &I, MachineRegisterInfo &MRI) const {
1642 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1643 I.getOpcode() == TargetOpcode::G_PHI) &&
1644 "unexpected instruction");
1645
1646 Register DstReg = I.getOperand(i: 0).getReg();
1647
1648 if (!MRI.getRegClassOrNull(Reg: DstReg)) {
1649 const LLT DstTy = MRI.getType(Reg: DstReg);
1650 const TargetRegisterClass *RC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1651
1652 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI)) {
1653 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1654 << " operand\n");
1655 return false;
1656 }
1657 }
1658
1659 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1660 I.setDesc(TII.get(Opcode: X86::IMPLICIT_DEF));
1661 else
1662 I.setDesc(TII.get(Opcode: X86::PHI));
1663
1664 return true;
1665}
1666
1667bool X86InstructionSelector::selectMulDivRem(MachineInstr &I,
1668 MachineRegisterInfo &MRI,
1669 MachineFunction &MF) const {
1670 // The implementation of this function is adapted from X86FastISel.
1671 assert((I.getOpcode() == TargetOpcode::G_MUL ||
1672 I.getOpcode() == TargetOpcode::G_SMULH ||
1673 I.getOpcode() == TargetOpcode::G_UMULH ||
1674 I.getOpcode() == TargetOpcode::G_SDIV ||
1675 I.getOpcode() == TargetOpcode::G_SREM ||
1676 I.getOpcode() == TargetOpcode::G_UDIV ||
1677 I.getOpcode() == TargetOpcode::G_UREM) &&
1678 "unexpected instruction");
1679
1680 const Register DstReg = I.getOperand(i: 0).getReg();
1681 const Register Op1Reg = I.getOperand(i: 1).getReg();
1682 const Register Op2Reg = I.getOperand(i: 2).getReg();
1683
1684 const LLT RegTy = MRI.getType(Reg: DstReg);
1685 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1686 "Arguments and return value types must match");
1687
1688 const RegisterBank *RegRB = RBI.getRegBank(Reg: DstReg, MRI, TRI);
1689 if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1690 return false;
1691
1692 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1693 const static unsigned NumOps = 7; // SDiv/SRem/UDiv/URem/Mul/SMulH/UMulh
1694 const static bool S = true; // IsSigned
1695 const static bool U = false; // !IsSigned
1696 const static unsigned Copy = TargetOpcode::COPY;
1697
1698 // For the X86 IDIV instruction, in most cases the dividend
1699 // (numerator) must be in a specific register pair highreg:lowreg,
1700 // producing the quotient in lowreg and the remainder in highreg.
1701 // For most data types, to set up the instruction, the dividend is
1702 // copied into lowreg, and lowreg is sign-extended into highreg. The
1703 // exception is i8, where the dividend is defined as a single register rather
1704 // than a register pair, and we therefore directly sign-extend the dividend
1705 // into lowreg, instead of copying, and ignore the highreg.
1706 const static struct MulDivRemEntry {
1707 // The following portion depends only on the data type.
1708 unsigned SizeInBits;
1709 unsigned LowInReg; // low part of the register pair
1710 unsigned HighInReg; // high part of the register pair
1711 // The following portion depends on both the data type and the operation.
1712 struct MulDivRemResult {
1713 unsigned OpMulDivRem; // The specific MUL/DIV opcode to use.
1714 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1715 // highreg, or copying a zero into highreg.
1716 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1717 // zero/sign-extending into lowreg for i8.
1718 unsigned ResultReg; // Register containing the desired result.
1719 bool IsOpSigned; // Whether to use signed or unsigned form.
1720 } ResultTable[NumOps];
1721 } OpTable[NumTypes] = {
1722 {.SizeInBits: 8,
1723 .LowInReg: X86::AX,
1724 .HighInReg: 0,
1725 .ResultTable: {
1726 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // SDiv
1727 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SRem
1728 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AL, .IsOpSigned: U}, // UDiv
1729 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // URem
1730 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // Mul
1731 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SMulH
1732 {.OpMulDivRem: X86::MUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // UMulH
1733 }}, // i8
1734 {.SizeInBits: 16,
1735 .LowInReg: X86::AX,
1736 .HighInReg: X86::DX,
1737 .ResultTable: {
1738 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // SDiv
1739 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SRem
1740 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: U}, // UDiv
1741 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // URem
1742 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // Mul
1743 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SMulH
1744 {.OpMulDivRem: X86::MUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // UMulH
1745 }}, // i16
1746 {.SizeInBits: 32,
1747 .LowInReg: X86::EAX,
1748 .HighInReg: X86::EDX,
1749 .ResultTable: {
1750 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // SDiv
1751 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SRem
1752 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: U}, // UDiv
1753 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // URem
1754 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // Mul
1755 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SMulH
1756 {.OpMulDivRem: X86::MUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // UMulH
1757 }}, // i32
1758 {.SizeInBits: 64,
1759 .LowInReg: X86::RAX,
1760 .HighInReg: X86::RDX,
1761 .ResultTable: {
1762 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // SDiv
1763 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SRem
1764 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: U}, // UDiv
1765 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // URem
1766 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // Mul
1767 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SMulH
1768 {.OpMulDivRem: X86::MUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // UMulH
1769 }}, // i64
1770 };
1771
1772 auto OpEntryIt = llvm::find_if(Range: OpTable, P: [RegTy](const MulDivRemEntry &El) {
1773 return El.SizeInBits == RegTy.getSizeInBits();
1774 });
1775 if (OpEntryIt == std::end(arr: OpTable))
1776 return false;
1777
1778 unsigned OpIndex;
1779 switch (I.getOpcode()) {
1780 default:
1781 llvm_unreachable("Unexpected mul/div/rem opcode");
1782 case TargetOpcode::G_SDIV:
1783 OpIndex = 0;
1784 break;
1785 case TargetOpcode::G_SREM:
1786 OpIndex = 1;
1787 break;
1788 case TargetOpcode::G_UDIV:
1789 OpIndex = 2;
1790 break;
1791 case TargetOpcode::G_UREM:
1792 OpIndex = 3;
1793 break;
1794 case TargetOpcode::G_MUL:
1795 OpIndex = 4;
1796 break;
1797 case TargetOpcode::G_SMULH:
1798 OpIndex = 5;
1799 break;
1800 case TargetOpcode::G_UMULH:
1801 OpIndex = 6;
1802 break;
1803 }
1804
1805 const MulDivRemEntry &TypeEntry = *OpEntryIt;
1806 const MulDivRemEntry::MulDivRemResult &OpEntry =
1807 TypeEntry.ResultTable[OpIndex];
1808
1809 const TargetRegisterClass *RegRC = getRegClass(Ty: RegTy, RB: *RegRB);
1810 if (!RBI.constrainGenericRegister(Reg: Op1Reg, RC: *RegRC, MRI) ||
1811 !RBI.constrainGenericRegister(Reg: Op2Reg, RC: *RegRC, MRI) ||
1812 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RegRC, MRI)) {
1813 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1814 << " operand\n");
1815 return false;
1816 }
1817
1818 // Move op1 into low-order input register.
1819 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpCopy),
1820 DestReg: TypeEntry.LowInReg)
1821 .addReg(RegNo: Op1Reg);
1822
1823 // Zero-extend or sign-extend into high-order input register.
1824 if (OpEntry.OpSignExtend) {
1825 if (OpEntry.IsOpSigned)
1826 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1827 MCID: TII.get(Opcode: OpEntry.OpSignExtend));
1828 else {
1829 Register Zero32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
1830 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::MOV32r0), DestReg: Zero32)
1831 .setOperandDead(1);
1832
1833 // Copy the zero into the appropriate sub/super/identical physical
1834 // register. Unfortunately the operations needed are not uniform enough
1835 // to fit neatly into the table above.
1836 if (RegTy.getSizeInBits() == 16) {
1837 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1838 DestReg: TypeEntry.HighInReg)
1839 .addReg(RegNo: Zero32, Flags: {}, SubReg: X86::sub_16bit);
1840 } else if (RegTy.getSizeInBits() == 32) {
1841 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1842 DestReg: TypeEntry.HighInReg)
1843 .addReg(RegNo: Zero32);
1844 } else if (RegTy.getSizeInBits() == 64) {
1845 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1846 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: TypeEntry.HighInReg)
1847 .addReg(RegNo: Zero32)
1848 .addImm(Val: X86::sub_32bit);
1849 }
1850 }
1851 }
1852
1853 // Generate the DIV/IDIV/MUL/IMUL instruction.
1854 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpMulDivRem))
1855 .addReg(RegNo: Op2Reg);
1856
1857 // For i8 remainder, we can't reference ah directly, as we'll end
1858 // up with bogus copies like %r9b = COPY %ah. Reference ax
1859 // instead to prevent ah references in a rex instruction.
1860 //
1861 // The current assumption of the fast register allocator is that isel
1862 // won't generate explicit references to the GR8_NOREX registers. If
1863 // the allocator and/or the backend get enhanced to be more robust in
1864 // that regard, this can be, and should be, removed.
1865 if (OpEntry.ResultReg == X86::AH && STI.is64Bit()) {
1866 Register SourceSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1867 Register ResultSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1868 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy), DestReg: SourceSuperReg)
1869 .addReg(RegNo: X86::AX);
1870
1871 // Shift AX right by 8 bits instead of using AH.
1872 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SHR16ri),
1873 DestReg: ResultSuperReg)
1874 .addReg(RegNo: SourceSuperReg)
1875 .addImm(Val: 8);
1876
1877 // Now reference the 8-bit subreg of the result.
1878 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1879 DestReg: DstReg)
1880 .addReg(RegNo: ResultSuperReg, Flags: {}, SubReg: X86::sub_8bit);
1881 } else {
1882 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1883 DestReg: DstReg)
1884 .addReg(RegNo: OpEntry.ResultReg);
1885 }
1886 I.eraseFromParent();
1887
1888 return true;
1889}
1890
1891bool X86InstructionSelector::selectSelect(MachineInstr &I,
1892 MachineRegisterInfo &MRI,
1893 MachineFunction &MF) const {
1894 GSelect &Sel = cast<GSelect>(Val&: I);
1895 Register DstReg = Sel.getReg(Idx: 0);
1896 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST32rr))
1897 .addReg(RegNo: Sel.getCondReg())
1898 .addReg(RegNo: Sel.getCondReg());
1899
1900 unsigned OpCmp;
1901 LLT Ty = MRI.getType(Reg: DstReg);
1902 if (Ty.getSizeInBits() == 80) {
1903 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::CMOVE_Fp80),
1904 DestReg: DstReg)
1905 .addReg(RegNo: Sel.getTrueReg())
1906 .addReg(RegNo: Sel.getFalseReg());
1907 } else {
1908 switch (Ty.getSizeInBits()) {
1909 default:
1910 return false;
1911 case 8:
1912 OpCmp = X86::CMOV_GR8;
1913 break;
1914 case 16:
1915 OpCmp = STI.canUseCMOV() ? X86::CMOV16rr : X86::CMOV_GR16;
1916 break;
1917 case 32:
1918 OpCmp = STI.canUseCMOV() ? X86::CMOV32rr : X86::CMOV_GR32;
1919 break;
1920 case 64:
1921 assert(STI.is64Bit() && STI.canUseCMOV());
1922 OpCmp = X86::CMOV64rr;
1923 break;
1924 }
1925 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: OpCmp), DestReg: DstReg)
1926 .addReg(RegNo: Sel.getTrueReg())
1927 .addReg(RegNo: Sel.getFalseReg())
1928 .addImm(Val: X86::COND_E);
1929 }
1930 const TargetRegisterClass *DstRC = getRegClass(Ty, Reg: DstReg, MRI);
1931 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1932 LLVM_DEBUG(dbgs() << "Failed to constrain CMOV\n");
1933 return false;
1934 }
1935
1936 Sel.eraseFromParent();
1937 return true;
1938}
1939
1940InstructionSelector::ComplexRendererFns
1941X86InstructionSelector::selectAddr(MachineOperand &Root) const {
1942 MachineInstr *MI = Root.getParent();
1943
1944 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
1945 MachineInstr *Ptr = MRI.getVRegDef(Reg: Root.getReg());
1946 X86AddressMode AM;
1947 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
1948 return std::nullopt;
1949
1950 if (AM.IndexReg)
1951 return std::nullopt;
1952
1953 return {// Base
1954 {[=](MachineInstrBuilder &MIB) {
1955 if (AM.BaseType == X86AddressMode::RegBase)
1956 MIB.addUse(RegNo: AM.Base.Reg);
1957 else {
1958 assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
1959 "Unknown type of address base");
1960 MIB.addFrameIndex(Idx: AM.Base.FrameIndex);
1961 }
1962 },
1963 // Scale
1964 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: AM.Scale); },
1965 // Index
1966 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); },
1967 // Disp
1968 [=](MachineInstrBuilder &MIB) {
1969 if (AM.GV)
1970 MIB.addGlobalAddress(GV: AM.GV, Offset: AM.Disp, TargetFlags: AM.GVOpFlags);
1971 else if (AM.CP)
1972 MIB.addConstantPoolIndex(Idx: AM.Disp, Offset: 0, TargetFlags: AM.GVOpFlags);
1973 else
1974 MIB.addImm(Val: AM.Disp);
1975 },
1976 // Segment
1977 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); }}};
1978}
1979
1980InstructionSelector *
1981llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1982 const X86Subtarget &Subtarget,
1983 const X86RegisterBankInfo &RBI) {
1984 return new X86InstructionSelector(TM, Subtarget, RBI);
1985}
1986