1//===- X86InstructionSelector.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// X86.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86BaseInfo.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86RegisterBankInfo.h"
19#include "X86RegisterInfo.h"
20#include "X86Subtarget.h"
21#include "X86TargetMachine.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
25#include "llvm/CodeGen/GlobalISel/Utils.h"
26#include "llvm/CodeGen/MachineBasicBlock.h"
27#include "llvm/CodeGen/MachineConstantPool.h"
28#include "llvm/CodeGen/MachineFunction.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineInstrBuilder.h"
31#include "llvm/CodeGen/MachineMemOperand.h"
32#include "llvm/CodeGen/MachineOperand.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/RegisterBank.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/CodeGen/TargetRegisterInfo.h"
37#include "llvm/CodeGenTypes/LowLevelType.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/InstrTypes.h"
40#include "llvm/IR/IntrinsicsX86.h"
41#include "llvm/Support/CodeGen.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include <cassert>
47#include <cstdint>
48#include <tuple>
49
50#define DEBUG_TYPE "X86-isel"
51
52using namespace llvm;
53
54namespace {
55
56#define GET_GLOBALISEL_PREDICATE_BITSET
57#include "X86GenGlobalISel.inc"
58#undef GET_GLOBALISEL_PREDICATE_BITSET
59
60class X86InstructionSelector : public InstructionSelector {
61public:
62 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
63 const X86RegisterBankInfo &RBI);
64
65 bool select(MachineInstr &I) override;
66 static const char *getName() { return DEBUG_TYPE; }
67
68private:
69 /// tblgen-erated 'select' implementation, used as the initial selector for
70 /// the patterns that don't require complex C++.
71 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
72
73 // TODO: remove after supported by Tablegen-erated instruction selection.
74 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
75 Align Alignment) const;
76 // TODO: remove once p0<->i32/i64 matching is available
77 unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
78 unsigned Opc) const;
79
80 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
81 MachineFunction &MF) const;
82 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
83 MachineFunction &MF) const;
84 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
85 MachineFunction &MF) const;
86 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
87 MachineFunction &MF) const;
88 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
89 MachineFunction &MF) const;
90 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
91 MachineFunction &MF) const;
92 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
93 MachineFunction &MF) const;
94 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
95 MachineFunction &MF) const;
96 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
97 MachineFunction &MF) const;
98 bool selectUAddSub(MachineInstr &I, MachineRegisterInfo &MRI,
99 MachineFunction &MF) const;
100 bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI) const;
101 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
102 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
103 MachineFunction &MF);
104 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
105 MachineFunction &MF);
106 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
107 MachineFunction &MF) const;
108 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
109 MachineFunction &MF) const;
110 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
111 MachineFunction &MF) const;
112 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
113 const Register DstReg,
114 const TargetRegisterClass *DstRC,
115 const Register SrcReg,
116 const TargetRegisterClass *SrcRC) const;
117 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
118 MachineFunction &MF) const;
119 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
120 bool selectMulDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
121 MachineFunction &MF) const;
122 bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
123 MachineFunction &MF) const;
124
125 ComplexRendererFns selectAddr(MachineOperand &Root) const;
126
127 // emit insert subreg instruction and insert it before MachineInstr &I
128 bool emitInsertSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
129 MachineRegisterInfo &MRI, MachineFunction &MF) const;
130 // emit extract subreg instruction and insert it before MachineInstr &I
131 bool emitExtractSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
132 MachineRegisterInfo &MRI, MachineFunction &MF) const;
133
134 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
135 const TargetRegisterClass *getRegClass(LLT Ty, Register Reg,
136 MachineRegisterInfo &MRI) const;
137
138 const X86TargetMachine &TM;
139 const X86Subtarget &STI;
140 const X86InstrInfo &TII;
141 const X86RegisterInfo &TRI;
142 const X86RegisterBankInfo &RBI;
143
144#define GET_GLOBALISEL_PREDICATES_DECL
145#include "X86GenGlobalISel.inc"
146#undef GET_GLOBALISEL_PREDICATES_DECL
147
148#define GET_GLOBALISEL_TEMPORARIES_DECL
149#include "X86GenGlobalISel.inc"
150#undef GET_GLOBALISEL_TEMPORARIES_DECL
151};
152
153} // end anonymous namespace
154
155#define GET_GLOBALISEL_IMPL
156#include "X86GenGlobalISel.inc"
157#undef GET_GLOBALISEL_IMPL
158
159X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
160 const X86Subtarget &STI,
161 const X86RegisterBankInfo &RBI)
162 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
163 RBI(RBI),
164#define GET_GLOBALISEL_PREDICATES_INIT
165#include "X86GenGlobalISel.inc"
166#undef GET_GLOBALISEL_PREDICATES_INIT
167#define GET_GLOBALISEL_TEMPORARIES_INIT
168#include "X86GenGlobalISel.inc"
169#undef GET_GLOBALISEL_TEMPORARIES_INIT
170{
171}
172
173// FIXME: This should be target-independent, inferred from the types declared
174// for each class in the bank.
175const TargetRegisterClass *
176X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
177 if (RB.getID() == X86::GPRRegBankID) {
178 if (Ty.getSizeInBits() <= 8)
179 return &X86::GR8RegClass;
180 if (Ty.getSizeInBits() == 16)
181 return &X86::GR16RegClass;
182 if (Ty.getSizeInBits() == 32)
183 return &X86::GR32RegClass;
184 if (Ty.getSizeInBits() == 64)
185 return &X86::GR64RegClass;
186 }
187 if (RB.getID() == X86::VECRRegBankID) {
188 if (Ty.getSizeInBits() == 16)
189 return STI.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
190 if (Ty.getSizeInBits() == 32)
191 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
192 if (Ty.getSizeInBits() == 64)
193 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
194 if (Ty.getSizeInBits() == 128)
195 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
196 if (Ty.getSizeInBits() == 256)
197 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
198 if (Ty.getSizeInBits() == 512)
199 return &X86::VR512RegClass;
200 }
201
202 if (RB.getID() == X86::PSRRegBankID) {
203 if (Ty.getSizeInBits() == 80)
204 return &X86::RFP80RegClass;
205 if (Ty.getSizeInBits() == 64)
206 return &X86::RFP64RegClass;
207 if (Ty.getSizeInBits() == 32)
208 return &X86::RFP32RegClass;
209 }
210
211 llvm_unreachable("Unknown RegBank!");
212}
213
214const TargetRegisterClass *
215X86InstructionSelector::getRegClass(LLT Ty, Register Reg,
216 MachineRegisterInfo &MRI) const {
217 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
218 return getRegClass(Ty, RB: RegBank);
219}
220
221static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
222 unsigned SubIdx = X86::NoSubRegister;
223 if (RC == &X86::GR32RegClass) {
224 SubIdx = X86::sub_32bit;
225 } else if (RC == &X86::GR16RegClass) {
226 SubIdx = X86::sub_16bit;
227 } else if (RC == &X86::GR8RegClass) {
228 SubIdx = X86::sub_8bit;
229 }
230
231 return SubIdx;
232}
233
234static const TargetRegisterClass *getRegClassFromGRPhysReg(Register Reg) {
235 assert(Reg.isPhysical());
236 if (X86::GR64RegClass.contains(Reg))
237 return &X86::GR64RegClass;
238 if (X86::GR32RegClass.contains(Reg))
239 return &X86::GR32RegClass;
240 if (X86::GR16RegClass.contains(Reg))
241 return &X86::GR16RegClass;
242 if (X86::GR8RegClass.contains(Reg))
243 return &X86::GR8RegClass;
244
245 llvm_unreachable("Unknown RegClass for PhysReg!");
246}
247
248// FIXME: We need some sort of API in RBI/TRI to allow generic code to
249// constrain operands of simple instructions given a TargetRegisterClass
250// and LLT
251bool X86InstructionSelector::selectDebugInstr(MachineInstr &I,
252 MachineRegisterInfo &MRI) const {
253 for (MachineOperand &MO : I.operands()) {
254 if (!MO.isReg())
255 continue;
256 Register Reg = MO.getReg();
257 if (!Reg)
258 continue;
259 if (Reg.isPhysical())
260 continue;
261 LLT Ty = MRI.getType(Reg);
262 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
263 const TargetRegisterClass *RC =
264 dyn_cast_if_present<const TargetRegisterClass *>(Val: RegClassOrBank);
265 if (!RC) {
266 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
267 RC = getRegClass(Ty, RB);
268 if (!RC) {
269 LLVM_DEBUG(
270 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
271 break;
272 }
273 }
274 RBI.constrainGenericRegister(Reg, RC: *RC, MRI);
275 }
276
277 return true;
278}
279
280// Set X86 Opcode and constrain DestReg.
281bool X86InstructionSelector::selectCopy(MachineInstr &I,
282 MachineRegisterInfo &MRI) const {
283 Register DstReg = I.getOperand(i: 0).getReg();
284 const unsigned DstSize = RBI.getSizeInBits(Reg: DstReg, MRI, TRI);
285 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
286
287 Register SrcReg = I.getOperand(i: 1).getReg();
288 const unsigned SrcSize = RBI.getSizeInBits(Reg: SrcReg, MRI, TRI);
289 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
290
291 if (DstReg.isPhysical()) {
292 assert(I.isCopy() && "Generic operators do not allow physical registers");
293
294 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
295 DstRegBank.getID() == X86::GPRRegBankID) {
296
297 const TargetRegisterClass *SrcRC =
298 getRegClass(Ty: MRI.getType(Reg: SrcReg), RB: SrcRegBank);
299 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(Reg: DstReg);
300
301 if (SrcRC != DstRC) {
302 // This case can be generated by ABI lowering, performe anyext
303 Register ExtSrc = MRI.createVirtualRegister(RegClass: DstRC);
304 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
305 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
306 .addDef(RegNo: ExtSrc)
307 .addReg(RegNo: SrcReg)
308 .addImm(Val: getSubRegIndex(RC: SrcRC));
309
310 I.getOperand(i: 1).setReg(ExtSrc);
311 }
312 }
313
314 // Special case GPR16 -> XMM
315 if (SrcSize == 16 && SrcRegBank.getID() == X86::GPRRegBankID &&
316 (DstRegBank.getID() == X86::VECRRegBankID)) {
317
318 const DebugLoc &DL = I.getDebugLoc();
319
320 // Any extend GPR16 -> GPR32
321 Register ExtReg = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
322 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG),
323 DestReg: ExtReg)
324 .addReg(RegNo: SrcReg)
325 .addImm(Val: X86::sub_16bit);
326
327 // Copy GR32 -> XMM
328 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
329 .addReg(RegNo: ExtReg);
330
331 I.eraseFromParent();
332 }
333
334 // Special case XMM -> GR16
335 if (DstSize == 16 && DstRegBank.getID() == X86::GPRRegBankID &&
336 (SrcRegBank.getID() == X86::VECRRegBankID)) {
337
338 const DebugLoc &DL = I.getDebugLoc();
339
340 // Move XMM to GR32 register.
341 Register Temp32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
342 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Temp32)
343 .addReg(RegNo: SrcReg);
344
345 // Extract the lower 16 bits
346 if (Register Dst32 = TRI.getMatchingSuperReg(Reg: DstReg, SubIdx: X86::sub_16bit,
347 RC: &X86::GR32RegClass)) {
348 // Optimization for Physical Dst (e.g. AX): Copy to EAX directly.
349 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst32)
350 .addReg(RegNo: Temp32);
351 } else {
352 // Handle if there is no super.
353 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
354 .addReg(RegNo: Temp32, Flags: {}, SubReg: X86::sub_16bit);
355 }
356
357 I.eraseFromParent();
358 }
359
360 return true;
361 }
362
363 assert((!SrcReg.isPhysical() || I.isCopy()) &&
364 "No phys reg on generic operators");
365 assert((DstSize == SrcSize ||
366 // Copies are a mean to setup initial types, the number of
367 // bits may not exactly match.
368 (SrcReg.isPhysical() &&
369 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
370 "Copy with different width?!");
371
372 const TargetRegisterClass *DstRC =
373 getRegClass(Ty: MRI.getType(Reg: DstReg), RB: DstRegBank);
374
375 if (SrcRegBank.getID() == X86::GPRRegBankID &&
376 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
377 SrcReg.isPhysical()) {
378 // Change the physical register to performe truncate.
379
380 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(Reg: SrcReg);
381
382 if (DstRC != SrcRC) {
383 I.getOperand(i: 1).setSubReg(getSubRegIndex(RC: DstRC));
384 I.getOperand(i: 1).substPhysReg(Reg: SrcReg, TRI);
385 }
386 }
387
388 // No need to constrain SrcReg. It will get constrained when
389 // we hit another of its use or its defs.
390 // Copies do not have constraints.
391 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(Reg: DstReg);
392 if (!OldRC || !DstRC->hasSubClassEq(RC: OldRC)) {
393 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
394 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
395 << " operand\n");
396 return false;
397 }
398 }
399 I.setDesc(TII.get(Opcode: X86::COPY));
400 return true;
401}
402
403bool X86InstructionSelector::select(MachineInstr &I) {
404 assert(I.getParent() && "Instruction should be in a basic block!");
405 assert(I.getParent()->getParent() && "Instruction should be in a function!");
406
407 MachineBasicBlock &MBB = *I.getParent();
408 MachineFunction &MF = *MBB.getParent();
409 MachineRegisterInfo &MRI = MF.getRegInfo();
410
411 unsigned Opcode = I.getOpcode();
412 if (!isPreISelGenericOpcode(Opcode) && !I.isPreISelOpcode()) {
413 // Certain non-generic instructions also need some special handling.
414
415 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
416 return false;
417
418 if (I.isCopy())
419 return selectCopy(I, MRI);
420
421 if (I.isDebugInstr())
422 return selectDebugInstr(I, MRI);
423
424 return true;
425 }
426
427 assert(I.getNumOperands() == I.getNumExplicitOperands() &&
428 "Generic instruction has unexpected implicit operands\n");
429
430 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
431 return true;
432
433 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
434
435 // TODO: This should be implemented by tblgen.
436 switch (I.getOpcode()) {
437 default:
438 return false;
439 case TargetOpcode::G_STORE:
440 case TargetOpcode::G_LOAD:
441 return selectLoadStoreOp(I, MRI, MF);
442 case TargetOpcode::G_PTR_ADD:
443 case TargetOpcode::G_FRAME_INDEX:
444 return selectFrameIndexOrGep(I, MRI, MF);
445 case TargetOpcode::G_GLOBAL_VALUE:
446 return selectGlobalValue(I, MRI, MF);
447 case TargetOpcode::G_CONSTANT:
448 return selectConstant(I, MRI, MF);
449 case TargetOpcode::G_FCONSTANT:
450 return materializeFP(I, MRI, MF);
451 case TargetOpcode::G_PTRTOINT:
452 case TargetOpcode::G_TRUNC:
453 return selectTruncOrPtrToInt(I, MRI, MF);
454 case TargetOpcode::G_INTTOPTR:
455 case TargetOpcode::G_FREEZE:
456 return selectCopy(I, MRI);
457 case TargetOpcode::G_ZEXT:
458 return selectZext(I, MRI, MF);
459 case TargetOpcode::G_ANYEXT:
460 return selectAnyext(I, MRI, MF);
461 case TargetOpcode::G_ICMP:
462 return selectCmp(I, MRI, MF);
463 case TargetOpcode::G_FCMP:
464 return selectFCmp(I, MRI, MF);
465 case TargetOpcode::G_UADDE:
466 case TargetOpcode::G_UADDO:
467 case TargetOpcode::G_USUBE:
468 case TargetOpcode::G_USUBO:
469 return selectUAddSub(I, MRI, MF);
470 case TargetOpcode::G_UNMERGE_VALUES:
471 return selectUnmergeValues(I, MRI, MF);
472 case TargetOpcode::G_MERGE_VALUES:
473 case TargetOpcode::G_CONCAT_VECTORS:
474 return selectMergeValues(I, MRI, MF);
475 case TargetOpcode::G_EXTRACT:
476 return selectExtract(I, MRI, MF);
477 case TargetOpcode::G_INSERT:
478 return selectInsert(I, MRI, MF);
479 case TargetOpcode::G_BRCOND:
480 return selectCondBranch(I, MRI, MF);
481 case TargetOpcode::G_IMPLICIT_DEF:
482 case TargetOpcode::G_PHI:
483 return selectImplicitDefOrPHI(I, MRI);
484 case TargetOpcode::G_MUL:
485 case TargetOpcode::G_SMULH:
486 case TargetOpcode::G_UMULH:
487 case TargetOpcode::G_SDIV:
488 case TargetOpcode::G_UDIV:
489 case TargetOpcode::G_SREM:
490 case TargetOpcode::G_UREM:
491 return selectMulDivRem(I, MRI, MF);
492 case TargetOpcode::G_SELECT:
493 return selectSelect(I, MRI, MF);
494 }
495
496 return false;
497}
498
499unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
500 const RegisterBank &RB,
501 unsigned Opc) const {
502 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
503 "Only G_STORE and G_LOAD are expected for selection");
504 if (Ty.isPointer() && X86::GPRRegBankID == RB.getID()) {
505 bool IsLoad = (Opc == TargetOpcode::G_LOAD);
506 switch (Ty.getSizeInBits()) {
507 default:
508 break;
509 case 32:
510 return IsLoad ? X86::MOV32rm : X86::MOV32mr;
511 case 64:
512 return IsLoad ? X86::MOV64rm : X86::MOV64mr;
513 }
514 }
515 return Opc;
516}
517
518unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
519 const RegisterBank &RB,
520 unsigned Opc,
521 Align Alignment) const {
522 bool Isload = (Opc == TargetOpcode::G_LOAD);
523 bool HasAVX = STI.hasAVX();
524 bool HasAVX512 = STI.hasAVX512();
525 bool HasVLX = STI.hasVLX();
526
527 if (Ty == LLT::scalar(SizeInBits: 8)) {
528 if (X86::GPRRegBankID == RB.getID())
529 return Isload ? X86::MOV8rm : X86::MOV8mr;
530 } else if (Ty == LLT::scalar(SizeInBits: 16)) {
531 if (X86::GPRRegBankID == RB.getID())
532 return Isload ? X86::MOV16rm : X86::MOV16mr;
533 } else if (Ty == LLT::scalar(SizeInBits: 32)) {
534 if (X86::GPRRegBankID == RB.getID())
535 return Isload ? X86::MOV32rm : X86::MOV32mr;
536 if (X86::VECRRegBankID == RB.getID())
537 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
538 HasAVX ? X86::VMOVSSrm_alt :
539 X86::MOVSSrm_alt)
540 : (HasAVX512 ? X86::VMOVSSZmr :
541 HasAVX ? X86::VMOVSSmr :
542 X86::MOVSSmr);
543 if (X86::PSRRegBankID == RB.getID())
544 return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
545 } else if (Ty == LLT::scalar(SizeInBits: 64)) {
546 if (X86::GPRRegBankID == RB.getID())
547 return Isload ? X86::MOV64rm : X86::MOV64mr;
548 if (X86::VECRRegBankID == RB.getID())
549 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
550 HasAVX ? X86::VMOVSDrm_alt :
551 X86::MOVSDrm_alt)
552 : (HasAVX512 ? X86::VMOVSDZmr :
553 HasAVX ? X86::VMOVSDmr :
554 X86::MOVSDmr);
555 if (X86::PSRRegBankID == RB.getID())
556 return Isload ? X86::LD_Fp64m : X86::ST_Fp64m;
557 } else if (Ty == LLT::scalar(SizeInBits: 80)) {
558 return Isload ? X86::LD_Fp80m : X86::ST_FpP80m;
559 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
560 if (Alignment >= Align(16))
561 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
562 : HasAVX512
563 ? X86::VMOVAPSZ128rm_NOVLX
564 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
565 : (HasVLX ? X86::VMOVAPSZ128mr
566 : HasAVX512
567 ? X86::VMOVAPSZ128mr_NOVLX
568 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
569 else
570 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
571 : HasAVX512
572 ? X86::VMOVUPSZ128rm_NOVLX
573 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
574 : (HasVLX ? X86::VMOVUPSZ128mr
575 : HasAVX512
576 ? X86::VMOVUPSZ128mr_NOVLX
577 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
578 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
579 if (Alignment >= Align(32))
580 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
581 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
582 : X86::VMOVAPSYrm)
583 : (HasVLX ? X86::VMOVAPSZ256mr
584 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
585 : X86::VMOVAPSYmr);
586 else
587 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
588 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
589 : X86::VMOVUPSYrm)
590 : (HasVLX ? X86::VMOVUPSZ256mr
591 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
592 : X86::VMOVUPSYmr);
593 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
594 if (Alignment >= Align(64))
595 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
596 else
597 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
598 }
599 return Opc;
600}
601
602// Fill in an address from the given instruction.
603static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
604 const MachineRegisterInfo &MRI,
605 const X86Subtarget &STI, X86AddressMode &AM) {
606 assert(I.getOperand(0).isReg() && "unsupported operand.");
607 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
608 "unsupported type.");
609
610 switch (I.getOpcode()) {
611 default:
612 break;
613 case TargetOpcode::G_FRAME_INDEX:
614 AM.Base.FrameIndex = I.getOperand(i: 1).getIndex();
615 AM.BaseType = X86AddressMode::FrameIndexBase;
616 return true;
617 case TargetOpcode::G_PTR_ADD: {
618 if (auto COff = getIConstantVRegSExtVal(VReg: I.getOperand(i: 2).getReg(), MRI)) {
619 int64_t Imm = *COff;
620 if (isInt<32>(x: Imm)) { // Check for displacement overflow.
621 AM.Disp = static_cast<int32_t>(Imm);
622 AM.Base.Reg = I.getOperand(i: 1).getReg();
623 return true;
624 }
625 }
626 break;
627 }
628 case TargetOpcode::G_GLOBAL_VALUE:
629 case X86::G_WRAPPER_RIP: {
630 auto GV = I.getOperand(i: 1).getGlobal();
631 if (GV->isThreadLocal()) {
632 return false; // TODO: we don't support TLS yet.
633 }
634 // Can't handle alternate code models yet.
635 if (TM.getCodeModel() != CodeModel::Small)
636 return false;
637 AM.GV = GV;
638 AM.GVOpFlags = STI.classifyGlobalReference(GV);
639
640 // TODO: This reference is relative to the pic base. not supported yet.
641 if (isGlobalRelativeToPICBase(TargetFlag: AM.GVOpFlags))
642 return false;
643
644 if (STI.isPICStyleRIPRel() || AM.GVOpFlags == X86II::MO_GOTPCREL ||
645 AM.GVOpFlags == X86II::MO_GOTPCREL_NORELAX) {
646 // Use rip-relative addressing.
647 assert(AM.Base.Reg == 0 && AM.IndexReg == 0 &&
648 "RIP-relative addresses can't have additional register operands");
649 AM.Base.Reg = X86::RIP;
650 }
651 return true;
652 }
653 case TargetOpcode::G_CONSTANT_POOL: {
654 // TODO: Need a separate move for Large model
655 if (TM.getCodeModel() == CodeModel::Large)
656 return false;
657
658 AM.GVOpFlags = STI.classifyLocalReference(GV: nullptr);
659 if (AM.GVOpFlags == X86II::MO_GOTOFF)
660 AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(MF: I.getMF());
661 else if (STI.is64Bit())
662 AM.Base.Reg = X86::RIP;
663 AM.CP = true;
664 AM.Disp = I.getOperand(i: 1).getIndex();
665 return true;
666 }
667 }
668 // Default behavior.
669 AM.Base.Reg = I.getOperand(i: 0).getReg();
670 return true;
671}
672
673bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
674 MachineRegisterInfo &MRI,
675 MachineFunction &MF) const {
676 unsigned Opc = I.getOpcode();
677
678 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
679 "Only G_STORE and G_LOAD are expected for selection");
680
681 const Register DefReg = I.getOperand(i: 0).getReg();
682 LLT Ty = MRI.getType(Reg: DefReg);
683 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
684
685 assert(I.hasOneMemOperand());
686 auto &MemOp = **I.memoperands_begin();
687 if (MemOp.isAtomic()) {
688 // Note: for unordered operations, we rely on the fact the appropriate MMO
689 // is already on the instruction we're mutating, and thus we don't need to
690 // make any changes. So long as we select an opcode which is capable of
691 // loading or storing the appropriate size atomically, the rest of the
692 // backend is required to respect the MMO state.
693 if (!MemOp.isUnordered()) {
694 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
695 return false;
696 }
697 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
698 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
699 return false;
700 }
701 }
702
703 unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
704 if (NewOpc == Opc)
705 return false;
706
707 I.setDesc(TII.get(Opcode: NewOpc));
708 MachineInstrBuilder MIB(MF, I);
709 MachineInstr *Ptr = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
710
711 X86AddressMode AM;
712 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
713 return false;
714
715 if (Opc == TargetOpcode::G_LOAD) {
716 I.removeOperand(OpNo: 1);
717 addFullAddress(MIB, AM);
718 } else {
719 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
720 I.removeOperand(OpNo: 1);
721 I.removeOperand(OpNo: 0);
722 addFullAddress(MIB, AM).addUse(RegNo: DefReg);
723 }
724 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
725 I.addImplicitDefUseOperands(MF);
726 return true;
727}
728
729static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
730 if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 64))
731 return X86::LEA64r;
732 else if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 32))
733 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
734 else
735 llvm_unreachable("Can't get LEA opcode. Unsupported type.");
736}
737
738bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
739 MachineRegisterInfo &MRI,
740 MachineFunction &MF) const {
741 unsigned Opc = I.getOpcode();
742
743 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
744 "unexpected instruction");
745
746 const Register DefReg = I.getOperand(i: 0).getReg();
747 LLT Ty = MRI.getType(Reg: DefReg);
748
749 // Use LEA to calculate frame index and GEP
750 unsigned NewOpc = getLeaOP(Ty, STI);
751 I.setDesc(TII.get(Opcode: NewOpc));
752 MachineInstrBuilder MIB(MF, I);
753
754 if (Opc == TargetOpcode::G_FRAME_INDEX) {
755 addOffset(MIB, Offset: 0);
756 } else {
757 MachineOperand &InxOp = I.getOperand(i: 2);
758 I.addOperand(Op: InxOp); // set IndexReg
759 InxOp.ChangeToImmediate(ImmVal: 1); // set Scale
760 MIB.addImm(Val: 0).addReg(RegNo: 0);
761 }
762
763 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
764 return true;
765}
766
767bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
768 MachineRegisterInfo &MRI,
769 MachineFunction &MF) const {
770 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
771 "unexpected instruction");
772
773 X86AddressMode AM;
774 if (!X86SelectAddress(I, TM, MRI, STI, AM))
775 return false;
776
777 const Register DefReg = I.getOperand(i: 0).getReg();
778 LLT Ty = MRI.getType(Reg: DefReg);
779 unsigned NewOpc = getLeaOP(Ty, STI);
780
781 I.setDesc(TII.get(Opcode: NewOpc));
782 MachineInstrBuilder MIB(MF, I);
783
784 I.removeOperand(OpNo: 1);
785 addFullAddress(MIB, AM);
786
787 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
788 return true;
789}
790
791bool X86InstructionSelector::selectConstant(MachineInstr &I,
792 MachineRegisterInfo &MRI,
793 MachineFunction &MF) const {
794 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
795 "unexpected instruction");
796
797 const Register DefReg = I.getOperand(i: 0).getReg();
798 LLT Ty = MRI.getType(Reg: DefReg);
799
800 if (RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
801 return false;
802
803 uint64_t Val = 0;
804 if (I.getOperand(i: 1).isCImm()) {
805 Val = I.getOperand(i: 1).getCImm()->getZExtValue();
806 I.getOperand(i: 1).ChangeToImmediate(ImmVal: Val);
807 } else if (I.getOperand(i: 1).isImm()) {
808 Val = I.getOperand(i: 1).getImm();
809 } else
810 llvm_unreachable("Unsupported operand type.");
811
812 unsigned NewOpc;
813 switch (Ty.getSizeInBits()) {
814 case 8:
815 NewOpc = X86::MOV8ri;
816 break;
817 case 16:
818 NewOpc = X86::MOV16ri;
819 break;
820 case 32:
821 NewOpc = X86::MOV32ri;
822 break;
823 case 64:
824 NewOpc = X86::getMOVriOpcode(/*Use64BitReg=*/true, Imm: Val);
825 break;
826 default:
827 llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
828 }
829
830 I.setDesc(TII.get(Opcode: NewOpc));
831 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
832 return true;
833}
834
835// Helper function for selectTruncOrPtrToInt and selectAnyext.
836// Returns true if DstRC lives on a floating register class and
837// SrcRC lives on a 128-bit vector class.
838static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
839 const TargetRegisterClass *SrcRC) {
840 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
841 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
842 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
843}
844
845bool X86InstructionSelector::selectTurnIntoCOPY(
846 MachineInstr &I, MachineRegisterInfo &MRI, const Register DstReg,
847 const TargetRegisterClass *DstRC, const Register SrcReg,
848 const TargetRegisterClass *SrcRC) const {
849
850 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
851 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
852 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
853 << " operand\n");
854 return false;
855 }
856 I.setDesc(TII.get(Opcode: X86::COPY));
857 return true;
858}
859
860bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
861 MachineRegisterInfo &MRI,
862 MachineFunction &MF) const {
863 assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
864 I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
865 "unexpected instruction");
866
867 const Register DstReg = I.getOperand(i: 0).getReg();
868 const Register SrcReg = I.getOperand(i: 1).getReg();
869
870 const LLT DstTy = MRI.getType(Reg: DstReg);
871 const LLT SrcTy = MRI.getType(Reg: SrcReg);
872
873 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
874 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
875
876 if (DstRB.getID() != SrcRB.getID()) {
877 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
878 << " input/output on different banks\n");
879 return false;
880 }
881
882 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
883 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
884
885 if (!DstRC || !SrcRC)
886 return false;
887
888 // If that's truncation of the value that lives on the vector class and goes
889 // into the floating class, just replace it with copy, as we are able to
890 // select it as a regular move.
891 if (canTurnIntoCOPY(DstRC, SrcRC))
892 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
893
894 if (DstRB.getID() != X86::GPRRegBankID)
895 return false;
896
897 unsigned SubIdx;
898 if (DstRC == SrcRC) {
899 // Nothing to be done
900 SubIdx = X86::NoSubRegister;
901 } else if (DstRC == &X86::GR32RegClass) {
902 SubIdx = X86::sub_32bit;
903 } else if (DstRC == &X86::GR16RegClass) {
904 SubIdx = X86::sub_16bit;
905 } else if (DstRC == &X86::GR8RegClass) {
906 SubIdx = X86::sub_8bit;
907 } else {
908 return false;
909 }
910
911 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
912
913 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
914 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
915 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
916 << "\n");
917 return false;
918 }
919
920 I.getOperand(i: 1).setSubReg(SubIdx);
921
922 I.setDesc(TII.get(Opcode: X86::COPY));
923 return true;
924}
925
926bool X86InstructionSelector::selectZext(MachineInstr &I,
927 MachineRegisterInfo &MRI,
928 MachineFunction &MF) const {
929 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
930
931 const Register DstReg = I.getOperand(i: 0).getReg();
932 const Register SrcReg = I.getOperand(i: 1).getReg();
933
934 const LLT DstTy = MRI.getType(Reg: DstReg);
935 const LLT SrcTy = MRI.getType(Reg: SrcReg);
936
937 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) &&
938 "8=>16 Zext is handled by tablegen");
939 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
940 "8=>32 Zext is handled by tablegen");
941 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
942 "16=>32 Zext is handled by tablegen");
943 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) &&
944 "8=>64 Zext is handled by tablegen");
945 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) &&
946 "16=>64 Zext is handled by tablegen");
947 assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) &&
948 "32=>64 Zext is handled by tablegen");
949
950 if (SrcTy != LLT::scalar(SizeInBits: 1))
951 return false;
952
953 unsigned AndOpc;
954 if (DstTy == LLT::scalar(SizeInBits: 8))
955 AndOpc = X86::AND8ri;
956 else if (DstTy == LLT::scalar(SizeInBits: 16))
957 AndOpc = X86::AND16ri;
958 else if (DstTy == LLT::scalar(SizeInBits: 32))
959 AndOpc = X86::AND32ri;
960 else if (DstTy == LLT::scalar(SizeInBits: 64))
961 AndOpc = X86::AND64ri32;
962 else
963 return false;
964
965 Register DefReg = SrcReg;
966 if (DstTy != LLT::scalar(SizeInBits: 8)) {
967 Register ImpDefReg =
968 MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
969 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
970 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImpDefReg);
971
972 DefReg = MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
973 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
974 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DefReg)
975 .addReg(RegNo: ImpDefReg)
976 .addReg(RegNo: SrcReg)
977 .addImm(Val: X86::sub_8bit);
978 }
979
980 MachineInstr &AndInst =
981 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AndOpc), DestReg: DstReg)
982 .addReg(RegNo: DefReg)
983 .addImm(Val: 1);
984
985 constrainSelectedInstRegOperands(I&: AndInst, TII, TRI, RBI);
986
987 I.eraseFromParent();
988 return true;
989}
990
991bool X86InstructionSelector::selectAnyext(MachineInstr &I,
992 MachineRegisterInfo &MRI,
993 MachineFunction &MF) const {
994 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
995
996 const Register DstReg = I.getOperand(i: 0).getReg();
997 const Register SrcReg = I.getOperand(i: 1).getReg();
998
999 const LLT DstTy = MRI.getType(Reg: DstReg);
1000 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1001
1002 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1003 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
1004
1005 assert(DstRB.getID() == SrcRB.getID() &&
1006 "G_ANYEXT input/output on different banks\n");
1007
1008 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
1009 "G_ANYEXT incorrect operand size");
1010
1011 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
1012 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
1013
1014 // If that's ANY_EXT of the value that lives on the floating class and goes
1015 // into the vector class, just replace it with copy, as we are able to select
1016 // it as a regular move.
1017 if (canTurnIntoCOPY(DstRC: SrcRC, SrcRC: DstRC))
1018 return selectTurnIntoCOPY(I, MRI, DstReg: SrcReg, DstRC: SrcRC, SrcReg: DstReg, SrcRC: DstRC);
1019
1020 if (DstRB.getID() != X86::GPRRegBankID)
1021 return false;
1022
1023 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1024 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1025 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1026 << " operand\n");
1027 return false;
1028 }
1029
1030 if (SrcRC == DstRC) {
1031 I.setDesc(TII.get(Opcode: X86::COPY));
1032 return true;
1033 }
1034
1035 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1036 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
1037 .addDef(RegNo: DstReg)
1038 .addReg(RegNo: SrcReg)
1039 .addImm(Val: getSubRegIndex(RC: SrcRC));
1040
1041 I.eraseFromParent();
1042 return true;
1043}
1044
1045bool X86InstructionSelector::selectCmp(MachineInstr &I,
1046 MachineRegisterInfo &MRI,
1047 MachineFunction &MF) const {
1048 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
1049
1050 X86::CondCode CC;
1051 bool SwapArgs;
1052 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(
1053 Predicate: (CmpInst::Predicate)I.getOperand(i: 1).getPredicate());
1054
1055 Register LHS = I.getOperand(i: 2).getReg();
1056 Register RHS = I.getOperand(i: 3).getReg();
1057
1058 if (SwapArgs)
1059 std::swap(a&: LHS, b&: RHS);
1060
1061 unsigned OpCmp;
1062 LLT Ty = MRI.getType(Reg: LHS);
1063
1064 switch (Ty.getSizeInBits()) {
1065 default:
1066 return false;
1067 case 8:
1068 OpCmp = X86::CMP8rr;
1069 break;
1070 case 16:
1071 OpCmp = X86::CMP16rr;
1072 break;
1073 case 32:
1074 OpCmp = X86::CMP32rr;
1075 break;
1076 case 64:
1077 OpCmp = X86::CMP64rr;
1078 break;
1079 }
1080
1081 MachineInstr &CmpInst =
1082 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1083 .addReg(RegNo: LHS)
1084 .addReg(RegNo: RHS);
1085
1086 MachineInstr &SetInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1087 MCID: TII.get(Opcode: X86::SETCCr), DestReg: I.getOperand(i: 0).getReg()).addImm(Val: CC);
1088
1089 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1090 constrainSelectedInstRegOperands(I&: SetInst, TII, TRI, RBI);
1091
1092 I.eraseFromParent();
1093 return true;
1094}
1095
1096bool X86InstructionSelector::selectFCmp(MachineInstr &I,
1097 MachineRegisterInfo &MRI,
1098 MachineFunction &MF) const {
1099 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
1100
1101 Register LhsReg = I.getOperand(i: 2).getReg();
1102 Register RhsReg = I.getOperand(i: 3).getReg();
1103 CmpInst::Predicate Predicate =
1104 (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1105
1106 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1107 static const uint16_t SETFOpcTable[2][3] = {
1108 {X86::COND_E, X86::COND_NP, X86::AND8rr},
1109 {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1110 const uint16_t *SETFOpc = nullptr;
1111 switch (Predicate) {
1112 default:
1113 break;
1114 case CmpInst::FCMP_OEQ:
1115 SETFOpc = &SETFOpcTable[0][0];
1116 break;
1117 case CmpInst::FCMP_UNE:
1118 SETFOpc = &SETFOpcTable[1][0];
1119 break;
1120 }
1121
1122 assert((LhsReg.isVirtual() && RhsReg.isVirtual()) &&
1123 "Both arguments of FCMP need to be virtual!");
1124 auto *LhsBank = RBI.getRegBank(Reg: LhsReg, MRI, TRI);
1125 [[maybe_unused]] auto *RhsBank = RBI.getRegBank(Reg: RhsReg, MRI, TRI);
1126 assert((LhsBank == RhsBank) &&
1127 "Both banks assigned to FCMP arguments need to be same!");
1128
1129 // Compute the opcode for the CMP instruction.
1130 unsigned OpCmp;
1131 LLT Ty = MRI.getType(Reg: LhsReg);
1132 switch (Ty.getSizeInBits()) {
1133 default:
1134 return false;
1135 case 32:
1136 OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr32
1137 : X86::UCOMISSrr;
1138 break;
1139 case 64:
1140 OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr64
1141 : X86::UCOMISDrr;
1142 break;
1143 case 80:
1144 OpCmp = X86::UCOM_FpIr80;
1145 break;
1146 }
1147
1148 Register ResultReg = I.getOperand(i: 0).getReg();
1149 RBI.constrainGenericRegister(
1150 Reg: ResultReg,
1151 RC: *getRegClass(Ty: LLT::scalar(SizeInBits: 8), RB: *RBI.getRegBank(Reg: ResultReg, MRI, TRI)), MRI);
1152 if (SETFOpc) {
1153 MachineInstr &CmpInst =
1154 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1155 .addReg(RegNo: LhsReg)
1156 .addReg(RegNo: RhsReg);
1157
1158 Register FlagReg1 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1159 Register FlagReg2 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1160 MachineInstr &Set1 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1161 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg1).addImm(Val: SETFOpc[0]);
1162 MachineInstr &Set2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1163 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg2).addImm(Val: SETFOpc[1]);
1164 MachineInstr &Set3 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1165 MCID: TII.get(Opcode: SETFOpc[2]), DestReg: ResultReg)
1166 .addReg(RegNo: FlagReg1)
1167 .addReg(RegNo: FlagReg2);
1168 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1169 constrainSelectedInstRegOperands(I&: Set1, TII, TRI, RBI);
1170 constrainSelectedInstRegOperands(I&: Set2, TII, TRI, RBI);
1171 constrainSelectedInstRegOperands(I&: Set3, TII, TRI, RBI);
1172
1173 I.eraseFromParent();
1174 return true;
1175 }
1176
1177 X86::CondCode CC;
1178 bool SwapArgs;
1179 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1180 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1181
1182 if (SwapArgs)
1183 std::swap(a&: LhsReg, b&: RhsReg);
1184
1185 // Emit a compare of LHS/RHS.
1186 MachineInstr &CmpInst =
1187 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1188 .addReg(RegNo: LhsReg)
1189 .addReg(RegNo: RhsReg);
1190
1191 MachineInstr &Set =
1192 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: ResultReg).addImm(Val: CC);
1193 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1194 constrainSelectedInstRegOperands(I&: Set, TII, TRI, RBI);
1195 I.eraseFromParent();
1196 return true;
1197}
1198
1199bool X86InstructionSelector::selectUAddSub(MachineInstr &I,
1200 MachineRegisterInfo &MRI,
1201 MachineFunction &MF) const {
1202 assert((I.getOpcode() == TargetOpcode::G_UADDE ||
1203 I.getOpcode() == TargetOpcode::G_UADDO ||
1204 I.getOpcode() == TargetOpcode::G_USUBE ||
1205 I.getOpcode() == TargetOpcode::G_USUBO) &&
1206 "unexpected instruction");
1207
1208 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
1209
1210 const Register DstReg = CarryMI.getDstReg();
1211 const Register CarryOutReg = CarryMI.getCarryOutReg();
1212 const Register Op0Reg = CarryMI.getLHSReg();
1213 const Register Op1Reg = CarryMI.getRHSReg();
1214 bool IsSub = CarryMI.isSub();
1215
1216 const LLT DstTy = MRI.getType(Reg: DstReg);
1217 assert(DstTy.isScalar() && "selectUAddSub only supported for scalar types");
1218
1219 // TODO: Handle immediate argument variants?
1220 unsigned OpADC, OpADD, OpSBB, OpSUB;
1221 switch (DstTy.getSizeInBits()) {
1222 case 8:
1223 OpADC = X86::ADC8rr;
1224 OpADD = X86::ADD8rr;
1225 OpSBB = X86::SBB8rr;
1226 OpSUB = X86::SUB8rr;
1227 break;
1228 case 16:
1229 OpADC = X86::ADC16rr;
1230 OpADD = X86::ADD16rr;
1231 OpSBB = X86::SBB16rr;
1232 OpSUB = X86::SUB16rr;
1233 break;
1234 case 32:
1235 OpADC = X86::ADC32rr;
1236 OpADD = X86::ADD32rr;
1237 OpSBB = X86::SBB32rr;
1238 OpSUB = X86::SUB32rr;
1239 break;
1240 case 64:
1241 OpADC = X86::ADC64rr;
1242 OpADD = X86::ADD64rr;
1243 OpSBB = X86::SBB64rr;
1244 OpSUB = X86::SUB64rr;
1245 break;
1246 default:
1247 llvm_unreachable("selectUAddSub unsupported type.");
1248 }
1249
1250 const RegisterBank &CarryRB = *RBI.getRegBank(Reg: CarryOutReg, MRI, TRI);
1251 const TargetRegisterClass *CarryRC =
1252 getRegClass(Ty: MRI.getType(Reg: CarryOutReg), RB: CarryRB);
1253
1254 unsigned Opcode = IsSub ? OpSUB : OpADD;
1255
1256 // G_UADDE/G_USUBE - find CarryIn def instruction.
1257 if (auto CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
1258 Register CarryInReg = CarryInMI->getCarryInReg();
1259 MachineInstr *Def = MRI.getVRegDef(Reg: CarryInReg);
1260 while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1261 CarryInReg = Def->getOperand(i: 1).getReg();
1262 Def = MRI.getVRegDef(Reg: CarryInReg);
1263 }
1264
1265 // TODO - handle more CF generating instructions
1266 if (Def->getOpcode() == TargetOpcode::G_UADDE ||
1267 Def->getOpcode() == TargetOpcode::G_UADDO ||
1268 Def->getOpcode() == TargetOpcode::G_USUBE ||
1269 Def->getOpcode() == TargetOpcode::G_USUBO) {
1270 // The carry-in is a SETB byte (0 or 1) from a chained add/sub.
1271 // Materialize EFLAGS.CF from that byte for the following ADC/SBB
1272 // by emitting NEG, which sets CF iff its operand is non-zero.
1273 if (!RBI.constrainGenericRegister(Reg: CarryInReg, RC: *CarryRC, MRI))
1274 return false;
1275
1276 Register NegDef = MRI.createVirtualRegister(RegClass: CarryRC);
1277 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::NEG8r), DestReg: NegDef)
1278 .addReg(RegNo: CarryInReg);
1279
1280 Opcode = IsSub ? OpSBB : OpADC;
1281 } else if (auto val = getIConstantVRegVal(VReg: CarryInReg, MRI)) {
1282 // carry is constant, support only 0.
1283 if (*val != 0)
1284 return false;
1285
1286 Opcode = IsSub ? OpSUB : OpADD;
1287 } else
1288 return false;
1289 }
1290
1291 MachineInstr &Inst =
1292 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1293 .addReg(RegNo: Op0Reg)
1294 .addReg(RegNo: Op1Reg);
1295
1296 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: CarryOutReg)
1297 .addImm(Val: X86::COND_B);
1298
1299 constrainSelectedInstRegOperands(I&: Inst, TII, TRI, RBI);
1300 if (!RBI.constrainGenericRegister(Reg: CarryOutReg, RC: *CarryRC, MRI))
1301 return false;
1302
1303 I.eraseFromParent();
1304 return true;
1305}
1306
1307bool X86InstructionSelector::selectExtract(MachineInstr &I,
1308 MachineRegisterInfo &MRI,
1309 MachineFunction &MF) const {
1310 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1311 "unexpected instruction");
1312
1313 const Register DstReg = I.getOperand(i: 0).getReg();
1314 const Register SrcReg = I.getOperand(i: 1).getReg();
1315 int64_t Index = I.getOperand(i: 2).getImm();
1316
1317 const LLT DstTy = MRI.getType(Reg: DstReg);
1318 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1319
1320 // Meanwile handle vector type only.
1321 if (!DstTy.isVector())
1322 return false;
1323
1324 if (Index % DstTy.getSizeInBits() != 0)
1325 return false; // Not extract subvector.
1326
1327 if (Index == 0) {
1328 // Replace by extract subreg copy.
1329 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1330 return false;
1331
1332 I.eraseFromParent();
1333 return true;
1334 }
1335
1336 bool HasAVX = STI.hasAVX();
1337 bool HasAVX512 = STI.hasAVX512();
1338 bool HasVLX = STI.hasVLX();
1339
1340 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1341 if (HasVLX)
1342 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Z256rri));
1343 else if (HasAVX)
1344 I.setDesc(TII.get(Opcode: X86::VEXTRACTF128rri));
1345 else
1346 return false;
1347 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1348 if (DstTy.getSizeInBits() == 128)
1349 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Zrri));
1350 else if (DstTy.getSizeInBits() == 256)
1351 I.setDesc(TII.get(Opcode: X86::VEXTRACTF64X4Zrri));
1352 else
1353 return false;
1354 } else
1355 return false;
1356
1357 // Convert to X86 VEXTRACT immediate.
1358 Index = Index / DstTy.getSizeInBits();
1359 I.getOperand(i: 2).setImm(Index);
1360
1361 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1362 return true;
1363}
1364
1365bool X86InstructionSelector::emitExtractSubreg(Register DstReg, Register SrcReg,
1366 MachineInstr &I,
1367 MachineRegisterInfo &MRI,
1368 MachineFunction &MF) const {
1369 const LLT DstTy = MRI.getType(Reg: DstReg);
1370 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1371 unsigned SubIdx = X86::NoSubRegister;
1372
1373 if (!DstTy.isVector() || !SrcTy.isVector())
1374 return false;
1375
1376 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1377 "Incorrect Src/Dst register size");
1378
1379 if (DstTy.getSizeInBits() == 128)
1380 SubIdx = X86::sub_xmm;
1381 else if (DstTy.getSizeInBits() == 256)
1382 SubIdx = X86::sub_ymm;
1383 else
1384 return false;
1385
1386 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1387 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1388
1389 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
1390
1391 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1392 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1393 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1394 return false;
1395 }
1396
1397 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::COPY), DestReg: DstReg)
1398 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubIdx);
1399
1400 return true;
1401}
1402
1403bool X86InstructionSelector::emitInsertSubreg(Register DstReg, Register SrcReg,
1404 MachineInstr &I,
1405 MachineRegisterInfo &MRI,
1406 MachineFunction &MF) const {
1407 const LLT DstTy = MRI.getType(Reg: DstReg);
1408 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1409 unsigned SubIdx = X86::NoSubRegister;
1410
1411 // TODO: support scalar types
1412 if (!DstTy.isVector() || !SrcTy.isVector())
1413 return false;
1414
1415 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1416 "Incorrect Src/Dst register size");
1417
1418 if (SrcTy.getSizeInBits() == 128)
1419 SubIdx = X86::sub_xmm;
1420 else if (SrcTy.getSizeInBits() == 256)
1421 SubIdx = X86::sub_ymm;
1422 else
1423 return false;
1424
1425 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1426 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1427
1428 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1429 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1430 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1431 return false;
1432 }
1433
1434 Register ImpDefReg = MRI.createVirtualRegister(RegClass: DstRC);
1435 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::IMPLICIT_DEF),
1436 DestReg: ImpDefReg);
1437
1438 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::INSERT_SUBREG),
1439 DestReg: DstReg)
1440 .addReg(RegNo: ImpDefReg)
1441 .addReg(RegNo: SrcReg)
1442 .addImm(Val: SubIdx);
1443
1444 return true;
1445}
1446
1447bool X86InstructionSelector::selectInsert(MachineInstr &I,
1448 MachineRegisterInfo &MRI,
1449 MachineFunction &MF) const {
1450 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1451
1452 const Register DstReg = I.getOperand(i: 0).getReg();
1453 const Register SrcReg = I.getOperand(i: 1).getReg();
1454 const Register InsertReg = I.getOperand(i: 2).getReg();
1455 int64_t Index = I.getOperand(i: 3).getImm();
1456
1457 const LLT DstTy = MRI.getType(Reg: DstReg);
1458 const LLT InsertRegTy = MRI.getType(Reg: InsertReg);
1459
1460 // Meanwile handle vector type only.
1461 if (!DstTy.isVector())
1462 return false;
1463
1464 if (Index % InsertRegTy.getSizeInBits() != 0)
1465 return false; // Not insert subvector.
1466
1467 if (Index == 0 && MRI.getVRegDef(Reg: SrcReg)->isImplicitDef()) {
1468 // Replace by subreg copy.
1469 if (!emitInsertSubreg(DstReg, SrcReg: InsertReg, I, MRI, MF))
1470 return false;
1471
1472 I.eraseFromParent();
1473 return true;
1474 }
1475
1476 bool HasAVX = STI.hasAVX();
1477 bool HasAVX512 = STI.hasAVX512();
1478 bool HasVLX = STI.hasVLX();
1479
1480 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1481 if (HasVLX)
1482 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Z256rri));
1483 else if (HasAVX)
1484 I.setDesc(TII.get(Opcode: X86::VINSERTF128rri));
1485 else
1486 return false;
1487 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1488 if (InsertRegTy.getSizeInBits() == 128)
1489 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Zrri));
1490 else if (InsertRegTy.getSizeInBits() == 256)
1491 I.setDesc(TII.get(Opcode: X86::VINSERTF64X4Zrri));
1492 else
1493 return false;
1494 } else
1495 return false;
1496
1497 // Convert to X86 VINSERT immediate.
1498 Index = Index / InsertRegTy.getSizeInBits();
1499
1500 I.getOperand(i: 3).setImm(Index);
1501
1502 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1503 return true;
1504}
1505
1506bool X86InstructionSelector::selectUnmergeValues(
1507 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1508 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1509 "unexpected instruction");
1510
1511 // Split to extracts.
1512 unsigned NumDefs = I.getNumOperands() - 1;
1513 Register SrcReg = I.getOperand(i: NumDefs).getReg();
1514 unsigned DefSize = MRI.getType(Reg: I.getOperand(i: 0).getReg()).getSizeInBits();
1515
1516 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1517 MachineInstr &ExtrInst =
1518 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1519 MCID: TII.get(Opcode: TargetOpcode::G_EXTRACT), DestReg: I.getOperand(i: Idx).getReg())
1520 .addReg(RegNo: SrcReg)
1521 .addImm(Val: Idx * DefSize);
1522
1523 if (!select(I&: ExtrInst))
1524 return false;
1525 }
1526
1527 I.eraseFromParent();
1528 return true;
1529}
1530
1531bool X86InstructionSelector::selectMergeValues(
1532 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1533 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1534 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1535 "unexpected instruction");
1536
1537 // Split to inserts.
1538 Register DstReg = I.getOperand(i: 0).getReg();
1539 Register SrcReg0 = I.getOperand(i: 1).getReg();
1540
1541 const LLT DstTy = MRI.getType(Reg: DstReg);
1542 const LLT SrcTy = MRI.getType(Reg: SrcReg0);
1543 unsigned SrcSize = SrcTy.getSizeInBits();
1544
1545 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1546
1547 // For the first src use insertSubReg.
1548 Register DefReg = MRI.createGenericVirtualRegister(Ty: DstTy);
1549 MRI.setRegBank(Reg: DefReg, RegBank);
1550 if (!emitInsertSubreg(DstReg: DefReg, SrcReg: I.getOperand(i: 1).getReg(), I, MRI, MF))
1551 return false;
1552
1553 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1554 Register Tmp = MRI.createGenericVirtualRegister(Ty: DstTy);
1555 MRI.setRegBank(Reg: Tmp, RegBank);
1556
1557 MachineInstr &InsertInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1558 MCID: TII.get(Opcode: TargetOpcode::G_INSERT), DestReg: Tmp)
1559 .addReg(RegNo: DefReg)
1560 .addReg(RegNo: I.getOperand(i: Idx).getReg())
1561 .addImm(Val: (Idx - 1) * SrcSize);
1562
1563 DefReg = Tmp;
1564
1565 if (!select(I&: InsertInst))
1566 return false;
1567 }
1568
1569 MachineInstr &CopyInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1570 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
1571 .addReg(RegNo: DefReg);
1572
1573 if (!select(I&: CopyInst))
1574 return false;
1575
1576 I.eraseFromParent();
1577 return true;
1578}
1579
1580bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1581 MachineRegisterInfo &MRI,
1582 MachineFunction &MF) const {
1583 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1584
1585 const Register CondReg = I.getOperand(i: 0).getReg();
1586 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1587
1588 MachineInstr &TestInst =
1589 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST8ri))
1590 .addReg(RegNo: CondReg)
1591 .addImm(Val: 1);
1592 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::JCC_1))
1593 .addMBB(MBB: DestMBB).addImm(Val: X86::COND_NE);
1594
1595 constrainSelectedInstRegOperands(I&: TestInst, TII, TRI, RBI);
1596
1597 I.eraseFromParent();
1598 return true;
1599}
1600
1601bool X86InstructionSelector::materializeFP(MachineInstr &I,
1602 MachineRegisterInfo &MRI,
1603 MachineFunction &MF) const {
1604 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1605 "unexpected instruction");
1606
1607 // Can't handle alternate code models yet.
1608 CodeModel::Model CM = TM.getCodeModel();
1609 if (CM != CodeModel::Small && CM != CodeModel::Large)
1610 return false;
1611
1612 const Register DstReg = I.getOperand(i: 0).getReg();
1613 const LLT DstTy = MRI.getType(Reg: DstReg);
1614 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1615 // Create the load from the constant pool.
1616 const ConstantFP *CFP = I.getOperand(i: 1).getFPImm();
1617 const auto &DL = MF.getDataLayout();
1618 Align Alignment = DL.getPrefTypeAlign(Ty: CFP->getType());
1619 const DebugLoc &DbgLoc = I.getDebugLoc();
1620
1621 unsigned Opc =
1622 getLoadStoreOp(Ty: DstTy, RB: RegBank, Opc: TargetOpcode::G_LOAD, Alignment);
1623
1624 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(C: CFP, Alignment);
1625 MachineInstr *LoadInst = nullptr;
1626 unsigned char OpFlag = STI.classifyLocalReference(GV: nullptr);
1627
1628 if (CM == CodeModel::Large && STI.is64Bit()) {
1629 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1630 // they cannot be folded into immediate fields.
1631
1632 Register AddrReg = MRI.createVirtualRegister(RegClass: &X86::GR64RegClass);
1633 BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: AddrReg)
1634 .addConstantPoolIndex(Idx: CPI, Offset: 0, TargetFlags: OpFlag);
1635
1636 MachineMemOperand *MMO = MF.getMachineMemOperand(
1637 PtrInfo: MachinePointerInfo::getConstantPool(MF), f: MachineMemOperand::MOLoad,
1638 MemTy: LLT::pointer(AddressSpace: 0, SizeInBits: DL.getPointerSizeInBits()), base_alignment: Alignment);
1639
1640 LoadInst =
1641 addDirectMem(MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg),
1642 Reg: AddrReg)
1643 .addMemOperand(MMO);
1644
1645 } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1646 // Handle the case when globals fit in our immediate field.
1647 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1648
1649 // x86-32 PIC requires a PIC base register for constant pools.
1650 unsigned PICBase = 0;
1651 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1652 // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1653 // In DAGISEL the code that initialize it generated by the CGBR pass.
1654 return false; // TODO support the mode.
1655 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1656 PICBase = X86::RIP;
1657
1658 LoadInst = addConstantPoolReference(
1659 MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg), CPI, GlobalBaseReg: PICBase,
1660 OpFlags: OpFlag);
1661 } else
1662 return false;
1663
1664 constrainSelectedInstRegOperands(I&: *LoadInst, TII, TRI, RBI);
1665 I.eraseFromParent();
1666 return true;
1667}
1668
1669bool X86InstructionSelector::selectImplicitDefOrPHI(
1670 MachineInstr &I, MachineRegisterInfo &MRI) const {
1671 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1672 I.getOpcode() == TargetOpcode::G_PHI) &&
1673 "unexpected instruction");
1674
1675 Register DstReg = I.getOperand(i: 0).getReg();
1676
1677 if (!MRI.getRegClassOrNull(Reg: DstReg)) {
1678 const LLT DstTy = MRI.getType(Reg: DstReg);
1679 const TargetRegisterClass *RC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1680
1681 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI)) {
1682 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1683 << " operand\n");
1684 return false;
1685 }
1686 }
1687
1688 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1689 I.setDesc(TII.get(Opcode: X86::IMPLICIT_DEF));
1690 else
1691 I.setDesc(TII.get(Opcode: X86::PHI));
1692
1693 return true;
1694}
1695
1696bool X86InstructionSelector::selectMulDivRem(MachineInstr &I,
1697 MachineRegisterInfo &MRI,
1698 MachineFunction &MF) const {
1699 // The implementation of this function is adapted from X86FastISel.
1700 assert((I.getOpcode() == TargetOpcode::G_MUL ||
1701 I.getOpcode() == TargetOpcode::G_SMULH ||
1702 I.getOpcode() == TargetOpcode::G_UMULH ||
1703 I.getOpcode() == TargetOpcode::G_SDIV ||
1704 I.getOpcode() == TargetOpcode::G_SREM ||
1705 I.getOpcode() == TargetOpcode::G_UDIV ||
1706 I.getOpcode() == TargetOpcode::G_UREM) &&
1707 "unexpected instruction");
1708
1709 const Register DstReg = I.getOperand(i: 0).getReg();
1710 const Register Op1Reg = I.getOperand(i: 1).getReg();
1711 const Register Op2Reg = I.getOperand(i: 2).getReg();
1712
1713 const LLT RegTy = MRI.getType(Reg: DstReg);
1714 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1715 "Arguments and return value types must match");
1716
1717 const RegisterBank *RegRB = RBI.getRegBank(Reg: DstReg, MRI, TRI);
1718 if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1719 return false;
1720
1721 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1722 const static unsigned NumOps = 7; // SDiv/SRem/UDiv/URem/Mul/SMulH/UMulh
1723 const static bool S = true; // IsSigned
1724 const static bool U = false; // !IsSigned
1725 const static unsigned Copy = TargetOpcode::COPY;
1726
1727 // For the X86 IDIV instruction, in most cases the dividend
1728 // (numerator) must be in a specific register pair highreg:lowreg,
1729 // producing the quotient in lowreg and the remainder in highreg.
1730 // For most data types, to set up the instruction, the dividend is
1731 // copied into lowreg, and lowreg is sign-extended into highreg. The
1732 // exception is i8, where the dividend is defined as a single register rather
1733 // than a register pair, and we therefore directly sign-extend the dividend
1734 // into lowreg, instead of copying, and ignore the highreg.
1735 const static struct MulDivRemEntry {
1736 // The following portion depends only on the data type.
1737 unsigned SizeInBits;
1738 unsigned LowInReg; // low part of the register pair
1739 unsigned HighInReg; // high part of the register pair
1740 // The following portion depends on both the data type and the operation.
1741 struct MulDivRemResult {
1742 unsigned OpMulDivRem; // The specific MUL/DIV opcode to use.
1743 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1744 // highreg, or copying a zero into highreg.
1745 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1746 // zero/sign-extending into lowreg for i8.
1747 unsigned ResultReg; // Register containing the desired result.
1748 bool IsOpSigned; // Whether to use signed or unsigned form.
1749 } ResultTable[NumOps];
1750 } OpTable[NumTypes] = {
1751 {.SizeInBits: 8,
1752 .LowInReg: X86::AX,
1753 .HighInReg: 0,
1754 .ResultTable: {
1755 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // SDiv
1756 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SRem
1757 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AL, .IsOpSigned: U}, // UDiv
1758 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // URem
1759 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // Mul
1760 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SMulH
1761 {.OpMulDivRem: X86::MUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // UMulH
1762 }}, // i8
1763 {.SizeInBits: 16,
1764 .LowInReg: X86::AX,
1765 .HighInReg: X86::DX,
1766 .ResultTable: {
1767 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // SDiv
1768 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SRem
1769 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: U}, // UDiv
1770 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // URem
1771 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // Mul
1772 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SMulH
1773 {.OpMulDivRem: X86::MUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // UMulH
1774 }}, // i16
1775 {.SizeInBits: 32,
1776 .LowInReg: X86::EAX,
1777 .HighInReg: X86::EDX,
1778 .ResultTable: {
1779 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // SDiv
1780 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SRem
1781 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: U}, // UDiv
1782 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // URem
1783 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // Mul
1784 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SMulH
1785 {.OpMulDivRem: X86::MUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // UMulH
1786 }}, // i32
1787 {.SizeInBits: 64,
1788 .LowInReg: X86::RAX,
1789 .HighInReg: X86::RDX,
1790 .ResultTable: {
1791 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // SDiv
1792 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SRem
1793 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: U}, // UDiv
1794 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // URem
1795 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // Mul
1796 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SMulH
1797 {.OpMulDivRem: X86::MUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // UMulH
1798 }}, // i64
1799 };
1800
1801 auto OpEntryIt = llvm::find_if(Range: OpTable, P: [RegTy](const MulDivRemEntry &El) {
1802 return El.SizeInBits == RegTy.getSizeInBits();
1803 });
1804 if (OpEntryIt == std::end(arr: OpTable))
1805 return false;
1806
1807 unsigned OpIndex;
1808 switch (I.getOpcode()) {
1809 default:
1810 llvm_unreachable("Unexpected mul/div/rem opcode");
1811 case TargetOpcode::G_SDIV:
1812 OpIndex = 0;
1813 break;
1814 case TargetOpcode::G_SREM:
1815 OpIndex = 1;
1816 break;
1817 case TargetOpcode::G_UDIV:
1818 OpIndex = 2;
1819 break;
1820 case TargetOpcode::G_UREM:
1821 OpIndex = 3;
1822 break;
1823 case TargetOpcode::G_MUL:
1824 OpIndex = 4;
1825 break;
1826 case TargetOpcode::G_SMULH:
1827 OpIndex = 5;
1828 break;
1829 case TargetOpcode::G_UMULH:
1830 OpIndex = 6;
1831 break;
1832 }
1833
1834 const MulDivRemEntry &TypeEntry = *OpEntryIt;
1835 const MulDivRemEntry::MulDivRemResult &OpEntry =
1836 TypeEntry.ResultTable[OpIndex];
1837
1838 const TargetRegisterClass *RegRC = getRegClass(Ty: RegTy, RB: *RegRB);
1839 if (!RBI.constrainGenericRegister(Reg: Op1Reg, RC: *RegRC, MRI) ||
1840 !RBI.constrainGenericRegister(Reg: Op2Reg, RC: *RegRC, MRI) ||
1841 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RegRC, MRI)) {
1842 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1843 << " operand\n");
1844 return false;
1845 }
1846
1847 // Move op1 into low-order input register.
1848 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpCopy),
1849 DestReg: TypeEntry.LowInReg)
1850 .addReg(RegNo: Op1Reg);
1851
1852 // Zero-extend or sign-extend into high-order input register.
1853 if (OpEntry.OpSignExtend) {
1854 if (OpEntry.IsOpSigned)
1855 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1856 MCID: TII.get(Opcode: OpEntry.OpSignExtend));
1857 else {
1858 Register Zero32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
1859 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::MOV32r0),
1860 DestReg: Zero32);
1861
1862 // Copy the zero into the appropriate sub/super/identical physical
1863 // register. Unfortunately the operations needed are not uniform enough
1864 // to fit neatly into the table above.
1865 if (RegTy.getSizeInBits() == 16) {
1866 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1867 DestReg: TypeEntry.HighInReg)
1868 .addReg(RegNo: Zero32, Flags: {}, SubReg: X86::sub_16bit);
1869 } else if (RegTy.getSizeInBits() == 32) {
1870 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1871 DestReg: TypeEntry.HighInReg)
1872 .addReg(RegNo: Zero32);
1873 } else if (RegTy.getSizeInBits() == 64) {
1874 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1875 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: TypeEntry.HighInReg)
1876 .addReg(RegNo: Zero32)
1877 .addImm(Val: X86::sub_32bit);
1878 }
1879 }
1880 }
1881
1882 // Generate the DIV/IDIV/MUL/IMUL instruction.
1883 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpMulDivRem))
1884 .addReg(RegNo: Op2Reg);
1885
1886 // For i8 remainder, we can't reference ah directly, as we'll end
1887 // up with bogus copies like %r9b = COPY %ah. Reference ax
1888 // instead to prevent ah references in a rex instruction.
1889 //
1890 // The current assumption of the fast register allocator is that isel
1891 // won't generate explicit references to the GR8_NOREX registers. If
1892 // the allocator and/or the backend get enhanced to be more robust in
1893 // that regard, this can be, and should be, removed.
1894 if (OpEntry.ResultReg == X86::AH && STI.is64Bit()) {
1895 Register SourceSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1896 Register ResultSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1897 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy), DestReg: SourceSuperReg)
1898 .addReg(RegNo: X86::AX);
1899
1900 // Shift AX right by 8 bits instead of using AH.
1901 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SHR16ri),
1902 DestReg: ResultSuperReg)
1903 .addReg(RegNo: SourceSuperReg)
1904 .addImm(Val: 8);
1905
1906 // Now reference the 8-bit subreg of the result.
1907 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1908 DestReg: DstReg)
1909 .addReg(RegNo: ResultSuperReg, Flags: {}, SubReg: X86::sub_8bit);
1910 } else {
1911 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1912 DestReg: DstReg)
1913 .addReg(RegNo: OpEntry.ResultReg);
1914 }
1915 I.eraseFromParent();
1916
1917 return true;
1918}
1919
1920bool X86InstructionSelector::selectSelect(MachineInstr &I,
1921 MachineRegisterInfo &MRI,
1922 MachineFunction &MF) const {
1923 GSelect &Sel = cast<GSelect>(Val&: I);
1924 Register DstReg = Sel.getReg(Idx: 0);
1925 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST32rr))
1926 .addReg(RegNo: Sel.getCondReg())
1927 .addReg(RegNo: Sel.getCondReg());
1928
1929 unsigned OpCmp;
1930 LLT Ty = MRI.getType(Reg: DstReg);
1931 if (Ty.getSizeInBits() == 80) {
1932 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::CMOVE_Fp80),
1933 DestReg: DstReg)
1934 .addReg(RegNo: Sel.getTrueReg())
1935 .addReg(RegNo: Sel.getFalseReg());
1936 } else {
1937 switch (Ty.getSizeInBits()) {
1938 default:
1939 return false;
1940 case 8:
1941 OpCmp = X86::CMOV_GR8;
1942 break;
1943 case 16:
1944 OpCmp = STI.canUseCMOV() ? X86::CMOV16rr : X86::CMOV_GR16;
1945 break;
1946 case 32:
1947 OpCmp = STI.canUseCMOV() ? X86::CMOV32rr : X86::CMOV_GR32;
1948 break;
1949 case 64:
1950 assert(STI.is64Bit() && STI.canUseCMOV());
1951 OpCmp = X86::CMOV64rr;
1952 break;
1953 }
1954 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: OpCmp), DestReg: DstReg)
1955 .addReg(RegNo: Sel.getTrueReg())
1956 .addReg(RegNo: Sel.getFalseReg())
1957 .addImm(Val: X86::COND_E);
1958 }
1959 const TargetRegisterClass *DstRC = getRegClass(Ty, Reg: DstReg, MRI);
1960 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1961 LLVM_DEBUG(dbgs() << "Failed to constrain CMOV\n");
1962 return false;
1963 }
1964
1965 Sel.eraseFromParent();
1966 return true;
1967}
1968
1969InstructionSelector::ComplexRendererFns
1970X86InstructionSelector::selectAddr(MachineOperand &Root) const {
1971 MachineInstr *MI = Root.getParent();
1972 MachineIRBuilder MIRBuilder(*MI);
1973
1974 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
1975 MachineInstr *Ptr = MRI.getVRegDef(Reg: Root.getReg());
1976 X86AddressMode AM;
1977 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
1978 return std::nullopt;
1979
1980 if (AM.IndexReg)
1981 return std::nullopt;
1982
1983 return {// Base
1984 {[=](MachineInstrBuilder &MIB) {
1985 if (AM.BaseType == X86AddressMode::RegBase)
1986 MIB.addUse(RegNo: AM.Base.Reg);
1987 else {
1988 assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
1989 "Unknown type of address base");
1990 MIB.addFrameIndex(Idx: AM.Base.FrameIndex);
1991 }
1992 },
1993 // Scale
1994 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: AM.Scale); },
1995 // Index
1996 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); },
1997 // Disp
1998 [=](MachineInstrBuilder &MIB) {
1999 if (AM.GV)
2000 MIB.addGlobalAddress(GV: AM.GV, Offset: AM.Disp, TargetFlags: AM.GVOpFlags);
2001 else if (AM.CP)
2002 MIB.addConstantPoolIndex(Idx: AM.Disp, Offset: 0, TargetFlags: AM.GVOpFlags);
2003 else
2004 MIB.addImm(Val: AM.Disp);
2005 },
2006 // Segment
2007 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); }}};
2008}
2009
2010InstructionSelector *
2011llvm::createX86InstructionSelector(const X86TargetMachine &TM,
2012 const X86Subtarget &Subtarget,
2013 const X86RegisterBankInfo &RBI) {
2014 return new X86InstructionSelector(TM, Subtarget, RBI);
2015}
2016