1//===- X86InstructionSelector.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// X86.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86BaseInfo.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86RegisterBankInfo.h"
19#include "X86RegisterInfo.h"
20#include "X86Subtarget.h"
21#include "X86TargetMachine.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/Utils.h"
27#include "llvm/CodeGen/MachineBasicBlock.h"
28#include "llvm/CodeGen/MachineConstantPool.h"
29#include "llvm/CodeGen/MachineFunction.h"
30#include "llvm/CodeGen/MachineInstr.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/MachineOperand.h"
34#include "llvm/CodeGen/MachineRegisterInfo.h"
35#include "llvm/CodeGen/RegisterBank.h"
36#include "llvm/CodeGen/TargetOpcodes.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/CodeGenTypes/LowLevelType.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/InstrTypes.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/Support/CodeGen.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MathExtras.h"
46#include "llvm/Support/raw_ostream.h"
47#include <cassert>
48#include <cstdint>
49#include <tuple>
50
51#define DEBUG_TYPE "X86-isel"
52
53using namespace llvm;
54using namespace MIPatternMatch;
55
56namespace {
57
58#define GET_GLOBALISEL_PREDICATE_BITSET
59#include "X86GenGlobalISel.inc"
60#undef GET_GLOBALISEL_PREDICATE_BITSET
61
62class X86InstructionSelector : public InstructionSelector {
63public:
64 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
65 const X86RegisterBankInfo &RBI);
66
67 bool select(MachineInstr &I) override;
68 static const char *getName() { return DEBUG_TYPE; }
69
70private:
71 /// tblgen-erated 'select' implementation, used as the initial selector for
72 /// the patterns that don't require complex C++.
73 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
74
75 // TODO: remove after supported by Tablegen-erated instruction selection.
76 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
77 Align Alignment) const;
78 // TODO: remove once p0<->i32/i64 matching is available
79 unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
80 unsigned Opc) const;
81
82 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
83 MachineFunction &MF) const;
84 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
85 MachineFunction &MF) const;
86 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
87 MachineFunction &MF) const;
88 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
89 MachineFunction &MF) const;
90 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
91 MachineFunction &MF) const;
92 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
93 MachineFunction &MF) const;
94 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
95 MachineFunction &MF) const;
96 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
97 MachineFunction &MF) const;
98 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
99 MachineFunction &MF) const;
100 bool selectUAddSub(MachineInstr &I, MachineRegisterInfo &MRI,
101 MachineFunction &MF) const;
102 bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI) const;
103 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
104 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
105 MachineFunction &MF);
106 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
107 MachineFunction &MF);
108 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
109 MachineFunction &MF) const;
110 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
111 MachineFunction &MF) const;
112 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
113 MachineFunction &MF) const;
114 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
115 const Register DstReg,
116 const TargetRegisterClass *DstRC,
117 const Register SrcReg,
118 const TargetRegisterClass *SrcRC) const;
119 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
120 MachineFunction &MF) const;
121 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
122 bool selectMulDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
123 MachineFunction &MF) const;
124 bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
125 MachineFunction &MF) const;
126
127 ComplexRendererFns selectAddr(MachineOperand &Root) const;
128
129 // emit insert subreg instruction and insert it before MachineInstr &I
130 bool emitInsertSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
131 MachineRegisterInfo &MRI, MachineFunction &MF) const;
132 // emit extract subreg instruction and insert it before MachineInstr &I
133 bool emitExtractSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
134 MachineRegisterInfo &MRI, MachineFunction &MF) const;
135
136 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
137 const TargetRegisterClass *getRegClass(LLT Ty, Register Reg,
138 MachineRegisterInfo &MRI) const;
139
140 const X86TargetMachine &TM;
141 const X86Subtarget &STI;
142 const X86InstrInfo &TII;
143 const X86RegisterInfo &TRI;
144 const X86RegisterBankInfo &RBI;
145
146#define GET_GLOBALISEL_PREDICATES_DECL
147#include "X86GenGlobalISel.inc"
148#undef GET_GLOBALISEL_PREDICATES_DECL
149
150#define GET_GLOBALISEL_TEMPORARIES_DECL
151#include "X86GenGlobalISel.inc"
152#undef GET_GLOBALISEL_TEMPORARIES_DECL
153};
154
155} // end anonymous namespace
156
157#define GET_GLOBALISEL_IMPL
158#include "X86GenGlobalISel.inc"
159#undef GET_GLOBALISEL_IMPL
160
161X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
162 const X86Subtarget &STI,
163 const X86RegisterBankInfo &RBI)
164 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
165 RBI(RBI),
166#define GET_GLOBALISEL_PREDICATES_INIT
167#include "X86GenGlobalISel.inc"
168#undef GET_GLOBALISEL_PREDICATES_INIT
169#define GET_GLOBALISEL_TEMPORARIES_INIT
170#include "X86GenGlobalISel.inc"
171#undef GET_GLOBALISEL_TEMPORARIES_INIT
172{
173}
174
175// FIXME: This should be target-independent, inferred from the types declared
176// for each class in the bank.
177const TargetRegisterClass *
178X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
179 if (RB.getID() == X86::GPRRegBankID) {
180 if (Ty.getSizeInBits() <= 8)
181 return &X86::GR8RegClass;
182 if (Ty.getSizeInBits() == 16)
183 return &X86::GR16RegClass;
184 if (Ty.getSizeInBits() == 32)
185 return &X86::GR32RegClass;
186 if (Ty.getSizeInBits() == 64)
187 return &X86::GR64RegClass;
188 }
189 if (RB.getID() == X86::VECRRegBankID) {
190 if (Ty.getSizeInBits() == 16)
191 return STI.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
192 if (Ty.getSizeInBits() == 32)
193 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
194 if (Ty.getSizeInBits() == 64)
195 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
196 if (Ty.getSizeInBits() == 128)
197 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
198 if (Ty.getSizeInBits() == 256)
199 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
200 if (Ty.getSizeInBits() == 512)
201 return &X86::VR512RegClass;
202 }
203
204 if (RB.getID() == X86::PSRRegBankID) {
205 if (Ty.getSizeInBits() == 80)
206 return &X86::RFP80RegClass;
207 if (Ty.getSizeInBits() == 64)
208 return &X86::RFP64RegClass;
209 if (Ty.getSizeInBits() == 32)
210 return &X86::RFP32RegClass;
211 }
212
213 llvm_unreachable("Unknown RegBank!");
214}
215
216const TargetRegisterClass *
217X86InstructionSelector::getRegClass(LLT Ty, Register Reg,
218 MachineRegisterInfo &MRI) const {
219 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
220 return getRegClass(Ty, RB: RegBank);
221}
222
223static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
224 unsigned SubIdx = X86::NoSubRegister;
225 if (RC == &X86::GR32RegClass) {
226 SubIdx = X86::sub_32bit;
227 } else if (RC == &X86::GR16RegClass) {
228 SubIdx = X86::sub_16bit;
229 } else if (RC == &X86::GR8RegClass) {
230 SubIdx = X86::sub_8bit;
231 }
232
233 return SubIdx;
234}
235
236static const TargetRegisterClass *getRegClassFromGRPhysReg(Register Reg) {
237 assert(Reg.isPhysical());
238 if (X86::GR64RegClass.contains(Reg))
239 return &X86::GR64RegClass;
240 if (X86::GR32RegClass.contains(Reg))
241 return &X86::GR32RegClass;
242 if (X86::GR16RegClass.contains(Reg))
243 return &X86::GR16RegClass;
244 if (X86::GR8RegClass.contains(Reg))
245 return &X86::GR8RegClass;
246
247 llvm_unreachable("Unknown RegClass for PhysReg!");
248}
249
250// FIXME: We need some sort of API in RBI/TRI to allow generic code to
251// constrain operands of simple instructions given a TargetRegisterClass
252// and LLT
253bool X86InstructionSelector::selectDebugInstr(MachineInstr &I,
254 MachineRegisterInfo &MRI) const {
255 for (MachineOperand &MO : I.operands()) {
256 if (!MO.isReg())
257 continue;
258 Register Reg = MO.getReg();
259 if (!Reg)
260 continue;
261 if (Reg.isPhysical())
262 continue;
263 LLT Ty = MRI.getType(Reg);
264 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
265 const TargetRegisterClass *RC =
266 dyn_cast_if_present<const TargetRegisterClass *>(Val: RegClassOrBank);
267 if (!RC) {
268 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
269 RC = getRegClass(Ty, RB);
270 if (!RC) {
271 LLVM_DEBUG(
272 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
273 break;
274 }
275 }
276 RBI.constrainGenericRegister(Reg, RC: *RC, MRI);
277 }
278
279 return true;
280}
281
282// Set X86 Opcode and constrain DestReg.
283bool X86InstructionSelector::selectCopy(MachineInstr &I,
284 MachineRegisterInfo &MRI) const {
285 Register DstReg = I.getOperand(i: 0).getReg();
286 const unsigned DstSize = RBI.getSizeInBits(Reg: DstReg, MRI, TRI);
287 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
288
289 Register SrcReg = I.getOperand(i: 1).getReg();
290 const unsigned SrcSize = RBI.getSizeInBits(Reg: SrcReg, MRI, TRI);
291 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
292
293 if (DstReg.isPhysical()) {
294 assert(I.isCopy() && "Generic operators do not allow physical registers");
295
296 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
297 DstRegBank.getID() == X86::GPRRegBankID) {
298
299 const TargetRegisterClass *SrcRC =
300 getRegClass(Ty: MRI.getType(Reg: SrcReg), RB: SrcRegBank);
301 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(Reg: DstReg);
302
303 if (SrcRC != DstRC) {
304 // This case can be generated by ABI lowering, performe anyext
305 Register ExtSrc = MRI.createVirtualRegister(RegClass: DstRC);
306 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
307 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
308 .addDef(RegNo: ExtSrc)
309 .addReg(RegNo: SrcReg)
310 .addImm(Val: getSubRegIndex(RC: SrcRC));
311
312 I.getOperand(i: 1).setReg(ExtSrc);
313 }
314 }
315
316 // Special case GPR16 -> XMM
317 if (SrcSize == 16 && SrcRegBank.getID() == X86::GPRRegBankID &&
318 (DstRegBank.getID() == X86::VECRRegBankID)) {
319
320 const DebugLoc &DL = I.getDebugLoc();
321
322 // Any extend GPR16 -> GPR32
323 Register ExtReg = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
324 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG),
325 DestReg: ExtReg)
326 .addReg(RegNo: SrcReg)
327 .addImm(Val: X86::sub_16bit);
328
329 // Copy GR32 -> XMM
330 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
331 .addReg(RegNo: ExtReg);
332
333 I.eraseFromParent();
334 }
335
336 // Special case XMM -> GR16
337 if (DstSize == 16 && DstRegBank.getID() == X86::GPRRegBankID &&
338 (SrcRegBank.getID() == X86::VECRRegBankID)) {
339
340 const DebugLoc &DL = I.getDebugLoc();
341
342 // Move XMM to GR32 register.
343 Register Temp32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
344 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Temp32)
345 .addReg(RegNo: SrcReg);
346
347 // Extract the lower 16 bits
348 if (Register Dst32 = TRI.getMatchingSuperReg(Reg: DstReg, SubIdx: X86::sub_16bit,
349 RC: &X86::GR32RegClass)) {
350 // Optimization for Physical Dst (e.g. AX): Copy to EAX directly.
351 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst32)
352 .addReg(RegNo: Temp32);
353 } else {
354 // Handle if there is no super.
355 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
356 .addReg(RegNo: Temp32, Flags: {}, SubReg: X86::sub_16bit);
357 }
358
359 I.eraseFromParent();
360 }
361
362 return true;
363 }
364
365 assert((!SrcReg.isPhysical() || I.isCopy()) &&
366 "No phys reg on generic operators");
367 assert((DstSize == SrcSize ||
368 // Copies are a mean to setup initial types, the number of
369 // bits may not exactly match.
370 (SrcReg.isPhysical() &&
371 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
372 "Copy with different width?!");
373
374 const TargetRegisterClass *DstRC =
375 getRegClass(Ty: MRI.getType(Reg: DstReg), RB: DstRegBank);
376
377 if (SrcRegBank.getID() == X86::GPRRegBankID &&
378 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
379 SrcReg.isPhysical()) {
380 // Change the physical register to performe truncate.
381
382 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(Reg: SrcReg);
383
384 if (DstRC != SrcRC) {
385 I.getOperand(i: 1).setSubReg(getSubRegIndex(RC: DstRC));
386 I.getOperand(i: 1).substPhysReg(Reg: SrcReg, TRI);
387 }
388 }
389
390 // No need to constrain SrcReg. It will get constrained when
391 // we hit another of its use or its defs.
392 // Copies do not have constraints.
393 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(Reg: DstReg);
394 if (!OldRC || !DstRC->hasSubClassEq(RC: OldRC)) {
395 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
396 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
397 << " operand\n");
398 return false;
399 }
400 }
401 I.setDesc(TII.get(Opcode: X86::COPY));
402 return true;
403}
404
405bool X86InstructionSelector::select(MachineInstr &I) {
406 assert(I.getParent() && "Instruction should be in a basic block!");
407 assert(I.getParent()->getParent() && "Instruction should be in a function!");
408
409 MachineBasicBlock &MBB = *I.getParent();
410 MachineFunction &MF = *MBB.getParent();
411 MachineRegisterInfo &MRI = MF.getRegInfo();
412
413 unsigned Opcode = I.getOpcode();
414 if (!isPreISelGenericOpcode(Opcode) && !I.isPreISelOpcode()) {
415 // Certain non-generic instructions also need some special handling.
416
417 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
418 return false;
419
420 if (I.isCopy())
421 return selectCopy(I, MRI);
422
423 if (I.isDebugInstr())
424 return selectDebugInstr(I, MRI);
425
426 return true;
427 }
428
429 assert(I.getNumOperands() == I.getNumExplicitOperands() &&
430 "Generic instruction has unexpected implicit operands\n");
431
432 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
433 return true;
434
435 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
436
437 // TODO: This should be implemented by tblgen.
438 switch (I.getOpcode()) {
439 default:
440 return false;
441 case TargetOpcode::G_STORE:
442 case TargetOpcode::G_LOAD:
443 return selectLoadStoreOp(I, MRI, MF);
444 case TargetOpcode::G_PTR_ADD:
445 case TargetOpcode::G_FRAME_INDEX:
446 return selectFrameIndexOrGep(I, MRI, MF);
447 case TargetOpcode::G_GLOBAL_VALUE:
448 return selectGlobalValue(I, MRI, MF);
449 case TargetOpcode::G_CONSTANT:
450 return selectConstant(I, MRI, MF);
451 case TargetOpcode::G_FCONSTANT:
452 return materializeFP(I, MRI, MF);
453 case TargetOpcode::G_PTRTOINT:
454 case TargetOpcode::G_TRUNC:
455 return selectTruncOrPtrToInt(I, MRI, MF);
456 case TargetOpcode::G_INTTOPTR:
457 case TargetOpcode::G_FREEZE:
458 return selectCopy(I, MRI);
459 case TargetOpcode::G_ZEXT:
460 return selectZext(I, MRI, MF);
461 case TargetOpcode::G_ANYEXT:
462 return selectAnyext(I, MRI, MF);
463 case TargetOpcode::G_ICMP:
464 return selectCmp(I, MRI, MF);
465 case TargetOpcode::G_FCMP:
466 return selectFCmp(I, MRI, MF);
467 case TargetOpcode::G_UADDE:
468 case TargetOpcode::G_UADDO:
469 case TargetOpcode::G_USUBE:
470 case TargetOpcode::G_USUBO:
471 return selectUAddSub(I, MRI, MF);
472 case TargetOpcode::G_UNMERGE_VALUES:
473 return selectUnmergeValues(I, MRI, MF);
474 case TargetOpcode::G_MERGE_VALUES:
475 case TargetOpcode::G_CONCAT_VECTORS:
476 return selectMergeValues(I, MRI, MF);
477 case TargetOpcode::G_EXTRACT:
478 return selectExtract(I, MRI, MF);
479 case TargetOpcode::G_INSERT:
480 return selectInsert(I, MRI, MF);
481 case TargetOpcode::G_BRCOND:
482 return selectCondBranch(I, MRI, MF);
483 case TargetOpcode::G_IMPLICIT_DEF:
484 case TargetOpcode::G_PHI:
485 return selectImplicitDefOrPHI(I, MRI);
486 case TargetOpcode::G_MUL:
487 case TargetOpcode::G_SMULH:
488 case TargetOpcode::G_UMULH:
489 case TargetOpcode::G_SDIV:
490 case TargetOpcode::G_UDIV:
491 case TargetOpcode::G_SREM:
492 case TargetOpcode::G_UREM:
493 return selectMulDivRem(I, MRI, MF);
494 case TargetOpcode::G_SELECT:
495 return selectSelect(I, MRI, MF);
496 }
497
498 return false;
499}
500
501unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
502 const RegisterBank &RB,
503 unsigned Opc) const {
504 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
505 "Only G_STORE and G_LOAD are expected for selection");
506 if (Ty.isPointer() && X86::GPRRegBankID == RB.getID()) {
507 bool IsLoad = (Opc == TargetOpcode::G_LOAD);
508 switch (Ty.getSizeInBits()) {
509 default:
510 break;
511 case 32:
512 return IsLoad ? X86::MOV32rm : X86::MOV32mr;
513 case 64:
514 return IsLoad ? X86::MOV64rm : X86::MOV64mr;
515 }
516 }
517 return Opc;
518}
519
520unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
521 const RegisterBank &RB,
522 unsigned Opc,
523 Align Alignment) const {
524 bool Isload = (Opc == TargetOpcode::G_LOAD);
525 bool HasAVX = STI.hasAVX();
526 bool HasAVX512 = STI.hasAVX512();
527 bool HasVLX = STI.hasVLX();
528
529 if (Ty == LLT::scalar(SizeInBits: 8)) {
530 if (X86::GPRRegBankID == RB.getID())
531 return Isload ? X86::MOV8rm : X86::MOV8mr;
532 } else if (Ty == LLT::scalar(SizeInBits: 16)) {
533 if (X86::GPRRegBankID == RB.getID())
534 return Isload ? X86::MOV16rm : X86::MOV16mr;
535 } else if (Ty == LLT::scalar(SizeInBits: 32)) {
536 if (X86::GPRRegBankID == RB.getID())
537 return Isload ? X86::MOV32rm : X86::MOV32mr;
538 if (X86::VECRRegBankID == RB.getID())
539 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
540 HasAVX ? X86::VMOVSSrm_alt :
541 X86::MOVSSrm_alt)
542 : (HasAVX512 ? X86::VMOVSSZmr :
543 HasAVX ? X86::VMOVSSmr :
544 X86::MOVSSmr);
545 if (X86::PSRRegBankID == RB.getID())
546 return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
547 } else if (Ty == LLT::scalar(SizeInBits: 64)) {
548 if (X86::GPRRegBankID == RB.getID())
549 return Isload ? X86::MOV64rm : X86::MOV64mr;
550 if (X86::VECRRegBankID == RB.getID())
551 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
552 HasAVX ? X86::VMOVSDrm_alt :
553 X86::MOVSDrm_alt)
554 : (HasAVX512 ? X86::VMOVSDZmr :
555 HasAVX ? X86::VMOVSDmr :
556 X86::MOVSDmr);
557 if (X86::PSRRegBankID == RB.getID())
558 return Isload ? X86::LD_Fp64m : X86::ST_Fp64m;
559 } else if (Ty == LLT::scalar(SizeInBits: 80)) {
560 return Isload ? X86::LD_Fp80m : X86::ST_FpP80m;
561 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
562 if (Alignment >= Align(16))
563 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
564 : HasAVX512
565 ? X86::VMOVAPSZ128rm_NOVLX
566 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
567 : (HasVLX ? X86::VMOVAPSZ128mr
568 : HasAVX512
569 ? X86::VMOVAPSZ128mr_NOVLX
570 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
571 else
572 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
573 : HasAVX512
574 ? X86::VMOVUPSZ128rm_NOVLX
575 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
576 : (HasVLX ? X86::VMOVUPSZ128mr
577 : HasAVX512
578 ? X86::VMOVUPSZ128mr_NOVLX
579 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
580 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
581 if (Alignment >= Align(32))
582 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
583 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
584 : X86::VMOVAPSYrm)
585 : (HasVLX ? X86::VMOVAPSZ256mr
586 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
587 : X86::VMOVAPSYmr);
588 else
589 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
590 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
591 : X86::VMOVUPSYrm)
592 : (HasVLX ? X86::VMOVUPSZ256mr
593 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
594 : X86::VMOVUPSYmr);
595 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
596 if (Alignment >= Align(64))
597 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
598 else
599 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
600 }
601 return Opc;
602}
603
604// Fill in an address from the given instruction.
605static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
606 const MachineRegisterInfo &MRI,
607 const X86Subtarget &STI, X86AddressMode &AM) {
608 assert(I.getOperand(0).isReg() && "unsupported operand.");
609 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
610 "unsupported type.");
611
612 switch (I.getOpcode()) {
613 default:
614 break;
615 case TargetOpcode::G_FRAME_INDEX:
616 AM.Base.FrameIndex = I.getOperand(i: 1).getIndex();
617 AM.BaseType = X86AddressMode::FrameIndexBase;
618 return true;
619 case TargetOpcode::G_PTR_ADD: {
620 if (auto COff = getIConstantVRegSExtVal(VReg: I.getOperand(i: 2).getReg(), MRI)) {
621 int64_t Imm = *COff;
622 if (isInt<32>(x: Imm)) { // Check for displacement overflow.
623 AM.Disp = static_cast<int32_t>(Imm);
624 AM.Base.Reg = I.getOperand(i: 1).getReg();
625 return true;
626 }
627 }
628 break;
629 }
630 case TargetOpcode::G_GLOBAL_VALUE:
631 case X86::G_WRAPPER_RIP: {
632 auto GV = I.getOperand(i: 1).getGlobal();
633 if (GV->isThreadLocal()) {
634 return false; // TODO: we don't support TLS yet.
635 }
636 // Can't handle alternate code models yet.
637 if (TM.getCodeModel() != CodeModel::Small)
638 return false;
639 AM.GV = GV;
640 AM.GVOpFlags = STI.classifyGlobalReference(GV);
641
642 // TODO: This reference is relative to the pic base. not supported yet.
643 if (isGlobalRelativeToPICBase(TargetFlag: AM.GVOpFlags))
644 return false;
645
646 if (STI.isPICStyleRIPRel() || AM.GVOpFlags == X86II::MO_GOTPCREL ||
647 AM.GVOpFlags == X86II::MO_GOTPCREL_NORELAX) {
648 // Use rip-relative addressing.
649 assert(AM.Base.Reg == 0 && AM.IndexReg == 0 &&
650 "RIP-relative addresses can't have additional register operands");
651 AM.Base.Reg = X86::RIP;
652 }
653 return true;
654 }
655 case TargetOpcode::G_CONSTANT_POOL: {
656 // TODO: Need a separate move for Large model
657 if (TM.getCodeModel() == CodeModel::Large)
658 return false;
659
660 AM.GVOpFlags = STI.classifyLocalReference(GV: nullptr);
661 if (AM.GVOpFlags == X86II::MO_GOTOFF)
662 AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(MF: I.getMF());
663 else if (STI.is64Bit())
664 AM.Base.Reg = X86::RIP;
665 AM.CP = true;
666 AM.Disp = I.getOperand(i: 1).getIndex();
667 return true;
668 }
669 }
670 // Default behavior.
671 AM.Base.Reg = I.getOperand(i: 0).getReg();
672 return true;
673}
674
675bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
676 MachineRegisterInfo &MRI,
677 MachineFunction &MF) const {
678 unsigned Opc = I.getOpcode();
679
680 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
681 "Only G_STORE and G_LOAD are expected for selection");
682
683 const Register DefReg = I.getOperand(i: 0).getReg();
684 LLT Ty = MRI.getType(Reg: DefReg);
685 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
686
687 assert(I.hasOneMemOperand());
688 auto &MemOp = **I.memoperands_begin();
689 if (MemOp.isAtomic()) {
690 // Note: for unordered operations, we rely on the fact the appropriate MMO
691 // is already on the instruction we're mutating, and thus we don't need to
692 // make any changes. So long as we select an opcode which is capable of
693 // loading or storing the appropriate size atomically, the rest of the
694 // backend is required to respect the MMO state.
695 if (!MemOp.isUnordered()) {
696 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
697 return false;
698 }
699 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
700 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
701 return false;
702 }
703 }
704
705 unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
706 if (NewOpc == Opc)
707 return false;
708
709 I.setDesc(TII.get(Opcode: NewOpc));
710 MachineInstrBuilder MIB(MF, I);
711 MachineInstr *Ptr = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
712
713 X86AddressMode AM;
714 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
715 return false;
716
717 if (Opc == TargetOpcode::G_LOAD) {
718 I.removeOperand(OpNo: 1);
719 addFullAddress(MIB, AM);
720 } else {
721 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
722 I.removeOperand(OpNo: 1);
723 I.removeOperand(OpNo: 0);
724 addFullAddress(MIB, AM).addUse(RegNo: DefReg);
725 }
726 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
727 I.addImplicitDefUseOperands(MF);
728 return true;
729}
730
731static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
732 if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 64))
733 return X86::LEA64r;
734 else if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 32))
735 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
736 else
737 llvm_unreachable("Can't get LEA opcode. Unsupported type.");
738}
739
740bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
741 MachineRegisterInfo &MRI,
742 MachineFunction &MF) const {
743 unsigned Opc = I.getOpcode();
744
745 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
746 "unexpected instruction");
747
748 const Register DefReg = I.getOperand(i: 0).getReg();
749 LLT Ty = MRI.getType(Reg: DefReg);
750
751 // Use LEA to calculate frame index and GEP
752 unsigned NewOpc = getLeaOP(Ty, STI);
753 I.setDesc(TII.get(Opcode: NewOpc));
754 MachineInstrBuilder MIB(MF, I);
755
756 if (Opc == TargetOpcode::G_FRAME_INDEX) {
757 addOffset(MIB, Offset: 0);
758 } else {
759 MachineOperand &InxOp = I.getOperand(i: 2);
760 I.addOperand(Op: InxOp); // set IndexReg
761 InxOp.ChangeToImmediate(ImmVal: 1); // set Scale
762 MIB.addImm(Val: 0).addReg(RegNo: 0);
763 }
764
765 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
766 return true;
767}
768
769bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
770 MachineRegisterInfo &MRI,
771 MachineFunction &MF) const {
772 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
773 "unexpected instruction");
774
775 X86AddressMode AM;
776 if (!X86SelectAddress(I, TM, MRI, STI, AM))
777 return false;
778
779 const Register DefReg = I.getOperand(i: 0).getReg();
780 LLT Ty = MRI.getType(Reg: DefReg);
781 unsigned NewOpc = getLeaOP(Ty, STI);
782
783 I.setDesc(TII.get(Opcode: NewOpc));
784 MachineInstrBuilder MIB(MF, I);
785
786 I.removeOperand(OpNo: 1);
787 addFullAddress(MIB, AM);
788
789 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
790 return true;
791}
792
793bool X86InstructionSelector::selectConstant(MachineInstr &I,
794 MachineRegisterInfo &MRI,
795 MachineFunction &MF) const {
796 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
797 "unexpected instruction");
798
799 const Register DefReg = I.getOperand(i: 0).getReg();
800 LLT Ty = MRI.getType(Reg: DefReg);
801
802 if (RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
803 return false;
804
805 uint64_t Val = 0;
806 if (I.getOperand(i: 1).isCImm()) {
807 Val = I.getOperand(i: 1).getCImm()->getZExtValue();
808 I.getOperand(i: 1).ChangeToImmediate(ImmVal: Val);
809 } else if (I.getOperand(i: 1).isImm()) {
810 Val = I.getOperand(i: 1).getImm();
811 } else
812 llvm_unreachable("Unsupported operand type.");
813
814 unsigned NewOpc;
815 switch (Ty.getSizeInBits()) {
816 case 8:
817 NewOpc = X86::MOV8ri;
818 break;
819 case 16:
820 NewOpc = X86::MOV16ri;
821 break;
822 case 32:
823 NewOpc = X86::MOV32ri;
824 break;
825 case 64:
826 NewOpc = X86::getMOVriOpcode(/*Use64BitReg=*/true, Imm: Val);
827 break;
828 default:
829 llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
830 }
831
832 I.setDesc(TII.get(Opcode: NewOpc));
833 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
834 return true;
835}
836
837// Helper function for selectTruncOrPtrToInt and selectAnyext.
838// Returns true if DstRC lives on a floating register class and
839// SrcRC lives on a 128-bit vector class.
840static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
841 const TargetRegisterClass *SrcRC) {
842 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
843 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
844 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
845}
846
847bool X86InstructionSelector::selectTurnIntoCOPY(
848 MachineInstr &I, MachineRegisterInfo &MRI, const Register DstReg,
849 const TargetRegisterClass *DstRC, const Register SrcReg,
850 const TargetRegisterClass *SrcRC) const {
851
852 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
853 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
854 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
855 << " operand\n");
856 return false;
857 }
858 I.setDesc(TII.get(Opcode: X86::COPY));
859 return true;
860}
861
862bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
863 MachineRegisterInfo &MRI,
864 MachineFunction &MF) const {
865 assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
866 I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
867 "unexpected instruction");
868
869 const Register DstReg = I.getOperand(i: 0).getReg();
870 const Register SrcReg = I.getOperand(i: 1).getReg();
871
872 const LLT DstTy = MRI.getType(Reg: DstReg);
873 const LLT SrcTy = MRI.getType(Reg: SrcReg);
874
875 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
876 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
877
878 if (DstRB.getID() != SrcRB.getID()) {
879 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
880 << " input/output on different banks\n");
881 return false;
882 }
883
884 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
885 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
886
887 if (!DstRC || !SrcRC)
888 return false;
889
890 // If that's truncation of the value that lives on the vector class and goes
891 // into the floating class, just replace it with copy, as we are able to
892 // select it as a regular move.
893 if (canTurnIntoCOPY(DstRC, SrcRC))
894 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
895
896 if (DstRB.getID() != X86::GPRRegBankID)
897 return false;
898
899 unsigned SubIdx;
900 if (DstRC == SrcRC) {
901 // Nothing to be done
902 SubIdx = X86::NoSubRegister;
903 } else if (DstRC == &X86::GR32RegClass) {
904 SubIdx = X86::sub_32bit;
905 } else if (DstRC == &X86::GR16RegClass) {
906 SubIdx = X86::sub_16bit;
907 } else if (DstRC == &X86::GR8RegClass) {
908 SubIdx = X86::sub_8bit;
909 } else {
910 return false;
911 }
912
913 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
914
915 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
916 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
917 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
918 << "\n");
919 return false;
920 }
921
922 I.getOperand(i: 1).setSubReg(SubIdx);
923
924 I.setDesc(TII.get(Opcode: X86::COPY));
925 return true;
926}
927
928bool X86InstructionSelector::selectZext(MachineInstr &I,
929 MachineRegisterInfo &MRI,
930 MachineFunction &MF) const {
931 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
932
933 const Register DstReg = I.getOperand(i: 0).getReg();
934 const Register SrcReg = I.getOperand(i: 1).getReg();
935
936 const LLT DstTy = MRI.getType(Reg: DstReg);
937 const LLT SrcTy = MRI.getType(Reg: SrcReg);
938
939 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) &&
940 "8=>16 Zext is handled by tablegen");
941 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
942 "8=>32 Zext is handled by tablegen");
943 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
944 "16=>32 Zext is handled by tablegen");
945 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) &&
946 "8=>64 Zext is handled by tablegen");
947 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) &&
948 "16=>64 Zext is handled by tablegen");
949 assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) &&
950 "32=>64 Zext is handled by tablegen");
951
952 if (SrcTy != LLT::scalar(SizeInBits: 1))
953 return false;
954
955 unsigned AndOpc;
956 if (DstTy == LLT::scalar(SizeInBits: 8))
957 AndOpc = X86::AND8ri;
958 else if (DstTy == LLT::scalar(SizeInBits: 16))
959 AndOpc = X86::AND16ri;
960 else if (DstTy == LLT::scalar(SizeInBits: 32))
961 AndOpc = X86::AND32ri;
962 else if (DstTy == LLT::scalar(SizeInBits: 64))
963 AndOpc = X86::AND64ri32;
964 else
965 return false;
966
967 Register DefReg = SrcReg;
968 if (DstTy != LLT::scalar(SizeInBits: 8)) {
969 Register ImpDefReg =
970 MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
971 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
972 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImpDefReg);
973
974 DefReg = MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
975 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
976 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DefReg)
977 .addReg(RegNo: ImpDefReg)
978 .addReg(RegNo: SrcReg)
979 .addImm(Val: X86::sub_8bit);
980 }
981
982 MachineInstr &AndInst =
983 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AndOpc), DestReg: DstReg)
984 .addReg(RegNo: DefReg)
985 .addImm(Val: 1);
986
987 constrainSelectedInstRegOperands(I&: AndInst, TII, TRI, RBI);
988
989 I.eraseFromParent();
990 return true;
991}
992
993bool X86InstructionSelector::selectAnyext(MachineInstr &I,
994 MachineRegisterInfo &MRI,
995 MachineFunction &MF) const {
996 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
997
998 const Register DstReg = I.getOperand(i: 0).getReg();
999 const Register SrcReg = I.getOperand(i: 1).getReg();
1000
1001 const LLT DstTy = MRI.getType(Reg: DstReg);
1002 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1003
1004 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1005 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
1006
1007 assert(DstRB.getID() == SrcRB.getID() &&
1008 "G_ANYEXT input/output on different banks\n");
1009
1010 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
1011 "G_ANYEXT incorrect operand size");
1012
1013 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
1014 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
1015
1016 // If that's ANY_EXT of the value that lives on the floating class and goes
1017 // into the vector class, just replace it with copy, as we are able to select
1018 // it as a regular move.
1019 if (canTurnIntoCOPY(DstRC: SrcRC, SrcRC: DstRC))
1020 return selectTurnIntoCOPY(I, MRI, DstReg: SrcReg, DstRC: SrcRC, SrcReg: DstReg, SrcRC: DstRC);
1021
1022 if (DstRB.getID() != X86::GPRRegBankID)
1023 return false;
1024
1025 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1026 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1027 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1028 << " operand\n");
1029 return false;
1030 }
1031
1032 if (SrcRC == DstRC) {
1033 I.setDesc(TII.get(Opcode: X86::COPY));
1034 return true;
1035 }
1036
1037 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1038 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
1039 .addDef(RegNo: DstReg)
1040 .addReg(RegNo: SrcReg)
1041 .addImm(Val: getSubRegIndex(RC: SrcRC));
1042
1043 I.eraseFromParent();
1044 return true;
1045}
1046
1047bool X86InstructionSelector::selectCmp(MachineInstr &I,
1048 MachineRegisterInfo &MRI,
1049 MachineFunction &MF) const {
1050 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
1051
1052 X86::CondCode CC;
1053 bool SwapArgs;
1054 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(
1055 Predicate: (CmpInst::Predicate)I.getOperand(i: 1).getPredicate());
1056
1057 Register LHS = I.getOperand(i: 2).getReg();
1058 Register RHS = I.getOperand(i: 3).getReg();
1059
1060 if (SwapArgs)
1061 std::swap(a&: LHS, b&: RHS);
1062
1063 unsigned OpCmp;
1064 LLT Ty = MRI.getType(Reg: LHS);
1065
1066 switch (Ty.getSizeInBits()) {
1067 default:
1068 return false;
1069 case 8:
1070 OpCmp = X86::CMP8rr;
1071 break;
1072 case 16:
1073 OpCmp = X86::CMP16rr;
1074 break;
1075 case 32:
1076 OpCmp = X86::CMP32rr;
1077 break;
1078 case 64:
1079 OpCmp = X86::CMP64rr;
1080 break;
1081 }
1082
1083 MachineInstr &CmpInst =
1084 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1085 .addReg(RegNo: LHS)
1086 .addReg(RegNo: RHS);
1087
1088 MachineInstr &SetInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1089 MCID: TII.get(Opcode: X86::SETCCr), DestReg: I.getOperand(i: 0).getReg()).addImm(Val: CC);
1090
1091 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1092 constrainSelectedInstRegOperands(I&: SetInst, TII, TRI, RBI);
1093
1094 I.eraseFromParent();
1095 return true;
1096}
1097
1098bool X86InstructionSelector::selectFCmp(MachineInstr &I,
1099 MachineRegisterInfo &MRI,
1100 MachineFunction &MF) const {
1101 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
1102
1103 Register LhsReg = I.getOperand(i: 2).getReg();
1104 Register RhsReg = I.getOperand(i: 3).getReg();
1105 CmpInst::Predicate Predicate =
1106 (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1107
1108 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1109 static const uint16_t SETFOpcTable[2][3] = {
1110 {X86::COND_E, X86::COND_NP, X86::AND8rr},
1111 {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1112 const uint16_t *SETFOpc = nullptr;
1113 switch (Predicate) {
1114 default:
1115 break;
1116 case CmpInst::FCMP_OEQ:
1117 SETFOpc = &SETFOpcTable[0][0];
1118 break;
1119 case CmpInst::FCMP_UNE:
1120 SETFOpc = &SETFOpcTable[1][0];
1121 break;
1122 }
1123
1124 assert((LhsReg.isVirtual() && RhsReg.isVirtual()) &&
1125 "Both arguments of FCMP need to be virtual!");
1126 auto *LhsBank = RBI.getRegBank(Reg: LhsReg, MRI, TRI);
1127 [[maybe_unused]] auto *RhsBank = RBI.getRegBank(Reg: RhsReg, MRI, TRI);
1128 assert((LhsBank == RhsBank) &&
1129 "Both banks assigned to FCMP arguments need to be same!");
1130
1131 // Compute the opcode for the CMP instruction.
1132 unsigned OpCmp;
1133 LLT Ty = MRI.getType(Reg: LhsReg);
1134 switch (Ty.getSizeInBits()) {
1135 default:
1136 return false;
1137 case 32:
1138 OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr32
1139 : X86::UCOMISSrr;
1140 break;
1141 case 64:
1142 OpCmp = LhsBank->getID() == X86::PSRRegBankID ? X86::UCOM_FpIr64
1143 : X86::UCOMISDrr;
1144 break;
1145 case 80:
1146 OpCmp = X86::UCOM_FpIr80;
1147 break;
1148 }
1149
1150 Register ResultReg = I.getOperand(i: 0).getReg();
1151 RBI.constrainGenericRegister(
1152 Reg: ResultReg,
1153 RC: *getRegClass(Ty: LLT::scalar(SizeInBits: 8), RB: *RBI.getRegBank(Reg: ResultReg, MRI, TRI)), MRI);
1154 if (SETFOpc) {
1155 MachineInstr &CmpInst =
1156 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1157 .addReg(RegNo: LhsReg)
1158 .addReg(RegNo: RhsReg);
1159
1160 Register FlagReg1 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1161 Register FlagReg2 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1162 MachineInstr &Set1 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1163 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg1).addImm(Val: SETFOpc[0]);
1164 MachineInstr &Set2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1165 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg2).addImm(Val: SETFOpc[1]);
1166 MachineInstr &Set3 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1167 MCID: TII.get(Opcode: SETFOpc[2]), DestReg: ResultReg)
1168 .addReg(RegNo: FlagReg1)
1169 .addReg(RegNo: FlagReg2);
1170 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1171 constrainSelectedInstRegOperands(I&: Set1, TII, TRI, RBI);
1172 constrainSelectedInstRegOperands(I&: Set2, TII, TRI, RBI);
1173 constrainSelectedInstRegOperands(I&: Set3, TII, TRI, RBI);
1174
1175 I.eraseFromParent();
1176 return true;
1177 }
1178
1179 X86::CondCode CC;
1180 bool SwapArgs;
1181 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1182 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1183
1184 if (SwapArgs)
1185 std::swap(a&: LhsReg, b&: RhsReg);
1186
1187 // Emit a compare of LHS/RHS.
1188 MachineInstr &CmpInst =
1189 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1190 .addReg(RegNo: LhsReg)
1191 .addReg(RegNo: RhsReg);
1192
1193 MachineInstr &Set =
1194 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: ResultReg).addImm(Val: CC);
1195 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1196 constrainSelectedInstRegOperands(I&: Set, TII, TRI, RBI);
1197 I.eraseFromParent();
1198 return true;
1199}
1200
1201bool X86InstructionSelector::selectUAddSub(MachineInstr &I,
1202 MachineRegisterInfo &MRI,
1203 MachineFunction &MF) const {
1204 assert((I.getOpcode() == TargetOpcode::G_UADDE ||
1205 I.getOpcode() == TargetOpcode::G_UADDO ||
1206 I.getOpcode() == TargetOpcode::G_USUBE ||
1207 I.getOpcode() == TargetOpcode::G_USUBO) &&
1208 "unexpected instruction");
1209
1210 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
1211
1212 const Register DstReg = CarryMI.getDstReg();
1213 const Register CarryOutReg = CarryMI.getCarryOutReg();
1214 const Register Op0Reg = CarryMI.getLHSReg();
1215 const Register Op1Reg = CarryMI.getRHSReg();
1216 bool IsSub = CarryMI.isSub();
1217
1218 const LLT DstTy = MRI.getType(Reg: DstReg);
1219 assert(DstTy.isScalar() && "selectUAddSub only supported for scalar types");
1220
1221 // TODO: Handle immediate argument variants?
1222 unsigned OpADC, OpADD, OpSBB, OpSUB;
1223 switch (DstTy.getSizeInBits()) {
1224 case 8:
1225 OpADC = X86::ADC8rr;
1226 OpADD = X86::ADD8rr;
1227 OpSBB = X86::SBB8rr;
1228 OpSUB = X86::SUB8rr;
1229 break;
1230 case 16:
1231 OpADC = X86::ADC16rr;
1232 OpADD = X86::ADD16rr;
1233 OpSBB = X86::SBB16rr;
1234 OpSUB = X86::SUB16rr;
1235 break;
1236 case 32:
1237 OpADC = X86::ADC32rr;
1238 OpADD = X86::ADD32rr;
1239 OpSBB = X86::SBB32rr;
1240 OpSUB = X86::SUB32rr;
1241 break;
1242 case 64:
1243 OpADC = X86::ADC64rr;
1244 OpADD = X86::ADD64rr;
1245 OpSBB = X86::SBB64rr;
1246 OpSUB = X86::SUB64rr;
1247 break;
1248 default:
1249 llvm_unreachable("selectUAddSub unsupported type.");
1250 }
1251
1252 const RegisterBank &CarryRB = *RBI.getRegBank(Reg: CarryOutReg, MRI, TRI);
1253 const TargetRegisterClass *CarryRC =
1254 getRegClass(Ty: MRI.getType(Reg: CarryOutReg), RB: CarryRB);
1255
1256 unsigned Opcode = IsSub ? OpSUB : OpADD;
1257
1258 // G_UADDE/G_USUBE - find CarryIn def instruction.
1259 if (auto CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
1260 Register CarryInReg = CarryInMI->getCarryInReg();
1261 MachineInstr *Def = MRI.getVRegDef(Reg: CarryInReg);
1262 while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1263 CarryInReg = Def->getOperand(i: 1).getReg();
1264 Def = MRI.getVRegDef(Reg: CarryInReg);
1265 }
1266
1267 // TODO - handle more CF generating instructions
1268 if (Def->getOpcode() == TargetOpcode::G_UADDE ||
1269 Def->getOpcode() == TargetOpcode::G_UADDO ||
1270 Def->getOpcode() == TargetOpcode::G_USUBE ||
1271 Def->getOpcode() == TargetOpcode::G_USUBO) {
1272 // The carry-in is a SETB byte (0 or 1) from a chained add/sub.
1273 // Materialize EFLAGS.CF from that byte for the following ADC/SBB
1274 // by emitting NEG, which sets CF iff its operand is non-zero.
1275 if (!RBI.constrainGenericRegister(Reg: CarryInReg, RC: *CarryRC, MRI))
1276 return false;
1277
1278 Register NegDef = MRI.createVirtualRegister(RegClass: CarryRC);
1279 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::NEG8r), DestReg: NegDef)
1280 .addReg(RegNo: CarryInReg);
1281
1282 Opcode = IsSub ? OpSBB : OpADC;
1283 } else if (auto val = getIConstantVRegVal(VReg: CarryInReg, MRI)) {
1284 // carry is constant, support only 0.
1285 if (*val != 0)
1286 return false;
1287
1288 Opcode = IsSub ? OpSUB : OpADD;
1289 } else
1290 return false;
1291 }
1292
1293 MachineInstr &Inst =
1294 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1295 .addReg(RegNo: Op0Reg)
1296 .addReg(RegNo: Op1Reg);
1297
1298 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: CarryOutReg)
1299 .addImm(Val: X86::COND_B);
1300
1301 constrainSelectedInstRegOperands(I&: Inst, TII, TRI, RBI);
1302 if (!RBI.constrainGenericRegister(Reg: CarryOutReg, RC: *CarryRC, MRI))
1303 return false;
1304
1305 I.eraseFromParent();
1306 return true;
1307}
1308
1309bool X86InstructionSelector::selectExtract(MachineInstr &I,
1310 MachineRegisterInfo &MRI,
1311 MachineFunction &MF) const {
1312 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1313 "unexpected instruction");
1314
1315 const Register DstReg = I.getOperand(i: 0).getReg();
1316 const Register SrcReg = I.getOperand(i: 1).getReg();
1317 int64_t Index = I.getOperand(i: 2).getImm();
1318
1319 const LLT DstTy = MRI.getType(Reg: DstReg);
1320 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1321
1322 // Meanwile handle vector type only.
1323 if (!DstTy.isVector())
1324 return false;
1325
1326 if (Index % DstTy.getSizeInBits() != 0)
1327 return false; // Not extract subvector.
1328
1329 if (Index == 0) {
1330 // Replace by extract subreg copy.
1331 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1332 return false;
1333
1334 I.eraseFromParent();
1335 return true;
1336 }
1337
1338 bool HasAVX = STI.hasAVX();
1339 bool HasAVX512 = STI.hasAVX512();
1340 bool HasVLX = STI.hasVLX();
1341
1342 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1343 if (HasVLX)
1344 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Z256rri));
1345 else if (HasAVX)
1346 I.setDesc(TII.get(Opcode: X86::VEXTRACTF128rri));
1347 else
1348 return false;
1349 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1350 if (DstTy.getSizeInBits() == 128)
1351 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Zrri));
1352 else if (DstTy.getSizeInBits() == 256)
1353 I.setDesc(TII.get(Opcode: X86::VEXTRACTF64X4Zrri));
1354 else
1355 return false;
1356 } else
1357 return false;
1358
1359 // Convert to X86 VEXTRACT immediate.
1360 Index = Index / DstTy.getSizeInBits();
1361 I.getOperand(i: 2).setImm(Index);
1362
1363 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1364 return true;
1365}
1366
1367bool X86InstructionSelector::emitExtractSubreg(Register DstReg, Register SrcReg,
1368 MachineInstr &I,
1369 MachineRegisterInfo &MRI,
1370 MachineFunction &MF) const {
1371 const LLT DstTy = MRI.getType(Reg: DstReg);
1372 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1373 unsigned SubIdx = X86::NoSubRegister;
1374
1375 if (!DstTy.isVector() || !SrcTy.isVector())
1376 return false;
1377
1378 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1379 "Incorrect Src/Dst register size");
1380
1381 if (DstTy.getSizeInBits() == 128)
1382 SubIdx = X86::sub_xmm;
1383 else if (DstTy.getSizeInBits() == 256)
1384 SubIdx = X86::sub_ymm;
1385 else
1386 return false;
1387
1388 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1389 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1390
1391 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
1392
1393 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1394 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1395 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1396 return false;
1397 }
1398
1399 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::COPY), DestReg: DstReg)
1400 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubIdx);
1401
1402 return true;
1403}
1404
1405bool X86InstructionSelector::emitInsertSubreg(Register DstReg, Register SrcReg,
1406 MachineInstr &I,
1407 MachineRegisterInfo &MRI,
1408 MachineFunction &MF) const {
1409 const LLT DstTy = MRI.getType(Reg: DstReg);
1410 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1411 unsigned SubIdx = X86::NoSubRegister;
1412
1413 // TODO: support scalar types
1414 if (!DstTy.isVector() || !SrcTy.isVector())
1415 return false;
1416
1417 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1418 "Incorrect Src/Dst register size");
1419
1420 if (SrcTy.getSizeInBits() == 128)
1421 SubIdx = X86::sub_xmm;
1422 else if (SrcTy.getSizeInBits() == 256)
1423 SubIdx = X86::sub_ymm;
1424 else
1425 return false;
1426
1427 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1428 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1429
1430 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1431 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1432 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1433 return false;
1434 }
1435
1436 Register ImpDefReg = MRI.createVirtualRegister(RegClass: DstRC);
1437 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::IMPLICIT_DEF),
1438 DestReg: ImpDefReg);
1439
1440 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::INSERT_SUBREG),
1441 DestReg: DstReg)
1442 .addReg(RegNo: ImpDefReg)
1443 .addReg(RegNo: SrcReg)
1444 .addImm(Val: SubIdx);
1445
1446 return true;
1447}
1448
1449bool X86InstructionSelector::selectInsert(MachineInstr &I,
1450 MachineRegisterInfo &MRI,
1451 MachineFunction &MF) const {
1452 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1453
1454 const Register DstReg = I.getOperand(i: 0).getReg();
1455 const Register SrcReg = I.getOperand(i: 1).getReg();
1456 const Register InsertReg = I.getOperand(i: 2).getReg();
1457 int64_t Index = I.getOperand(i: 3).getImm();
1458
1459 const LLT DstTy = MRI.getType(Reg: DstReg);
1460 const LLT InsertRegTy = MRI.getType(Reg: InsertReg);
1461
1462 // Meanwile handle vector type only.
1463 if (!DstTy.isVector())
1464 return false;
1465
1466 if (Index % InsertRegTy.getSizeInBits() != 0)
1467 return false; // Not insert subvector.
1468
1469 if (Index == 0 && mi_match(R: SrcReg, MRI, P: m_GImplicitDef())) {
1470 // Replace by subreg copy.
1471 if (!emitInsertSubreg(DstReg, SrcReg: InsertReg, I, MRI, MF))
1472 return false;
1473
1474 I.eraseFromParent();
1475 return true;
1476 }
1477
1478 bool HasAVX = STI.hasAVX();
1479 bool HasAVX512 = STI.hasAVX512();
1480 bool HasVLX = STI.hasVLX();
1481
1482 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1483 if (HasVLX)
1484 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Z256rri));
1485 else if (HasAVX)
1486 I.setDesc(TII.get(Opcode: X86::VINSERTF128rri));
1487 else
1488 return false;
1489 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1490 if (InsertRegTy.getSizeInBits() == 128)
1491 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Zrri));
1492 else if (InsertRegTy.getSizeInBits() == 256)
1493 I.setDesc(TII.get(Opcode: X86::VINSERTF64X4Zrri));
1494 else
1495 return false;
1496 } else
1497 return false;
1498
1499 // Convert to X86 VINSERT immediate.
1500 Index = Index / InsertRegTy.getSizeInBits();
1501
1502 I.getOperand(i: 3).setImm(Index);
1503
1504 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1505 return true;
1506}
1507
1508bool X86InstructionSelector::selectUnmergeValues(
1509 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1510 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1511 "unexpected instruction");
1512
1513 // Split to extracts.
1514 unsigned NumDefs = I.getNumOperands() - 1;
1515 Register SrcReg = I.getOperand(i: NumDefs).getReg();
1516 unsigned DefSize = MRI.getType(Reg: I.getOperand(i: 0).getReg()).getSizeInBits();
1517
1518 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1519 MachineInstr &ExtrInst =
1520 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1521 MCID: TII.get(Opcode: TargetOpcode::G_EXTRACT), DestReg: I.getOperand(i: Idx).getReg())
1522 .addReg(RegNo: SrcReg)
1523 .addImm(Val: Idx * DefSize);
1524
1525 if (!select(I&: ExtrInst))
1526 return false;
1527 }
1528
1529 I.eraseFromParent();
1530 return true;
1531}
1532
1533bool X86InstructionSelector::selectMergeValues(
1534 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1535 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1536 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1537 "unexpected instruction");
1538
1539 // Split to inserts.
1540 Register DstReg = I.getOperand(i: 0).getReg();
1541 Register SrcReg0 = I.getOperand(i: 1).getReg();
1542
1543 const LLT DstTy = MRI.getType(Reg: DstReg);
1544 const LLT SrcTy = MRI.getType(Reg: SrcReg0);
1545 unsigned SrcSize = SrcTy.getSizeInBits();
1546
1547 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1548
1549 // For the first src use insertSubReg.
1550 Register DefReg = MRI.createGenericVirtualRegister(Ty: DstTy);
1551 MRI.setRegBank(Reg: DefReg, RegBank);
1552 if (!emitInsertSubreg(DstReg: DefReg, SrcReg: I.getOperand(i: 1).getReg(), I, MRI, MF))
1553 return false;
1554
1555 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1556 Register Tmp = MRI.createGenericVirtualRegister(Ty: DstTy);
1557 MRI.setRegBank(Reg: Tmp, RegBank);
1558
1559 MachineInstr &InsertInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1560 MCID: TII.get(Opcode: TargetOpcode::G_INSERT), DestReg: Tmp)
1561 .addReg(RegNo: DefReg)
1562 .addReg(RegNo: I.getOperand(i: Idx).getReg())
1563 .addImm(Val: (Idx - 1) * SrcSize);
1564
1565 DefReg = Tmp;
1566
1567 if (!select(I&: InsertInst))
1568 return false;
1569 }
1570
1571 MachineInstr &CopyInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1572 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
1573 .addReg(RegNo: DefReg);
1574
1575 if (!select(I&: CopyInst))
1576 return false;
1577
1578 I.eraseFromParent();
1579 return true;
1580}
1581
1582bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1583 MachineRegisterInfo &MRI,
1584 MachineFunction &MF) const {
1585 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1586
1587 const Register CondReg = I.getOperand(i: 0).getReg();
1588 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1589
1590 MachineInstr &TestInst =
1591 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST8ri))
1592 .addReg(RegNo: CondReg)
1593 .addImm(Val: 1);
1594 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::JCC_1))
1595 .addMBB(MBB: DestMBB).addImm(Val: X86::COND_NE);
1596
1597 constrainSelectedInstRegOperands(I&: TestInst, TII, TRI, RBI);
1598
1599 I.eraseFromParent();
1600 return true;
1601}
1602
1603bool X86InstructionSelector::materializeFP(MachineInstr &I,
1604 MachineRegisterInfo &MRI,
1605 MachineFunction &MF) const {
1606 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1607 "unexpected instruction");
1608
1609 // Can't handle alternate code models yet.
1610 CodeModel::Model CM = TM.getCodeModel();
1611 if (CM != CodeModel::Small && CM != CodeModel::Large)
1612 return false;
1613
1614 const Register DstReg = I.getOperand(i: 0).getReg();
1615 const LLT DstTy = MRI.getType(Reg: DstReg);
1616 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1617 // Create the load from the constant pool.
1618 const ConstantFP *CFP = I.getOperand(i: 1).getFPImm();
1619 const auto &DL = MF.getDataLayout();
1620 Align Alignment = DL.getPrefTypeAlign(Ty: CFP->getType());
1621 const DebugLoc &DbgLoc = I.getDebugLoc();
1622
1623 unsigned Opc =
1624 getLoadStoreOp(Ty: DstTy, RB: RegBank, Opc: TargetOpcode::G_LOAD, Alignment);
1625
1626 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(C: CFP, Alignment);
1627 MachineInstr *LoadInst = nullptr;
1628 unsigned char OpFlag = STI.classifyLocalReference(GV: nullptr);
1629
1630 if (CM == CodeModel::Large && STI.is64Bit()) {
1631 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1632 // they cannot be folded into immediate fields.
1633
1634 Register AddrReg = MRI.createVirtualRegister(RegClass: &X86::GR64RegClass);
1635 BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: AddrReg)
1636 .addConstantPoolIndex(Idx: CPI, Offset: 0, TargetFlags: OpFlag);
1637
1638 MachineMemOperand *MMO = MF.getMachineMemOperand(
1639 PtrInfo: MachinePointerInfo::getConstantPool(MF), F: MachineMemOperand::MOLoad,
1640 MemTy: LLT::pointer(AddressSpace: 0, SizeInBits: DL.getPointerSizeInBits()), BaseAlignment: Alignment);
1641
1642 LoadInst =
1643 addDirectMem(MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg),
1644 Reg: AddrReg)
1645 .addMemOperand(MMO);
1646
1647 } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1648 // Handle the case when globals fit in our immediate field.
1649 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1650
1651 // x86-32 PIC requires a PIC base register for constant pools.
1652 unsigned PICBase = 0;
1653 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1654 // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1655 // In DAGISEL the code that initialize it generated by the CGBR pass.
1656 return false; // TODO support the mode.
1657 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1658 PICBase = X86::RIP;
1659
1660 LoadInst = addConstantPoolReference(
1661 MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg), CPI, GlobalBaseReg: PICBase,
1662 OpFlags: OpFlag);
1663 } else
1664 return false;
1665
1666 constrainSelectedInstRegOperands(I&: *LoadInst, TII, TRI, RBI);
1667 I.eraseFromParent();
1668 return true;
1669}
1670
1671bool X86InstructionSelector::selectImplicitDefOrPHI(
1672 MachineInstr &I, MachineRegisterInfo &MRI) const {
1673 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1674 I.getOpcode() == TargetOpcode::G_PHI) &&
1675 "unexpected instruction");
1676
1677 Register DstReg = I.getOperand(i: 0).getReg();
1678
1679 if (!MRI.getRegClassOrNull(Reg: DstReg)) {
1680 const LLT DstTy = MRI.getType(Reg: DstReg);
1681 const TargetRegisterClass *RC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1682
1683 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI)) {
1684 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1685 << " operand\n");
1686 return false;
1687 }
1688 }
1689
1690 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1691 I.setDesc(TII.get(Opcode: X86::IMPLICIT_DEF));
1692 else
1693 I.setDesc(TII.get(Opcode: X86::PHI));
1694
1695 return true;
1696}
1697
1698bool X86InstructionSelector::selectMulDivRem(MachineInstr &I,
1699 MachineRegisterInfo &MRI,
1700 MachineFunction &MF) const {
1701 // The implementation of this function is adapted from X86FastISel.
1702 assert((I.getOpcode() == TargetOpcode::G_MUL ||
1703 I.getOpcode() == TargetOpcode::G_SMULH ||
1704 I.getOpcode() == TargetOpcode::G_UMULH ||
1705 I.getOpcode() == TargetOpcode::G_SDIV ||
1706 I.getOpcode() == TargetOpcode::G_SREM ||
1707 I.getOpcode() == TargetOpcode::G_UDIV ||
1708 I.getOpcode() == TargetOpcode::G_UREM) &&
1709 "unexpected instruction");
1710
1711 const Register DstReg = I.getOperand(i: 0).getReg();
1712 const Register Op1Reg = I.getOperand(i: 1).getReg();
1713 const Register Op2Reg = I.getOperand(i: 2).getReg();
1714
1715 const LLT RegTy = MRI.getType(Reg: DstReg);
1716 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1717 "Arguments and return value types must match");
1718
1719 const RegisterBank *RegRB = RBI.getRegBank(Reg: DstReg, MRI, TRI);
1720 if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1721 return false;
1722
1723 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1724 const static unsigned NumOps = 7; // SDiv/SRem/UDiv/URem/Mul/SMulH/UMulh
1725 const static bool S = true; // IsSigned
1726 const static bool U = false; // !IsSigned
1727 const static unsigned Copy = TargetOpcode::COPY;
1728
1729 // For the X86 IDIV instruction, in most cases the dividend
1730 // (numerator) must be in a specific register pair highreg:lowreg,
1731 // producing the quotient in lowreg and the remainder in highreg.
1732 // For most data types, to set up the instruction, the dividend is
1733 // copied into lowreg, and lowreg is sign-extended into highreg. The
1734 // exception is i8, where the dividend is defined as a single register rather
1735 // than a register pair, and we therefore directly sign-extend the dividend
1736 // into lowreg, instead of copying, and ignore the highreg.
1737 const static struct MulDivRemEntry {
1738 // The following portion depends only on the data type.
1739 unsigned SizeInBits;
1740 unsigned LowInReg; // low part of the register pair
1741 unsigned HighInReg; // high part of the register pair
1742 // The following portion depends on both the data type and the operation.
1743 struct MulDivRemResult {
1744 unsigned OpMulDivRem; // The specific MUL/DIV opcode to use.
1745 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1746 // highreg, or copying a zero into highreg.
1747 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1748 // zero/sign-extending into lowreg for i8.
1749 unsigned ResultReg; // Register containing the desired result.
1750 bool IsOpSigned; // Whether to use signed or unsigned form.
1751 } ResultTable[NumOps];
1752 } OpTable[NumTypes] = {
1753 {.SizeInBits: 8,
1754 .LowInReg: X86::AX,
1755 .HighInReg: 0,
1756 .ResultTable: {
1757 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // SDiv
1758 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SRem
1759 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AL, .IsOpSigned: U}, // UDiv
1760 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // URem
1761 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // Mul
1762 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SMulH
1763 {.OpMulDivRem: X86::MUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // UMulH
1764 }}, // i8
1765 {.SizeInBits: 16,
1766 .LowInReg: X86::AX,
1767 .HighInReg: X86::DX,
1768 .ResultTable: {
1769 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // SDiv
1770 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SRem
1771 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: U}, // UDiv
1772 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // URem
1773 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // Mul
1774 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SMulH
1775 {.OpMulDivRem: X86::MUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // UMulH
1776 }}, // i16
1777 {.SizeInBits: 32,
1778 .LowInReg: X86::EAX,
1779 .HighInReg: X86::EDX,
1780 .ResultTable: {
1781 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // SDiv
1782 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SRem
1783 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: U}, // UDiv
1784 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // URem
1785 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // Mul
1786 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SMulH
1787 {.OpMulDivRem: X86::MUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // UMulH
1788 }}, // i32
1789 {.SizeInBits: 64,
1790 .LowInReg: X86::RAX,
1791 .HighInReg: X86::RDX,
1792 .ResultTable: {
1793 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // SDiv
1794 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SRem
1795 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: U}, // UDiv
1796 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // URem
1797 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // Mul
1798 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SMulH
1799 {.OpMulDivRem: X86::MUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // UMulH
1800 }}, // i64
1801 };
1802
1803 auto OpEntryIt = llvm::find_if(Range: OpTable, P: [RegTy](const MulDivRemEntry &El) {
1804 return El.SizeInBits == RegTy.getSizeInBits();
1805 });
1806 if (OpEntryIt == std::end(arr: OpTable))
1807 return false;
1808
1809 unsigned OpIndex;
1810 switch (I.getOpcode()) {
1811 default:
1812 llvm_unreachable("Unexpected mul/div/rem opcode");
1813 case TargetOpcode::G_SDIV:
1814 OpIndex = 0;
1815 break;
1816 case TargetOpcode::G_SREM:
1817 OpIndex = 1;
1818 break;
1819 case TargetOpcode::G_UDIV:
1820 OpIndex = 2;
1821 break;
1822 case TargetOpcode::G_UREM:
1823 OpIndex = 3;
1824 break;
1825 case TargetOpcode::G_MUL:
1826 OpIndex = 4;
1827 break;
1828 case TargetOpcode::G_SMULH:
1829 OpIndex = 5;
1830 break;
1831 case TargetOpcode::G_UMULH:
1832 OpIndex = 6;
1833 break;
1834 }
1835
1836 const MulDivRemEntry &TypeEntry = *OpEntryIt;
1837 const MulDivRemEntry::MulDivRemResult &OpEntry =
1838 TypeEntry.ResultTable[OpIndex];
1839
1840 const TargetRegisterClass *RegRC = getRegClass(Ty: RegTy, RB: *RegRB);
1841 if (!RBI.constrainGenericRegister(Reg: Op1Reg, RC: *RegRC, MRI) ||
1842 !RBI.constrainGenericRegister(Reg: Op2Reg, RC: *RegRC, MRI) ||
1843 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RegRC, MRI)) {
1844 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1845 << " operand\n");
1846 return false;
1847 }
1848
1849 // Move op1 into low-order input register.
1850 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpCopy),
1851 DestReg: TypeEntry.LowInReg)
1852 .addReg(RegNo: Op1Reg);
1853
1854 // Zero-extend or sign-extend into high-order input register.
1855 if (OpEntry.OpSignExtend) {
1856 if (OpEntry.IsOpSigned)
1857 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1858 MCID: TII.get(Opcode: OpEntry.OpSignExtend));
1859 else {
1860 Register Zero32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
1861 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::MOV32r0),
1862 DestReg: Zero32);
1863
1864 // Copy the zero into the appropriate sub/super/identical physical
1865 // register. Unfortunately the operations needed are not uniform enough
1866 // to fit neatly into the table above.
1867 if (RegTy.getSizeInBits() == 16) {
1868 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1869 DestReg: TypeEntry.HighInReg)
1870 .addReg(RegNo: Zero32, Flags: {}, SubReg: X86::sub_16bit);
1871 } else if (RegTy.getSizeInBits() == 32) {
1872 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1873 DestReg: TypeEntry.HighInReg)
1874 .addReg(RegNo: Zero32);
1875 } else if (RegTy.getSizeInBits() == 64) {
1876 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1877 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: TypeEntry.HighInReg)
1878 .addReg(RegNo: Zero32)
1879 .addImm(Val: X86::sub_32bit);
1880 }
1881 }
1882 }
1883
1884 // Generate the DIV/IDIV/MUL/IMUL instruction.
1885 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpMulDivRem))
1886 .addReg(RegNo: Op2Reg);
1887
1888 // For i8 remainder, we can't reference ah directly, as we'll end
1889 // up with bogus copies like %r9b = COPY %ah. Reference ax
1890 // instead to prevent ah references in a rex instruction.
1891 //
1892 // The current assumption of the fast register allocator is that isel
1893 // won't generate explicit references to the GR8_NOREX registers. If
1894 // the allocator and/or the backend get enhanced to be more robust in
1895 // that regard, this can be, and should be, removed.
1896 if (OpEntry.ResultReg == X86::AH && STI.is64Bit()) {
1897 Register SourceSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1898 Register ResultSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1899 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy), DestReg: SourceSuperReg)
1900 .addReg(RegNo: X86::AX);
1901
1902 // Shift AX right by 8 bits instead of using AH.
1903 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SHR16ri),
1904 DestReg: ResultSuperReg)
1905 .addReg(RegNo: SourceSuperReg)
1906 .addImm(Val: 8);
1907
1908 // Now reference the 8-bit subreg of the result.
1909 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1910 DestReg: DstReg)
1911 .addReg(RegNo: ResultSuperReg, Flags: {}, SubReg: X86::sub_8bit);
1912 } else {
1913 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1914 DestReg: DstReg)
1915 .addReg(RegNo: OpEntry.ResultReg);
1916 }
1917 I.eraseFromParent();
1918
1919 return true;
1920}
1921
1922bool X86InstructionSelector::selectSelect(MachineInstr &I,
1923 MachineRegisterInfo &MRI,
1924 MachineFunction &MF) const {
1925 GSelect &Sel = cast<GSelect>(Val&: I);
1926 Register DstReg = Sel.getReg(Idx: 0);
1927 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST32rr))
1928 .addReg(RegNo: Sel.getCondReg())
1929 .addReg(RegNo: Sel.getCondReg());
1930
1931 unsigned OpCmp;
1932 LLT Ty = MRI.getType(Reg: DstReg);
1933 if (Ty.getSizeInBits() == 80) {
1934 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::CMOVE_Fp80),
1935 DestReg: DstReg)
1936 .addReg(RegNo: Sel.getTrueReg())
1937 .addReg(RegNo: Sel.getFalseReg());
1938 } else {
1939 switch (Ty.getSizeInBits()) {
1940 default:
1941 return false;
1942 case 8:
1943 OpCmp = X86::CMOV_GR8;
1944 break;
1945 case 16:
1946 OpCmp = STI.canUseCMOV() ? X86::CMOV16rr : X86::CMOV_GR16;
1947 break;
1948 case 32:
1949 OpCmp = STI.canUseCMOV() ? X86::CMOV32rr : X86::CMOV_GR32;
1950 break;
1951 case 64:
1952 assert(STI.is64Bit() && STI.canUseCMOV());
1953 OpCmp = X86::CMOV64rr;
1954 break;
1955 }
1956 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: OpCmp), DestReg: DstReg)
1957 .addReg(RegNo: Sel.getTrueReg())
1958 .addReg(RegNo: Sel.getFalseReg())
1959 .addImm(Val: X86::COND_E);
1960 }
1961 const TargetRegisterClass *DstRC = getRegClass(Ty, Reg: DstReg, MRI);
1962 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1963 LLVM_DEBUG(dbgs() << "Failed to constrain CMOV\n");
1964 return false;
1965 }
1966
1967 Sel.eraseFromParent();
1968 return true;
1969}
1970
1971InstructionSelector::ComplexRendererFns
1972X86InstructionSelector::selectAddr(MachineOperand &Root) const {
1973 MachineInstr *MI = Root.getParent();
1974 MachineIRBuilder MIRBuilder(*MI);
1975
1976 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
1977 MachineInstr *Ptr = MRI.getVRegDef(Reg: Root.getReg());
1978 X86AddressMode AM;
1979 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
1980 return std::nullopt;
1981
1982 if (AM.IndexReg)
1983 return std::nullopt;
1984
1985 return {// Base
1986 {[=](MachineInstrBuilder &MIB) {
1987 if (AM.BaseType == X86AddressMode::RegBase)
1988 MIB.addUse(RegNo: AM.Base.Reg);
1989 else {
1990 assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
1991 "Unknown type of address base");
1992 MIB.addFrameIndex(Idx: AM.Base.FrameIndex);
1993 }
1994 },
1995 // Scale
1996 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: AM.Scale); },
1997 // Index
1998 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); },
1999 // Disp
2000 [=](MachineInstrBuilder &MIB) {
2001 if (AM.GV)
2002 MIB.addGlobalAddress(GV: AM.GV, Offset: AM.Disp, TargetFlags: AM.GVOpFlags);
2003 else if (AM.CP)
2004 MIB.addConstantPoolIndex(Idx: AM.Disp, Offset: 0, TargetFlags: AM.GVOpFlags);
2005 else
2006 MIB.addImm(Val: AM.Disp);
2007 },
2008 // Segment
2009 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); }}};
2010}
2011
2012InstructionSelector *
2013llvm::createX86InstructionSelector(const X86TargetMachine &TM,
2014 const X86Subtarget &Subtarget,
2015 const X86RegisterBankInfo &RBI) {
2016 return new X86InstructionSelector(TM, Subtarget, RBI);
2017}
2018