1//===- X86InstructionSelector.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// X86.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86BaseInfo.h"
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86RegisterBankInfo.h"
19#include "X86RegisterInfo.h"
20#include "X86Subtarget.h"
21#include "X86TargetMachine.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/Utils.h"
27#include "llvm/CodeGen/MachineBasicBlock.h"
28#include "llvm/CodeGen/MachineConstantPool.h"
29#include "llvm/CodeGen/MachineFunction.h"
30#include "llvm/CodeGen/MachineInstr.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/MachineOperand.h"
34#include "llvm/CodeGen/MachineRegisterInfo.h"
35#include "llvm/CodeGen/RegisterBank.h"
36#include "llvm/CodeGen/TargetOpcodes.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/CodeGenTypes/LowLevelType.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/InstrTypes.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/Support/CodeGen.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MathExtras.h"
46#include "llvm/Support/raw_ostream.h"
47#include <cassert>
48#include <cstdint>
49#include <tuple>
50
51#define DEBUG_TYPE "X86-isel"
52
53using namespace llvm;
54using namespace MIPatternMatch;
55
56namespace {
57
58#define GET_GLOBALISEL_PREDICATE_BITSET
59#include "X86GenGlobalISel.inc"
60#undef GET_GLOBALISEL_PREDICATE_BITSET
61
62class X86InstructionSelector : public InstructionSelector {
63public:
64 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
65 const X86RegisterBankInfo &RBI);
66
67 bool select(MachineInstr &I) override;
68 static const char *getName() { return DEBUG_TYPE; }
69
70private:
71 /// tblgen-erated 'select' implementation, used as the initial selector for
72 /// the patterns that don't require complex C++.
73 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
74
75 // TODO: remove after supported by Tablegen-erated instruction selection.
76 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
77 Align Alignment) const;
78 // TODO: remove once p0<->i32/i64 matching is available
79 unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
80 unsigned Opc) const;
81
82 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
83 MachineFunction &MF) const;
84 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
85 MachineFunction &MF) const;
86 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
87 MachineFunction &MF) const;
88 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
89 MachineFunction &MF) const;
90 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
91 MachineFunction &MF) const;
92 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
93 MachineFunction &MF) const;
94 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
95 MachineFunction &MF) const;
96 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
97 MachineFunction &MF) const;
98 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
99 MachineFunction &MF) const;
100 bool selectUAddSub(MachineInstr &I, MachineRegisterInfo &MRI,
101 MachineFunction &MF) const;
102 bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI) const;
103 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
104 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
105 MachineFunction &MF);
106 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
107 MachineFunction &MF);
108 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
109 MachineFunction &MF) const;
110 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
111 MachineFunction &MF) const;
112 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
113 MachineFunction &MF) const;
114 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
115 const Register DstReg,
116 const TargetRegisterClass *DstRC,
117 const Register SrcReg,
118 const TargetRegisterClass *SrcRC) const;
119 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
120 MachineFunction &MF) const;
121 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
122 bool selectMulDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
123 MachineFunction &MF) const;
124 bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
125 MachineFunction &MF) const;
126
127 ComplexRendererFns selectAddr(MachineOperand &Root) const;
128
129 // emit insert subreg instruction and insert it before MachineInstr &I
130 bool emitInsertSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
131 MachineRegisterInfo &MRI, MachineFunction &MF) const;
132 // emit extract subreg instruction and insert it before MachineInstr &I
133 bool emitExtractSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
134 MachineRegisterInfo &MRI, MachineFunction &MF) const;
135
136 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
137 const TargetRegisterClass *getRegClass(LLT Ty, Register Reg,
138 MachineRegisterInfo &MRI) const;
139
140 const X86TargetMachine &TM;
141 const X86Subtarget &STI;
142 const X86InstrInfo &TII;
143 const X86RegisterInfo &TRI;
144 const X86RegisterBankInfo &RBI;
145
146#define GET_GLOBALISEL_PREDICATES_DECL
147#include "X86GenGlobalISel.inc"
148#undef GET_GLOBALISEL_PREDICATES_DECL
149
150#define GET_GLOBALISEL_TEMPORARIES_DECL
151#include "X86GenGlobalISel.inc"
152#undef GET_GLOBALISEL_TEMPORARIES_DECL
153};
154
155} // end anonymous namespace
156
157#define GET_GLOBALISEL_IMPL
158#include "X86GenGlobalISel.inc"
159#undef GET_GLOBALISEL_IMPL
160
161X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
162 const X86Subtarget &STI,
163 const X86RegisterBankInfo &RBI)
164 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
165 RBI(RBI),
166#define GET_GLOBALISEL_PREDICATES_INIT
167#include "X86GenGlobalISel.inc"
168#undef GET_GLOBALISEL_PREDICATES_INIT
169#define GET_GLOBALISEL_TEMPORARIES_INIT
170#include "X86GenGlobalISel.inc"
171#undef GET_GLOBALISEL_TEMPORARIES_INIT
172{
173}
174
175// FIXME: This should be target-independent, inferred from the types declared
176// for each class in the bank.
177const TargetRegisterClass *
178X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
179 if (RB.getID() == X86::GPRRegBankID) {
180 if (Ty.getSizeInBits() <= 8)
181 return &X86::GR8RegClass;
182 if (Ty.getSizeInBits() == 16)
183 return &X86::GR16RegClass;
184 if (Ty.getSizeInBits() == 32)
185 return &X86::GR32RegClass;
186 if (Ty.getSizeInBits() == 64)
187 return &X86::GR64RegClass;
188 }
189 if (RB.getID() == X86::VECRRegBankID) {
190 if (Ty.getSizeInBits() == 16)
191 return STI.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
192 if (Ty.getSizeInBits() == 32)
193 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
194 if (Ty.getSizeInBits() == 64)
195 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
196 if (Ty.getSizeInBits() == 128)
197 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
198 if (Ty.getSizeInBits() == 256)
199 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
200 if (Ty.getSizeInBits() == 512)
201 return &X86::VR512RegClass;
202 }
203
204 if (RB.getID() == X86::PSRRegBankID) {
205 if (Ty.getSizeInBits() == 80)
206 return &X86::RFP80RegClass;
207 if (Ty.getSizeInBits() == 64)
208 return &X86::RFP64RegClass;
209 if (Ty.getSizeInBits() == 32)
210 return &X86::RFP32RegClass;
211 }
212
213 llvm_unreachable("Unknown RegBank!");
214}
215
216const TargetRegisterClass *
217X86InstructionSelector::getRegClass(LLT Ty, Register Reg,
218 MachineRegisterInfo &MRI) const {
219 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
220 return getRegClass(Ty, RB: RegBank);
221}
222
223static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
224 unsigned SubIdx = X86::NoSubRegister;
225 if (RC == &X86::GR32RegClass) {
226 SubIdx = X86::sub_32bit;
227 } else if (RC == &X86::GR16RegClass) {
228 SubIdx = X86::sub_16bit;
229 } else if (RC == &X86::GR8RegClass) {
230 SubIdx = X86::sub_8bit;
231 }
232
233 return SubIdx;
234}
235
236static const TargetRegisterClass *getRegClassFromGRPhysReg(Register Reg) {
237 assert(Reg.isPhysical());
238 if (X86::GR64RegClass.contains(Reg))
239 return &X86::GR64RegClass;
240 if (X86::GR32RegClass.contains(Reg))
241 return &X86::GR32RegClass;
242 if (X86::GR16RegClass.contains(Reg))
243 return &X86::GR16RegClass;
244 if (X86::GR8RegClass.contains(Reg))
245 return &X86::GR8RegClass;
246
247 llvm_unreachable("Unknown RegClass for PhysReg!");
248}
249
250// FIXME: We need some sort of API in RBI/TRI to allow generic code to
251// constrain operands of simple instructions given a TargetRegisterClass
252// and LLT
253bool X86InstructionSelector::selectDebugInstr(MachineInstr &I,
254 MachineRegisterInfo &MRI) const {
255 for (MachineOperand &MO : I.operands()) {
256 if (!MO.isReg())
257 continue;
258 Register Reg = MO.getReg();
259 if (!Reg)
260 continue;
261 if (Reg.isPhysical())
262 continue;
263 LLT Ty = MRI.getType(Reg);
264 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
265 const TargetRegisterClass *RC =
266 dyn_cast_if_present<const TargetRegisterClass *>(Val: RegClassOrBank);
267 if (!RC) {
268 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
269 RC = getRegClass(Ty, RB);
270 if (!RC) {
271 LLVM_DEBUG(
272 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
273 break;
274 }
275 }
276 RBI.constrainGenericRegister(Reg, RC: *RC, MRI);
277 }
278
279 return true;
280}
281
282// Set X86 Opcode and constrain DestReg.
283bool X86InstructionSelector::selectCopy(MachineInstr &I,
284 MachineRegisterInfo &MRI) const {
285 Register DstReg = I.getOperand(i: 0).getReg();
286 const unsigned DstSize = RBI.getSizeInBits(Reg: DstReg, MRI, TRI);
287 const RegisterBank &DstRegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
288
289 Register SrcReg = I.getOperand(i: 1).getReg();
290 const unsigned SrcSize = RBI.getSizeInBits(Reg: SrcReg, MRI, TRI);
291 const RegisterBank &SrcRegBank = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
292
293 if (DstReg.isPhysical()) {
294 assert(I.isCopy() && "Generic operators do not allow physical registers");
295
296 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
297 DstRegBank.getID() == X86::GPRRegBankID) {
298
299 const TargetRegisterClass *SrcRC =
300 getRegClass(Ty: MRI.getType(Reg: SrcReg), RB: SrcRegBank);
301 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(Reg: DstReg);
302
303 if (SrcRC != DstRC) {
304 // This case can be generated by ABI lowering, performe anyext
305 Register ExtSrc = MRI.createVirtualRegister(RegClass: DstRC);
306 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
307 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
308 .addDef(RegNo: ExtSrc)
309 .addReg(RegNo: SrcReg)
310 .addImm(Val: getSubRegIndex(RC: SrcRC));
311
312 I.getOperand(i: 1).setReg(ExtSrc);
313 }
314 }
315
316 // Special case GPR16 -> XMM
317 if (SrcSize == 16 && SrcRegBank.getID() == X86::GPRRegBankID &&
318 (DstRegBank.getID() == X86::VECRRegBankID)) {
319
320 const DebugLoc &DL = I.getDebugLoc();
321
322 // Any extend GPR16 -> GPR32
323 Register ExtReg = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
324 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG),
325 DestReg: ExtReg)
326 .addReg(RegNo: SrcReg)
327 .addImm(Val: X86::sub_16bit);
328
329 // Copy GR32 -> XMM
330 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
331 .addReg(RegNo: ExtReg);
332
333 I.eraseFromParent();
334 }
335
336 // Special case XMM -> GR16
337 if (DstSize == 16 && DstRegBank.getID() == X86::GPRRegBankID &&
338 (SrcRegBank.getID() == X86::VECRRegBankID)) {
339
340 const DebugLoc &DL = I.getDebugLoc();
341
342 // Move XMM to GR32 register.
343 Register Temp32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
344 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Temp32)
345 .addReg(RegNo: SrcReg);
346
347 // Extract the lower 16 bits
348 if (Register Dst32 = TRI.getMatchingSuperReg(Reg: DstReg, SubIdx: X86::sub_16bit,
349 RC: &X86::GR32RegClass)) {
350 // Optimization for Physical Dst (e.g. AX): Copy to EAX directly.
351 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst32)
352 .addReg(RegNo: Temp32);
353 } else {
354 // Handle if there is no super.
355 BuildMI(BB&: *I.getParent(), I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
356 .addReg(RegNo: Temp32, Flags: {}, SubReg: X86::sub_16bit);
357 }
358
359 I.eraseFromParent();
360 }
361
362 return true;
363 }
364
365 assert((!SrcReg.isPhysical() || I.isCopy()) &&
366 "No phys reg on generic operators");
367 assert((DstSize == SrcSize ||
368 // Copies are a mean to setup initial types, the number of
369 // bits may not exactly match.
370 (SrcReg.isPhysical() &&
371 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
372 "Copy with different width?!");
373
374 const TargetRegisterClass *DstRC =
375 getRegClass(Ty: MRI.getType(Reg: DstReg), RB: DstRegBank);
376
377 if (SrcRegBank.getID() == X86::GPRRegBankID &&
378 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
379 SrcReg.isPhysical()) {
380 // Change the physical register to performe truncate.
381
382 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(Reg: SrcReg);
383
384 if (DstRC != SrcRC) {
385 I.getOperand(i: 1).setSubReg(getSubRegIndex(RC: DstRC));
386 I.getOperand(i: 1).substPhysReg(Reg: SrcReg, TRI);
387 }
388 }
389
390 // No need to constrain SrcReg. It will get constrained when
391 // we hit another of its use or its defs.
392 // Copies do not have constraints.
393 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(Reg: DstReg);
394 if (!OldRC || !DstRC->hasSubClassEq(RC: OldRC)) {
395 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
396 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
397 << " operand\n");
398 return false;
399 }
400 }
401 I.setDesc(TII.get(Opcode: X86::COPY));
402 return true;
403}
404
405bool X86InstructionSelector::select(MachineInstr &I) {
406 assert(I.getParent() && "Instruction should be in a basic block!");
407 assert(I.getParent()->getParent() && "Instruction should be in a function!");
408
409 MachineBasicBlock &MBB = *I.getParent();
410 MachineFunction &MF = *MBB.getParent();
411 MachineRegisterInfo &MRI = MF.getRegInfo();
412
413 unsigned Opcode = I.getOpcode();
414 if (!isPreISelGenericOpcode(Opcode) && !I.isPreISelOpcode()) {
415 // Certain non-generic instructions also need some special handling.
416
417 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
418 return false;
419
420 if (I.isCopy())
421 return selectCopy(I, MRI);
422
423 if (I.isDebugInstr())
424 return selectDebugInstr(I, MRI);
425
426 return true;
427 }
428
429 assert(I.getNumOperands() == I.getNumExplicitOperands() &&
430 "Generic instruction has unexpected implicit operands\n");
431
432 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
433 return true;
434
435 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
436
437 // TODO: This should be implemented by tblgen.
438 switch (I.getOpcode()) {
439 default:
440 return false;
441 case TargetOpcode::G_STORE:
442 case TargetOpcode::G_LOAD:
443 return selectLoadStoreOp(I, MRI, MF);
444 case TargetOpcode::G_PTR_ADD:
445 case TargetOpcode::G_FRAME_INDEX:
446 return selectFrameIndexOrGep(I, MRI, MF);
447 case TargetOpcode::G_GLOBAL_VALUE:
448 return selectGlobalValue(I, MRI, MF);
449 case TargetOpcode::G_CONSTANT:
450 return selectConstant(I, MRI, MF);
451 case TargetOpcode::G_FCONSTANT:
452 return materializeFP(I, MRI, MF);
453 case TargetOpcode::G_PTRTOINT:
454 case TargetOpcode::G_TRUNC:
455 return selectTruncOrPtrToInt(I, MRI, MF);
456 case TargetOpcode::G_INTTOPTR:
457 case TargetOpcode::G_FREEZE:
458 return selectCopy(I, MRI);
459 case TargetOpcode::G_ZEXT:
460 return selectZext(I, MRI, MF);
461 case TargetOpcode::G_ANYEXT:
462 return selectAnyext(I, MRI, MF);
463 case TargetOpcode::G_ICMP:
464 return selectCmp(I, MRI, MF);
465 case TargetOpcode::G_FCMP:
466 return selectFCmp(I, MRI, MF);
467 case TargetOpcode::G_UADDE:
468 case TargetOpcode::G_UADDO:
469 case TargetOpcode::G_USUBE:
470 case TargetOpcode::G_USUBO:
471 return selectUAddSub(I, MRI, MF);
472 case TargetOpcode::G_UNMERGE_VALUES:
473 return selectUnmergeValues(I, MRI, MF);
474 case TargetOpcode::G_MERGE_VALUES:
475 case TargetOpcode::G_CONCAT_VECTORS:
476 return selectMergeValues(I, MRI, MF);
477 case TargetOpcode::G_EXTRACT:
478 return selectExtract(I, MRI, MF);
479 case TargetOpcode::G_INSERT:
480 return selectInsert(I, MRI, MF);
481 case TargetOpcode::G_BRCOND:
482 return selectCondBranch(I, MRI, MF);
483 case TargetOpcode::G_IMPLICIT_DEF:
484 case TargetOpcode::G_PHI:
485 return selectImplicitDefOrPHI(I, MRI);
486 case TargetOpcode::G_MUL:
487 case TargetOpcode::G_SMULH:
488 case TargetOpcode::G_UMULH:
489 case TargetOpcode::G_SDIV:
490 case TargetOpcode::G_UDIV:
491 case TargetOpcode::G_SREM:
492 case TargetOpcode::G_UREM:
493 return selectMulDivRem(I, MRI, MF);
494 case TargetOpcode::G_SELECT:
495 return selectSelect(I, MRI, MF);
496 }
497
498 return false;
499}
500
501unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
502 const RegisterBank &RB,
503 unsigned Opc) const {
504 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
505 "Only G_STORE and G_LOAD are expected for selection");
506 if (Ty.isPointer() && X86::GPRRegBankID == RB.getID()) {
507 bool IsLoad = (Opc == TargetOpcode::G_LOAD);
508 switch (Ty.getSizeInBits()) {
509 default:
510 break;
511 case 32:
512 return IsLoad ? X86::MOV32rm : X86::MOV32mr;
513 case 64:
514 return IsLoad ? X86::MOV64rm : X86::MOV64mr;
515 }
516 }
517 return Opc;
518}
519
520unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
521 const RegisterBank &RB,
522 unsigned Opc,
523 Align Alignment) const {
524 bool Isload = (Opc == TargetOpcode::G_LOAD);
525 bool HasAVX = STI.hasAVX();
526 bool HasAVX512 = STI.hasAVX512();
527 bool HasVLX = STI.hasVLX();
528
529 if (Ty == LLT::scalar(SizeInBits: 8)) {
530 if (X86::GPRRegBankID == RB.getID())
531 return Isload ? X86::MOV8rm : X86::MOV8mr;
532 } else if (Ty == LLT::scalar(SizeInBits: 16)) {
533 if (X86::GPRRegBankID == RB.getID())
534 return Isload ? X86::MOV16rm : X86::MOV16mr;
535 } else if (Ty == LLT::scalar(SizeInBits: 32)) {
536 if (X86::GPRRegBankID == RB.getID())
537 return Isload ? X86::MOV32rm : X86::MOV32mr;
538 if (X86::VECRRegBankID == RB.getID())
539 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
540 HasAVX ? X86::VMOVSSrm_alt :
541 X86::MOVSSrm_alt)
542 : (HasAVX512 ? X86::VMOVSSZmr :
543 HasAVX ? X86::VMOVSSmr :
544 X86::MOVSSmr);
545 if (X86::PSRRegBankID == RB.getID())
546 return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
547 } else if (Ty == LLT::scalar(SizeInBits: 64)) {
548 if (X86::GPRRegBankID == RB.getID())
549 return Isload ? X86::MOV64rm : X86::MOV64mr;
550 if (X86::VECRRegBankID == RB.getID())
551 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
552 HasAVX ? X86::VMOVSDrm_alt :
553 X86::MOVSDrm_alt)
554 : (HasAVX512 ? X86::VMOVSDZmr :
555 HasAVX ? X86::VMOVSDmr :
556 X86::MOVSDmr);
557 if (X86::PSRRegBankID == RB.getID())
558 return Isload ? X86::LD_Fp64m : X86::ST_Fp64m;
559 } else if (Ty == LLT::scalar(SizeInBits: 80)) {
560 return Isload ? X86::LD_Fp80m : X86::ST_FpP80m;
561 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
562 if (Alignment >= Align(16))
563 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
564 : HasAVX512
565 ? X86::VMOVAPSZ128rm_NOVLX
566 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
567 : (HasVLX ? X86::VMOVAPSZ128mr
568 : HasAVX512
569 ? X86::VMOVAPSZ128mr_NOVLX
570 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
571 else
572 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
573 : HasAVX512
574 ? X86::VMOVUPSZ128rm_NOVLX
575 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
576 : (HasVLX ? X86::VMOVUPSZ128mr
577 : HasAVX512
578 ? X86::VMOVUPSZ128mr_NOVLX
579 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
580 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
581 if (Alignment >= Align(32))
582 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
583 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
584 : X86::VMOVAPSYrm)
585 : (HasVLX ? X86::VMOVAPSZ256mr
586 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
587 : X86::VMOVAPSYmr);
588 else
589 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
590 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
591 : X86::VMOVUPSYrm)
592 : (HasVLX ? X86::VMOVUPSZ256mr
593 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
594 : X86::VMOVUPSYmr);
595 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
596 if (Alignment >= Align(64))
597 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
598 else
599 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
600 }
601 return Opc;
602}
603
604// Fill in an address from the given instruction.
605static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
606 const MachineRegisterInfo &MRI,
607 const X86Subtarget &STI, X86AddressMode &AM) {
608 assert(I.getOperand(0).isReg() && "unsupported operand.");
609 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
610 "unsupported type.");
611
612 switch (I.getOpcode()) {
613 default:
614 break;
615 case TargetOpcode::G_FRAME_INDEX:
616 AM.Base.FrameIndex = I.getOperand(i: 1).getIndex();
617 AM.BaseType = X86AddressMode::FrameIndexBase;
618 return true;
619 case TargetOpcode::G_PTR_ADD: {
620 if (auto COff = getIConstantVRegSExtVal(VReg: I.getOperand(i: 2).getReg(), MRI)) {
621 int64_t Imm = *COff;
622 if (isInt<32>(x: Imm)) { // Check for displacement overflow.
623 AM.Disp = static_cast<int32_t>(Imm);
624 AM.Base.Reg = I.getOperand(i: 1).getReg();
625 return true;
626 }
627 }
628 break;
629 }
630 case TargetOpcode::G_GLOBAL_VALUE:
631 case X86::G_WRAPPER_RIP: {
632 auto GV = I.getOperand(i: 1).getGlobal();
633 if (GV->isThreadLocal()) {
634 return false; // TODO: we don't support TLS yet.
635 }
636 // Can't handle alternate code models yet.
637 if (TM.getCodeModel() != CodeModel::Small)
638 return false;
639 AM.GV = GV;
640 AM.GVOpFlags = STI.classifyGlobalReference(GV);
641
642 // TODO: This reference is relative to the pic base. not supported yet.
643 if (isGlobalRelativeToPICBase(TargetFlag: AM.GVOpFlags))
644 return false;
645
646 if (STI.isPICStyleRIPRel() || AM.GVOpFlags == X86II::MO_GOTPCREL ||
647 AM.GVOpFlags == X86II::MO_GOTPCREL_NORELAX) {
648 // Use rip-relative addressing.
649 assert(AM.Base.Reg == 0 && AM.IndexReg == 0 &&
650 "RIP-relative addresses can't have additional register operands");
651 AM.Base.Reg = X86::RIP;
652 }
653 return true;
654 }
655 case TargetOpcode::G_CONSTANT_POOL: {
656 // TODO: Need a separate move for Large model
657 if (TM.getCodeModel() == CodeModel::Large)
658 return false;
659
660 AM.GVOpFlags = STI.classifyLocalReference(GV: nullptr);
661 if (AM.GVOpFlags == X86II::MO_GOTOFF)
662 AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(MF: I.getMF());
663 else if (STI.is64Bit())
664 AM.Base.Reg = X86::RIP;
665 AM.CP = true;
666 AM.Disp = I.getOperand(i: 1).getIndex();
667 return true;
668 }
669 }
670 // Default behavior.
671 AM.Base.Reg = I.getOperand(i: 0).getReg();
672 return true;
673}
674
675bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
676 MachineRegisterInfo &MRI,
677 MachineFunction &MF) const {
678 unsigned Opc = I.getOpcode();
679
680 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
681 "Only G_STORE and G_LOAD are expected for selection");
682
683 const Register DefReg = I.getOperand(i: 0).getReg();
684 LLT Ty = MRI.getType(Reg: DefReg);
685 const RegisterBank &RB = *RBI.getRegBank(Reg: DefReg, MRI, TRI);
686
687 assert(I.hasOneMemOperand());
688 auto &MemOp = **I.memoperands_begin();
689 if (MemOp.isAtomic()) {
690 // Note: for unordered operations, we rely on the fact the appropriate MMO
691 // is already on the instruction we're mutating, and thus we don't need to
692 // make any changes. So long as we select an opcode which is capable of
693 // loading or storing the appropriate size atomically, the rest of the
694 // backend is required to respect the MMO state.
695 if (!MemOp.isUnordered()) {
696 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
697 return false;
698 }
699 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
700 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
701 return false;
702 }
703 }
704
705 unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
706 if (NewOpc == Opc)
707 return false;
708
709 I.setDesc(TII.get(Opcode: NewOpc));
710 MachineInstrBuilder MIB(MF, I);
711 MachineInstr *Ptr = MRI.getVRegDef(Reg: I.getOperand(i: 1).getReg());
712
713 X86AddressMode AM;
714 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
715 return false;
716
717 if (Opc == TargetOpcode::G_LOAD) {
718 I.removeOperand(OpNo: 1);
719 addFullAddress(MIB, AM);
720 } else {
721 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
722 I.removeOperand(OpNo: 1);
723 I.removeOperand(OpNo: 0);
724 addFullAddress(MIB, AM).addUse(RegNo: DefReg);
725 }
726 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
727 I.addImplicitDefUseOperands(MF);
728 return true;
729}
730
731static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
732 if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 64))
733 return X86::LEA64r;
734 else if (Ty == LLT::pointer(AddressSpace: 0, SizeInBits: 32))
735 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
736 else
737 llvm_unreachable("Can't get LEA opcode. Unsupported type.");
738}
739
740bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
741 MachineRegisterInfo &MRI,
742 MachineFunction &MF) const {
743 unsigned Opc = I.getOpcode();
744
745 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
746 "unexpected instruction");
747
748 const Register DefReg = I.getOperand(i: 0).getReg();
749 LLT Ty = MRI.getType(Reg: DefReg);
750
751 // Use LEA to calculate frame index and GEP
752 unsigned NewOpc = getLeaOP(Ty, STI);
753 I.setDesc(TII.get(Opcode: NewOpc));
754 MachineInstrBuilder MIB(MF, I);
755
756 if (Opc == TargetOpcode::G_FRAME_INDEX) {
757 addOffset(MIB, Offset: 0);
758 } else {
759 MachineOperand &InxOp = I.getOperand(i: 2);
760 I.addOperand(Op: InxOp); // set IndexReg
761 InxOp.ChangeToImmediate(ImmVal: 1); // set Scale
762 MIB.addImm(Val: 0).addReg(RegNo: 0);
763 }
764
765 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
766 return true;
767}
768
769bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
770 MachineRegisterInfo &MRI,
771 MachineFunction &MF) const {
772 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
773 "unexpected instruction");
774
775 X86AddressMode AM;
776 if (!X86SelectAddress(I, TM, MRI, STI, AM))
777 return false;
778
779 const Register DefReg = I.getOperand(i: 0).getReg();
780 LLT Ty = MRI.getType(Reg: DefReg);
781 unsigned NewOpc = getLeaOP(Ty, STI);
782
783 I.setDesc(TII.get(Opcode: NewOpc));
784 MachineInstrBuilder MIB(MF, I);
785
786 I.removeOperand(OpNo: 1);
787 addFullAddress(MIB, AM);
788
789 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
790 return true;
791}
792
793bool X86InstructionSelector::selectConstant(MachineInstr &I,
794 MachineRegisterInfo &MRI,
795 MachineFunction &MF) const {
796 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
797 "unexpected instruction");
798
799 const Register DefReg = I.getOperand(i: 0).getReg();
800 LLT Ty = MRI.getType(Reg: DefReg);
801
802 if (RBI.getRegBank(Reg: DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
803 return false;
804
805 uint64_t Val = 0;
806 if (I.getOperand(i: 1).isCImm()) {
807 Val = I.getOperand(i: 1).getCImm()->getZExtValue();
808 I.getOperand(i: 1).ChangeToImmediate(ImmVal: Val);
809 } else if (I.getOperand(i: 1).isImm()) {
810 Val = I.getOperand(i: 1).getImm();
811 } else
812 llvm_unreachable("Unsupported operand type.");
813
814 unsigned NewOpc;
815 switch (Ty.getSizeInBits()) {
816 case 8:
817 NewOpc = X86::MOV8ri;
818 break;
819 case 16:
820 NewOpc = X86::MOV16ri;
821 break;
822 case 32:
823 NewOpc = X86::MOV32ri;
824 break;
825 case 64:
826 NewOpc = X86::getMOVriOpcode(/*Use64BitReg=*/true, Imm: Val);
827 break;
828 default:
829 llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
830 }
831
832 I.setDesc(TII.get(Opcode: NewOpc));
833 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
834 return true;
835}
836
837// Helper function for selectTruncOrPtrToInt and selectAnyext.
838// Returns true if DstRC lives on a floating register class and
839// SrcRC lives on a 128-bit vector class.
840static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
841 const TargetRegisterClass *SrcRC) {
842 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
843 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
844 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
845}
846
847bool X86InstructionSelector::selectTurnIntoCOPY(
848 MachineInstr &I, MachineRegisterInfo &MRI, const Register DstReg,
849 const TargetRegisterClass *DstRC, const Register SrcReg,
850 const TargetRegisterClass *SrcRC) const {
851
852 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
853 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
854 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
855 << " operand\n");
856 return false;
857 }
858 I.setDesc(TII.get(Opcode: X86::COPY));
859 return true;
860}
861
862bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
863 MachineRegisterInfo &MRI,
864 MachineFunction &MF) const {
865 assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
866 I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
867 "unexpected instruction");
868
869 const Register DstReg = I.getOperand(i: 0).getReg();
870 const Register SrcReg = I.getOperand(i: 1).getReg();
871
872 const LLT DstTy = MRI.getType(Reg: DstReg);
873 const LLT SrcTy = MRI.getType(Reg: SrcReg);
874
875 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
876 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
877
878 if (DstRB.getID() != SrcRB.getID()) {
879 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
880 << " input/output on different banks\n");
881 return false;
882 }
883
884 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
885 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
886
887 if (!DstRC || !SrcRC)
888 return false;
889
890 // If that's truncation of the value that lives on the vector class and goes
891 // into the floating class, just replace it with copy, as we are able to
892 // select it as a regular move.
893 if (canTurnIntoCOPY(DstRC, SrcRC))
894 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
895
896 if (DstRB.getID() != X86::GPRRegBankID)
897 return false;
898
899 unsigned SubIdx;
900 if (DstRC == SrcRC) {
901 // Nothing to be done
902 SubIdx = X86::NoSubRegister;
903 } else if (DstRC == &X86::GR32RegClass) {
904 SubIdx = X86::sub_32bit;
905 } else if (DstRC == &X86::GR16RegClass) {
906 SubIdx = X86::sub_16bit;
907 } else if (DstRC == &X86::GR8RegClass) {
908 SubIdx = X86::sub_8bit;
909 } else {
910 return false;
911 }
912
913 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
914
915 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
916 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
917 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
918 << "\n");
919 return false;
920 }
921
922 I.getOperand(i: 1).setSubReg(SubIdx);
923
924 I.setDesc(TII.get(Opcode: X86::COPY));
925 return true;
926}
927
928bool X86InstructionSelector::selectZext(MachineInstr &I,
929 MachineRegisterInfo &MRI,
930 MachineFunction &MF) const {
931 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
932
933 const Register DstReg = I.getOperand(i: 0).getReg();
934 const Register SrcReg = I.getOperand(i: 1).getReg();
935
936 const LLT DstTy = MRI.getType(Reg: DstReg);
937 const LLT SrcTy = MRI.getType(Reg: SrcReg);
938
939 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) &&
940 "8=>16 Zext is handled by tablegen");
941 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
942 "8=>32 Zext is handled by tablegen");
943 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
944 "16=>32 Zext is handled by tablegen");
945 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) &&
946 "8=>64 Zext is handled by tablegen");
947 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) &&
948 "16=>64 Zext is handled by tablegen");
949 assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) &&
950 "32=>64 Zext is handled by tablegen");
951
952 if (SrcTy != LLT::scalar(SizeInBits: 1))
953 return false;
954
955 unsigned AndOpc;
956 if (DstTy == LLT::scalar(SizeInBits: 8))
957 AndOpc = X86::AND8ri;
958 else if (DstTy == LLT::scalar(SizeInBits: 16))
959 AndOpc = X86::AND16ri;
960 else if (DstTy == LLT::scalar(SizeInBits: 32))
961 AndOpc = X86::AND32ri;
962 else if (DstTy == LLT::scalar(SizeInBits: 64))
963 AndOpc = X86::AND64ri32;
964 else
965 return false;
966
967 Register DefReg = SrcReg;
968 if (DstTy != LLT::scalar(SizeInBits: 8)) {
969 Register ImpDefReg =
970 MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
971 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
972 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImpDefReg);
973
974 DefReg = MRI.createVirtualRegister(RegClass: getRegClass(Ty: DstTy, Reg: DstReg, MRI));
975 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
976 MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DefReg)
977 .addReg(RegNo: ImpDefReg)
978 .addReg(RegNo: SrcReg)
979 .addImm(Val: X86::sub_8bit);
980 }
981
982 MachineInstr &AndInst =
983 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AndOpc), DestReg: DstReg)
984 .addReg(RegNo: DefReg)
985 .addImm(Val: 1);
986
987 constrainSelectedInstRegOperands(I&: AndInst, TII, TRI, RBI);
988
989 I.eraseFromParent();
990 return true;
991}
992
993bool X86InstructionSelector::selectAnyext(MachineInstr &I,
994 MachineRegisterInfo &MRI,
995 MachineFunction &MF) const {
996 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
997
998 const Register DstReg = I.getOperand(i: 0).getReg();
999 const Register SrcReg = I.getOperand(i: 1).getReg();
1000
1001 const LLT DstTy = MRI.getType(Reg: DstReg);
1002 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1003
1004 const RegisterBank &DstRB = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1005 const RegisterBank &SrcRB = *RBI.getRegBank(Reg: SrcReg, MRI, TRI);
1006
1007 assert(DstRB.getID() == SrcRB.getID() &&
1008 "G_ANYEXT input/output on different banks\n");
1009
1010 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
1011 "G_ANYEXT incorrect operand size");
1012
1013 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, RB: DstRB);
1014 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, RB: SrcRB);
1015
1016 // If that's ANY_EXT of the value that lives on the floating class and goes
1017 // into the vector class, just replace it with copy, as we are able to select
1018 // it as a regular move.
1019 if (canTurnIntoCOPY(DstRC: SrcRC, SrcRC: DstRC))
1020 return selectTurnIntoCOPY(I, MRI, DstReg: SrcReg, DstRC: SrcRC, SrcReg: DstReg, SrcRC: DstRC);
1021
1022 if (DstRB.getID() != X86::GPRRegBankID)
1023 return false;
1024
1025 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1026 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1027 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1028 << " operand\n");
1029 return false;
1030 }
1031
1032 if (SrcRC == DstRC) {
1033 I.setDesc(TII.get(Opcode: X86::COPY));
1034 return true;
1035 }
1036
1037 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1038 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG))
1039 .addDef(RegNo: DstReg)
1040 .addReg(RegNo: SrcReg)
1041 .addImm(Val: getSubRegIndex(RC: SrcRC));
1042
1043 I.eraseFromParent();
1044 return true;
1045}
1046
1047bool X86InstructionSelector::selectCmp(MachineInstr &I,
1048 MachineRegisterInfo &MRI,
1049 MachineFunction &MF) const {
1050 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
1051
1052 X86::CondCode CC;
1053 bool SwapArgs;
1054 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(
1055 Predicate: (CmpInst::Predicate)I.getOperand(i: 1).getPredicate());
1056
1057 Register LHS = I.getOperand(i: 2).getReg();
1058 Register RHS = I.getOperand(i: 3).getReg();
1059
1060 if (SwapArgs)
1061 std::swap(a&: LHS, b&: RHS);
1062
1063 unsigned OpCmp;
1064 LLT Ty = MRI.getType(Reg: LHS);
1065
1066 switch (Ty.getSizeInBits()) {
1067 default:
1068 return false;
1069 case 8:
1070 OpCmp = X86::CMP8rr;
1071 break;
1072 case 16:
1073 OpCmp = X86::CMP16rr;
1074 break;
1075 case 32:
1076 OpCmp = X86::CMP32rr;
1077 break;
1078 case 64:
1079 OpCmp = X86::CMP64rr;
1080 break;
1081 }
1082
1083 MachineInstr &CmpInst =
1084 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1085 .addReg(RegNo: LHS)
1086 .addReg(RegNo: RHS);
1087
1088 MachineInstr &SetInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1089 MCID: TII.get(Opcode: X86::SETCCr), DestReg: I.getOperand(i: 0).getReg()).addImm(Val: CC);
1090
1091 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1092 constrainSelectedInstRegOperands(I&: SetInst, TII, TRI, RBI);
1093
1094 I.eraseFromParent();
1095 return true;
1096}
1097
1098bool X86InstructionSelector::selectFCmp(MachineInstr &I,
1099 MachineRegisterInfo &MRI,
1100 MachineFunction &MF) const {
1101 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
1102
1103 Register LhsReg = I.getOperand(i: 2).getReg();
1104 Register RhsReg = I.getOperand(i: 3).getReg();
1105 CmpInst::Predicate Predicate =
1106 (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1107
1108 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1109 static const uint16_t SETFOpcTable[2][3] = {
1110 {X86::COND_E, X86::COND_NP, X86::AND8rr},
1111 {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1112 const uint16_t *SETFOpc = nullptr;
1113 switch (Predicate) {
1114 default:
1115 break;
1116 case CmpInst::FCMP_OEQ:
1117 SETFOpc = &SETFOpcTable[0][0];
1118 break;
1119 case CmpInst::FCMP_UNE:
1120 SETFOpc = &SETFOpcTable[1][0];
1121 break;
1122 }
1123
1124 assert((LhsReg.isVirtual() && RhsReg.isVirtual()) &&
1125 "Both arguments of FCMP need to be virtual!");
1126 auto *LhsBank = RBI.getRegBank(Reg: LhsReg, MRI, TRI);
1127 [[maybe_unused]] auto *RhsBank = RBI.getRegBank(Reg: RhsReg, MRI, TRI);
1128 assert((LhsBank == RhsBank) &&
1129 "Both banks assigned to FCMP arguments need to be same!");
1130
1131 bool IsX87 = LhsBank->getID() == X86::PSRRegBankID;
1132
1133 // Compute the opcode for the CMP instruction.
1134 unsigned OpCmp;
1135 LLT Ty = MRI.getType(Reg: LhsReg);
1136 switch (Ty.getSizeInBits()) {
1137 default:
1138 return false;
1139 case 32:
1140 OpCmp = IsX87 ? X86::UCOM_FpIr32 : X86::UCOMISSrr;
1141 break;
1142 case 64:
1143 OpCmp = IsX87 ? X86::UCOM_FpIr64 : X86::UCOMISDrr;
1144 break;
1145 case 80:
1146 OpCmp = X86::UCOM_FpIr80;
1147 break;
1148 }
1149
1150 Register ResultReg = I.getOperand(i: 0).getReg();
1151 RBI.constrainGenericRegister(
1152 Reg: ResultReg,
1153 RC: *getRegClass(Ty: LLT::scalar(SizeInBits: 8), RB: *RBI.getRegBank(Reg: ResultReg, MRI, TRI)), MRI);
1154 if (SETFOpc) {
1155 MachineInstr &CmpInst =
1156 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1157 .addReg(RegNo: LhsReg)
1158 .addReg(RegNo: RhsReg);
1159 if (IsX87)
1160 CmpInst.getOperand(i: 3).setIsDead(); // Mark unused FPSW def dead.
1161
1162 Register FlagReg1 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1163 Register FlagReg2 = MRI.createVirtualRegister(RegClass: &X86::GR8RegClass);
1164 MachineInstr &Set1 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1165 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg1).addImm(Val: SETFOpc[0]);
1166 MachineInstr &Set2 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1167 MCID: TII.get(Opcode: X86::SETCCr), DestReg: FlagReg2).addImm(Val: SETFOpc[1]);
1168 MachineInstr &Set3 = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1169 MCID: TII.get(Opcode: SETFOpc[2]), DestReg: ResultReg)
1170 .addReg(RegNo: FlagReg1)
1171 .addReg(RegNo: FlagReg2)
1172 .setOperandDead(3);
1173 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1174 constrainSelectedInstRegOperands(I&: Set1, TII, TRI, RBI);
1175 constrainSelectedInstRegOperands(I&: Set2, TII, TRI, RBI);
1176 constrainSelectedInstRegOperands(I&: Set3, TII, TRI, RBI);
1177
1178 I.eraseFromParent();
1179 return true;
1180 }
1181
1182 X86::CondCode CC;
1183 bool SwapArgs;
1184 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1185 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1186
1187 if (SwapArgs)
1188 std::swap(a&: LhsReg, b&: RhsReg);
1189
1190 // Emit a compare of LHS/RHS.
1191 MachineInstr &CmpInst =
1192 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpCmp))
1193 .addReg(RegNo: LhsReg)
1194 .addReg(RegNo: RhsReg);
1195 if (IsX87)
1196 CmpInst.getOperand(i: 3).setIsDead(); // Mark unused FPSW def dead.
1197
1198 MachineInstr &Set =
1199 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: ResultReg).addImm(Val: CC);
1200 constrainSelectedInstRegOperands(I&: CmpInst, TII, TRI, RBI);
1201 constrainSelectedInstRegOperands(I&: Set, TII, TRI, RBI);
1202 I.eraseFromParent();
1203 return true;
1204}
1205
1206bool X86InstructionSelector::selectUAddSub(MachineInstr &I,
1207 MachineRegisterInfo &MRI,
1208 MachineFunction &MF) const {
1209 assert((I.getOpcode() == TargetOpcode::G_UADDE ||
1210 I.getOpcode() == TargetOpcode::G_UADDO ||
1211 I.getOpcode() == TargetOpcode::G_USUBE ||
1212 I.getOpcode() == TargetOpcode::G_USUBO) &&
1213 "unexpected instruction");
1214
1215 auto &CarryMI = cast<GAddSubCarryOut>(Val&: I);
1216
1217 const Register DstReg = CarryMI.getDstReg();
1218 const Register CarryOutReg = CarryMI.getCarryOutReg();
1219 const Register Op0Reg = CarryMI.getLHSReg();
1220 const Register Op1Reg = CarryMI.getRHSReg();
1221 bool IsSub = CarryMI.isSub();
1222
1223 const LLT DstTy = MRI.getType(Reg: DstReg);
1224 assert(DstTy.isScalar() && "selectUAddSub only supported for scalar types");
1225
1226 // TODO: Handle immediate argument variants?
1227 unsigned OpADC, OpADD, OpSBB, OpSUB;
1228 switch (DstTy.getSizeInBits()) {
1229 case 8:
1230 OpADC = X86::ADC8rr;
1231 OpADD = X86::ADD8rr;
1232 OpSBB = X86::SBB8rr;
1233 OpSUB = X86::SUB8rr;
1234 break;
1235 case 16:
1236 OpADC = X86::ADC16rr;
1237 OpADD = X86::ADD16rr;
1238 OpSBB = X86::SBB16rr;
1239 OpSUB = X86::SUB16rr;
1240 break;
1241 case 32:
1242 OpADC = X86::ADC32rr;
1243 OpADD = X86::ADD32rr;
1244 OpSBB = X86::SBB32rr;
1245 OpSUB = X86::SUB32rr;
1246 break;
1247 case 64:
1248 OpADC = X86::ADC64rr;
1249 OpADD = X86::ADD64rr;
1250 OpSBB = X86::SBB64rr;
1251 OpSUB = X86::SUB64rr;
1252 break;
1253 default:
1254 llvm_unreachable("selectUAddSub unsupported type.");
1255 }
1256
1257 const RegisterBank &CarryRB = *RBI.getRegBank(Reg: CarryOutReg, MRI, TRI);
1258 const TargetRegisterClass *CarryRC =
1259 getRegClass(Ty: MRI.getType(Reg: CarryOutReg), RB: CarryRB);
1260
1261 unsigned Opcode = IsSub ? OpSUB : OpADD;
1262
1263 // G_UADDE/G_USUBE - find CarryIn def instruction.
1264 if (auto CarryInMI = dyn_cast<GAddSubCarryInOut>(Val: &I)) {
1265 Register CarryInReg = CarryInMI->getCarryInReg();
1266 MachineInstr *Def = MRI.getVRegDef(Reg: CarryInReg);
1267 while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1268 CarryInReg = Def->getOperand(i: 1).getReg();
1269 Def = MRI.getVRegDef(Reg: CarryInReg);
1270 }
1271
1272 // TODO - handle more CF generating instructions
1273 if (Def->getOpcode() == TargetOpcode::G_UADDE ||
1274 Def->getOpcode() == TargetOpcode::G_UADDO ||
1275 Def->getOpcode() == TargetOpcode::G_USUBE ||
1276 Def->getOpcode() == TargetOpcode::G_USUBO) {
1277 // The carry-in is a SETB byte (0 or 1) from a chained add/sub.
1278 // Materialize EFLAGS.CF from that byte for the following ADC/SBB
1279 // by emitting NEG, which sets CF iff its operand is non-zero.
1280 if (!RBI.constrainGenericRegister(Reg: CarryInReg, RC: *CarryRC, MRI))
1281 return false;
1282
1283 Register NegDef = MRI.createVirtualRegister(RegClass: CarryRC);
1284 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::NEG8r), DestReg: NegDef)
1285 .addReg(RegNo: CarryInReg);
1286
1287 Opcode = IsSub ? OpSBB : OpADC;
1288 } else if (auto val = getIConstantVRegVal(VReg: CarryInReg, MRI)) {
1289 // carry is constant, support only 0.
1290 if (*val != 0)
1291 return false;
1292
1293 Opcode = IsSub ? OpSUB : OpADD;
1294 } else
1295 return false;
1296 }
1297
1298 MachineInstr &Inst =
1299 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode), DestReg: DstReg)
1300 .addReg(RegNo: Op0Reg)
1301 .addReg(RegNo: Op1Reg);
1302
1303 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SETCCr), DestReg: CarryOutReg)
1304 .addImm(Val: X86::COND_B);
1305
1306 constrainSelectedInstRegOperands(I&: Inst, TII, TRI, RBI);
1307 if (!RBI.constrainGenericRegister(Reg: CarryOutReg, RC: *CarryRC, MRI))
1308 return false;
1309
1310 I.eraseFromParent();
1311 return true;
1312}
1313
1314bool X86InstructionSelector::selectExtract(MachineInstr &I,
1315 MachineRegisterInfo &MRI,
1316 MachineFunction &MF) const {
1317 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1318 "unexpected instruction");
1319
1320 const Register DstReg = I.getOperand(i: 0).getReg();
1321 const Register SrcReg = I.getOperand(i: 1).getReg();
1322 int64_t Index = I.getOperand(i: 2).getImm();
1323
1324 const LLT DstTy = MRI.getType(Reg: DstReg);
1325 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1326
1327 // Meanwile handle vector type only.
1328 if (!DstTy.isVector())
1329 return false;
1330
1331 if (Index % DstTy.getSizeInBits() != 0)
1332 return false; // Not extract subvector.
1333
1334 if (Index == 0) {
1335 // Replace by extract subreg copy.
1336 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1337 return false;
1338
1339 I.eraseFromParent();
1340 return true;
1341 }
1342
1343 bool HasAVX = STI.hasAVX();
1344 bool HasAVX512 = STI.hasAVX512();
1345 bool HasVLX = STI.hasVLX();
1346
1347 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1348 if (HasVLX)
1349 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Z256rri));
1350 else if (HasAVX)
1351 I.setDesc(TII.get(Opcode: X86::VEXTRACTF128rri));
1352 else
1353 return false;
1354 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1355 if (DstTy.getSizeInBits() == 128)
1356 I.setDesc(TII.get(Opcode: X86::VEXTRACTF32X4Zrri));
1357 else if (DstTy.getSizeInBits() == 256)
1358 I.setDesc(TII.get(Opcode: X86::VEXTRACTF64X4Zrri));
1359 else
1360 return false;
1361 } else
1362 return false;
1363
1364 // Convert to X86 VEXTRACT immediate.
1365 Index = Index / DstTy.getSizeInBits();
1366 I.getOperand(i: 2).setImm(Index);
1367
1368 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1369 return true;
1370}
1371
1372bool X86InstructionSelector::emitExtractSubreg(Register DstReg, Register SrcReg,
1373 MachineInstr &I,
1374 MachineRegisterInfo &MRI,
1375 MachineFunction &MF) const {
1376 const LLT DstTy = MRI.getType(Reg: DstReg);
1377 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1378 unsigned SubIdx = X86::NoSubRegister;
1379
1380 if (!DstTy.isVector() || !SrcTy.isVector())
1381 return false;
1382
1383 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1384 "Incorrect Src/Dst register size");
1385
1386 if (DstTy.getSizeInBits() == 128)
1387 SubIdx = X86::sub_xmm;
1388 else if (DstTy.getSizeInBits() == 256)
1389 SubIdx = X86::sub_ymm;
1390 else
1391 return false;
1392
1393 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1394 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1395
1396 SrcRC = TRI.getSubClassWithSubReg(RC: SrcRC, Idx: SubIdx);
1397
1398 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1399 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1400 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1401 return false;
1402 }
1403
1404 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::COPY), DestReg: DstReg)
1405 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubIdx);
1406
1407 return true;
1408}
1409
1410bool X86InstructionSelector::emitInsertSubreg(Register DstReg, Register SrcReg,
1411 MachineInstr &I,
1412 MachineRegisterInfo &MRI,
1413 MachineFunction &MF) const {
1414 const LLT DstTy = MRI.getType(Reg: DstReg);
1415 const LLT SrcTy = MRI.getType(Reg: SrcReg);
1416 unsigned SubIdx = X86::NoSubRegister;
1417
1418 // TODO: support scalar types
1419 if (!DstTy.isVector() || !SrcTy.isVector())
1420 return false;
1421
1422 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1423 "Incorrect Src/Dst register size");
1424
1425 if (SrcTy.getSizeInBits() == 128)
1426 SubIdx = X86::sub_xmm;
1427 else if (SrcTy.getSizeInBits() == 256)
1428 SubIdx = X86::sub_ymm;
1429 else
1430 return false;
1431
1432 const TargetRegisterClass *SrcRC = getRegClass(Ty: SrcTy, Reg: SrcReg, MRI);
1433 const TargetRegisterClass *DstRC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1434
1435 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI) ||
1436 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1437 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1438 return false;
1439 }
1440
1441 Register ImpDefReg = MRI.createVirtualRegister(RegClass: DstRC);
1442 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::IMPLICIT_DEF),
1443 DestReg: ImpDefReg);
1444
1445 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::INSERT_SUBREG),
1446 DestReg: DstReg)
1447 .addReg(RegNo: ImpDefReg)
1448 .addReg(RegNo: SrcReg)
1449 .addImm(Val: SubIdx);
1450
1451 return true;
1452}
1453
1454bool X86InstructionSelector::selectInsert(MachineInstr &I,
1455 MachineRegisterInfo &MRI,
1456 MachineFunction &MF) const {
1457 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1458
1459 const Register DstReg = I.getOperand(i: 0).getReg();
1460 const Register SrcReg = I.getOperand(i: 1).getReg();
1461 const Register InsertReg = I.getOperand(i: 2).getReg();
1462 int64_t Index = I.getOperand(i: 3).getImm();
1463
1464 const LLT DstTy = MRI.getType(Reg: DstReg);
1465 const LLT InsertRegTy = MRI.getType(Reg: InsertReg);
1466
1467 // Meanwile handle vector type only.
1468 if (!DstTy.isVector())
1469 return false;
1470
1471 if (Index % InsertRegTy.getSizeInBits() != 0)
1472 return false; // Not insert subvector.
1473
1474 if (Index == 0 && mi_match(R: SrcReg, MRI, P: m_GImplicitDef())) {
1475 // Replace by subreg copy.
1476 if (!emitInsertSubreg(DstReg, SrcReg: InsertReg, I, MRI, MF))
1477 return false;
1478
1479 I.eraseFromParent();
1480 return true;
1481 }
1482
1483 bool HasAVX = STI.hasAVX();
1484 bool HasAVX512 = STI.hasAVX512();
1485 bool HasVLX = STI.hasVLX();
1486
1487 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1488 if (HasVLX)
1489 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Z256rri));
1490 else if (HasAVX)
1491 I.setDesc(TII.get(Opcode: X86::VINSERTF128rri));
1492 else
1493 return false;
1494 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1495 if (InsertRegTy.getSizeInBits() == 128)
1496 I.setDesc(TII.get(Opcode: X86::VINSERTF32X4Zrri));
1497 else if (InsertRegTy.getSizeInBits() == 256)
1498 I.setDesc(TII.get(Opcode: X86::VINSERTF64X4Zrri));
1499 else
1500 return false;
1501 } else
1502 return false;
1503
1504 // Convert to X86 VINSERT immediate.
1505 Index = Index / InsertRegTy.getSizeInBits();
1506
1507 I.getOperand(i: 3).setImm(Index);
1508
1509 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1510 return true;
1511}
1512
1513bool X86InstructionSelector::selectUnmergeValues(
1514 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1515 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1516 "unexpected instruction");
1517
1518 // Split to extracts.
1519 unsigned NumDefs = I.getNumOperands() - 1;
1520 Register SrcReg = I.getOperand(i: NumDefs).getReg();
1521 unsigned DefSize = MRI.getType(Reg: I.getOperand(i: 0).getReg()).getSizeInBits();
1522
1523 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1524 MachineInstr &ExtrInst =
1525 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1526 MCID: TII.get(Opcode: TargetOpcode::G_EXTRACT), DestReg: I.getOperand(i: Idx).getReg())
1527 .addReg(RegNo: SrcReg)
1528 .addImm(Val: Idx * DefSize);
1529
1530 if (!select(I&: ExtrInst))
1531 return false;
1532 }
1533
1534 I.eraseFromParent();
1535 return true;
1536}
1537
1538bool X86InstructionSelector::selectMergeValues(
1539 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1540 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1541 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1542 "unexpected instruction");
1543
1544 // Split to inserts.
1545 Register DstReg = I.getOperand(i: 0).getReg();
1546 Register SrcReg0 = I.getOperand(i: 1).getReg();
1547
1548 const LLT DstTy = MRI.getType(Reg: DstReg);
1549 const LLT SrcTy = MRI.getType(Reg: SrcReg0);
1550 unsigned SrcSize = SrcTy.getSizeInBits();
1551
1552 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1553
1554 // For the first src use insertSubReg.
1555 Register DefReg = MRI.createGenericVirtualRegister(Ty: DstTy);
1556 MRI.setRegBank(Reg: DefReg, RegBank);
1557 if (!emitInsertSubreg(DstReg: DefReg, SrcReg: I.getOperand(i: 1).getReg(), I, MRI, MF))
1558 return false;
1559
1560 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1561 Register Tmp = MRI.createGenericVirtualRegister(Ty: DstTy);
1562 MRI.setRegBank(Reg: Tmp, RegBank);
1563
1564 MachineInstr &InsertInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1565 MCID: TII.get(Opcode: TargetOpcode::G_INSERT), DestReg: Tmp)
1566 .addReg(RegNo: DefReg)
1567 .addReg(RegNo: I.getOperand(i: Idx).getReg())
1568 .addImm(Val: (Idx - 1) * SrcSize);
1569
1570 DefReg = Tmp;
1571
1572 if (!select(I&: InsertInst))
1573 return false;
1574 }
1575
1576 MachineInstr &CopyInst = *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1577 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
1578 .addReg(RegNo: DefReg);
1579
1580 if (!select(I&: CopyInst))
1581 return false;
1582
1583 I.eraseFromParent();
1584 return true;
1585}
1586
1587bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1588 MachineRegisterInfo &MRI,
1589 MachineFunction &MF) const {
1590 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1591
1592 const Register CondReg = I.getOperand(i: 0).getReg();
1593 MachineBasicBlock *DestMBB = I.getOperand(i: 1).getMBB();
1594
1595 MachineInstr &TestInst =
1596 *BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST8ri))
1597 .addReg(RegNo: CondReg)
1598 .addImm(Val: 1);
1599 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::JCC_1))
1600 .addMBB(MBB: DestMBB).addImm(Val: X86::COND_NE);
1601
1602 constrainSelectedInstRegOperands(I&: TestInst, TII, TRI, RBI);
1603
1604 I.eraseFromParent();
1605 return true;
1606}
1607
1608bool X86InstructionSelector::materializeFP(MachineInstr &I,
1609 MachineRegisterInfo &MRI,
1610 MachineFunction &MF) const {
1611 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1612 "unexpected instruction");
1613
1614 // Can't handle alternate code models yet.
1615 CodeModel::Model CM = TM.getCodeModel();
1616 if (CM != CodeModel::Small && CM != CodeModel::Large)
1617 return false;
1618
1619 const Register DstReg = I.getOperand(i: 0).getReg();
1620 const LLT DstTy = MRI.getType(Reg: DstReg);
1621 const RegisterBank &RegBank = *RBI.getRegBank(Reg: DstReg, MRI, TRI);
1622 // Create the load from the constant pool.
1623 const ConstantFP *CFP = I.getOperand(i: 1).getFPImm();
1624 const auto &DL = MF.getDataLayout();
1625 Align Alignment = DL.getPrefTypeAlign(Ty: CFP->getType());
1626 const DebugLoc &DbgLoc = I.getDebugLoc();
1627
1628 unsigned Opc =
1629 getLoadStoreOp(Ty: DstTy, RB: RegBank, Opc: TargetOpcode::G_LOAD, Alignment);
1630
1631 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(C: CFP, Alignment);
1632 MachineInstr *LoadInst = nullptr;
1633 unsigned char OpFlag = STI.classifyLocalReference(GV: nullptr);
1634
1635 if (CM == CodeModel::Large && STI.is64Bit()) {
1636 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1637 // they cannot be folded into immediate fields.
1638
1639 Register AddrReg = MRI.createVirtualRegister(RegClass: &X86::GR64RegClass);
1640 BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: X86::MOV64ri), DestReg: AddrReg)
1641 .addConstantPoolIndex(Idx: CPI, Offset: 0, TargetFlags: OpFlag);
1642
1643 MachineMemOperand *MMO = MF.getMachineMemOperand(
1644 PtrInfo: MachinePointerInfo::getConstantPool(MF), F: MachineMemOperand::MOLoad,
1645 MemTy: LLT::pointer(AddressSpace: 0, SizeInBits: DL.getPointerSizeInBits()), BaseAlignment: Alignment);
1646
1647 LoadInst =
1648 addDirectMem(MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg),
1649 Reg: AddrReg)
1650 .addMemOperand(MMO);
1651
1652 } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1653 // Handle the case when globals fit in our immediate field.
1654 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1655
1656 // x86-32 PIC requires a PIC base register for constant pools.
1657 unsigned PICBase = 0;
1658 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1659 // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1660 // In DAGISEL the code that initialize it generated by the CGBR pass.
1661 return false; // TODO support the mode.
1662 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1663 PICBase = X86::RIP;
1664
1665 LoadInst = addConstantPoolReference(
1666 MIB: BuildMI(BB&: *I.getParent(), I, MIMD: DbgLoc, MCID: TII.get(Opcode: Opc), DestReg: DstReg), CPI, GlobalBaseReg: PICBase,
1667 OpFlags: OpFlag);
1668 } else
1669 return false;
1670
1671 constrainSelectedInstRegOperands(I&: *LoadInst, TII, TRI, RBI);
1672 I.eraseFromParent();
1673 return true;
1674}
1675
1676bool X86InstructionSelector::selectImplicitDefOrPHI(
1677 MachineInstr &I, MachineRegisterInfo &MRI) const {
1678 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1679 I.getOpcode() == TargetOpcode::G_PHI) &&
1680 "unexpected instruction");
1681
1682 Register DstReg = I.getOperand(i: 0).getReg();
1683
1684 if (!MRI.getRegClassOrNull(Reg: DstReg)) {
1685 const LLT DstTy = MRI.getType(Reg: DstReg);
1686 const TargetRegisterClass *RC = getRegClass(Ty: DstTy, Reg: DstReg, MRI);
1687
1688 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI)) {
1689 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1690 << " operand\n");
1691 return false;
1692 }
1693 }
1694
1695 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1696 I.setDesc(TII.get(Opcode: X86::IMPLICIT_DEF));
1697 else
1698 I.setDesc(TII.get(Opcode: X86::PHI));
1699
1700 return true;
1701}
1702
1703bool X86InstructionSelector::selectMulDivRem(MachineInstr &I,
1704 MachineRegisterInfo &MRI,
1705 MachineFunction &MF) const {
1706 // The implementation of this function is adapted from X86FastISel.
1707 assert((I.getOpcode() == TargetOpcode::G_MUL ||
1708 I.getOpcode() == TargetOpcode::G_SMULH ||
1709 I.getOpcode() == TargetOpcode::G_UMULH ||
1710 I.getOpcode() == TargetOpcode::G_SDIV ||
1711 I.getOpcode() == TargetOpcode::G_SREM ||
1712 I.getOpcode() == TargetOpcode::G_UDIV ||
1713 I.getOpcode() == TargetOpcode::G_UREM) &&
1714 "unexpected instruction");
1715
1716 const Register DstReg = I.getOperand(i: 0).getReg();
1717 const Register Op1Reg = I.getOperand(i: 1).getReg();
1718 const Register Op2Reg = I.getOperand(i: 2).getReg();
1719
1720 const LLT RegTy = MRI.getType(Reg: DstReg);
1721 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1722 "Arguments and return value types must match");
1723
1724 const RegisterBank *RegRB = RBI.getRegBank(Reg: DstReg, MRI, TRI);
1725 if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1726 return false;
1727
1728 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1729 const static unsigned NumOps = 7; // SDiv/SRem/UDiv/URem/Mul/SMulH/UMulh
1730 const static bool S = true; // IsSigned
1731 const static bool U = false; // !IsSigned
1732 const static unsigned Copy = TargetOpcode::COPY;
1733
1734 // For the X86 IDIV instruction, in most cases the dividend
1735 // (numerator) must be in a specific register pair highreg:lowreg,
1736 // producing the quotient in lowreg and the remainder in highreg.
1737 // For most data types, to set up the instruction, the dividend is
1738 // copied into lowreg, and lowreg is sign-extended into highreg. The
1739 // exception is i8, where the dividend is defined as a single register rather
1740 // than a register pair, and we therefore directly sign-extend the dividend
1741 // into lowreg, instead of copying, and ignore the highreg.
1742 const static struct MulDivRemEntry {
1743 // The following portion depends only on the data type.
1744 unsigned SizeInBits;
1745 unsigned LowInReg; // low part of the register pair
1746 unsigned HighInReg; // high part of the register pair
1747 // The following portion depends on both the data type and the operation.
1748 struct MulDivRemResult {
1749 unsigned OpMulDivRem; // The specific MUL/DIV opcode to use.
1750 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1751 // highreg, or copying a zero into highreg.
1752 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1753 // zero/sign-extending into lowreg for i8.
1754 unsigned ResultReg; // Register containing the desired result.
1755 bool IsOpSigned; // Whether to use signed or unsigned form.
1756 } ResultTable[NumOps];
1757 } OpTable[NumTypes] = {
1758 {.SizeInBits: 8,
1759 .LowInReg: X86::AX,
1760 .HighInReg: 0,
1761 .ResultTable: {
1762 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // SDiv
1763 {.OpMulDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SRem
1764 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AL, .IsOpSigned: U}, // UDiv
1765 {.OpMulDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // URem
1766 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AL, .IsOpSigned: S}, // Mul
1767 {.OpMulDivRem: X86::IMUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .ResultReg: X86::AH, .IsOpSigned: S}, // SMulH
1768 {.OpMulDivRem: X86::MUL8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .ResultReg: X86::AH, .IsOpSigned: U}, // UMulH
1769 }}, // i8
1770 {.SizeInBits: 16,
1771 .LowInReg: X86::AX,
1772 .HighInReg: X86::DX,
1773 .ResultTable: {
1774 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // SDiv
1775 {.OpMulDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SRem
1776 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: U}, // UDiv
1777 {.OpMulDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // URem
1778 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::AX, .IsOpSigned: S}, // Mul
1779 {.OpMulDivRem: X86::IMUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: S}, // SMulH
1780 {.OpMulDivRem: X86::MUL16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::DX, .IsOpSigned: U}, // UMulH
1781 }}, // i16
1782 {.SizeInBits: 32,
1783 .LowInReg: X86::EAX,
1784 .HighInReg: X86::EDX,
1785 .ResultTable: {
1786 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // SDiv
1787 {.OpMulDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SRem
1788 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: U}, // UDiv
1789 {.OpMulDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // URem
1790 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EAX, .IsOpSigned: S}, // Mul
1791 {.OpMulDivRem: X86::IMUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: S}, // SMulH
1792 {.OpMulDivRem: X86::MUL32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::EDX, .IsOpSigned: U}, // UMulH
1793 }}, // i32
1794 {.SizeInBits: 64,
1795 .LowInReg: X86::RAX,
1796 .HighInReg: X86::RDX,
1797 .ResultTable: {
1798 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // SDiv
1799 {.OpMulDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SRem
1800 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: U}, // UDiv
1801 {.OpMulDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // URem
1802 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RAX, .IsOpSigned: S}, // Mul
1803 {.OpMulDivRem: X86::IMUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: S}, // SMulH
1804 {.OpMulDivRem: X86::MUL64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .ResultReg: X86::RDX, .IsOpSigned: U}, // UMulH
1805 }}, // i64
1806 };
1807
1808 auto OpEntryIt = llvm::find_if(Range: OpTable, P: [RegTy](const MulDivRemEntry &El) {
1809 return El.SizeInBits == RegTy.getSizeInBits();
1810 });
1811 if (OpEntryIt == std::end(arr: OpTable))
1812 return false;
1813
1814 unsigned OpIndex;
1815 switch (I.getOpcode()) {
1816 default:
1817 llvm_unreachable("Unexpected mul/div/rem opcode");
1818 case TargetOpcode::G_SDIV:
1819 OpIndex = 0;
1820 break;
1821 case TargetOpcode::G_SREM:
1822 OpIndex = 1;
1823 break;
1824 case TargetOpcode::G_UDIV:
1825 OpIndex = 2;
1826 break;
1827 case TargetOpcode::G_UREM:
1828 OpIndex = 3;
1829 break;
1830 case TargetOpcode::G_MUL:
1831 OpIndex = 4;
1832 break;
1833 case TargetOpcode::G_SMULH:
1834 OpIndex = 5;
1835 break;
1836 case TargetOpcode::G_UMULH:
1837 OpIndex = 6;
1838 break;
1839 }
1840
1841 const MulDivRemEntry &TypeEntry = *OpEntryIt;
1842 const MulDivRemEntry::MulDivRemResult &OpEntry =
1843 TypeEntry.ResultTable[OpIndex];
1844
1845 const TargetRegisterClass *RegRC = getRegClass(Ty: RegTy, RB: *RegRB);
1846 if (!RBI.constrainGenericRegister(Reg: Op1Reg, RC: *RegRC, MRI) ||
1847 !RBI.constrainGenericRegister(Reg: Op2Reg, RC: *RegRC, MRI) ||
1848 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RegRC, MRI)) {
1849 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1850 << " operand\n");
1851 return false;
1852 }
1853
1854 // Move op1 into low-order input register.
1855 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpCopy),
1856 DestReg: TypeEntry.LowInReg)
1857 .addReg(RegNo: Op1Reg);
1858
1859 // Zero-extend or sign-extend into high-order input register.
1860 if (OpEntry.OpSignExtend) {
1861 if (OpEntry.IsOpSigned)
1862 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1863 MCID: TII.get(Opcode: OpEntry.OpSignExtend));
1864 else {
1865 Register Zero32 = MRI.createVirtualRegister(RegClass: &X86::GR32RegClass);
1866 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::MOV32r0), DestReg: Zero32)
1867 .setOperandDead(1);
1868
1869 // Copy the zero into the appropriate sub/super/identical physical
1870 // register. Unfortunately the operations needed are not uniform enough
1871 // to fit neatly into the table above.
1872 if (RegTy.getSizeInBits() == 16) {
1873 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1874 DestReg: TypeEntry.HighInReg)
1875 .addReg(RegNo: Zero32, Flags: {}, SubReg: X86::sub_16bit);
1876 } else if (RegTy.getSizeInBits() == 32) {
1877 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy),
1878 DestReg: TypeEntry.HighInReg)
1879 .addReg(RegNo: Zero32);
1880 } else if (RegTy.getSizeInBits() == 64) {
1881 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(),
1882 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: TypeEntry.HighInReg)
1883 .addReg(RegNo: Zero32)
1884 .addImm(Val: X86::sub_32bit);
1885 }
1886 }
1887 }
1888
1889 // Generate the DIV/IDIV/MUL/IMUL instruction.
1890 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: OpEntry.OpMulDivRem))
1891 .addReg(RegNo: Op2Reg);
1892
1893 // For i8 remainder, we can't reference ah directly, as we'll end
1894 // up with bogus copies like %r9b = COPY %ah. Reference ax
1895 // instead to prevent ah references in a rex instruction.
1896 //
1897 // The current assumption of the fast register allocator is that isel
1898 // won't generate explicit references to the GR8_NOREX registers. If
1899 // the allocator and/or the backend get enhanced to be more robust in
1900 // that regard, this can be, and should be, removed.
1901 if (OpEntry.ResultReg == X86::AH && STI.is64Bit()) {
1902 Register SourceSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1903 Register ResultSuperReg = MRI.createVirtualRegister(RegClass: &X86::GR16RegClass);
1904 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: Copy), DestReg: SourceSuperReg)
1905 .addReg(RegNo: X86::AX);
1906
1907 // Shift AX right by 8 bits instead of using AH.
1908 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: X86::SHR16ri),
1909 DestReg: ResultSuperReg)
1910 .addReg(RegNo: SourceSuperReg)
1911 .addImm(Val: 8);
1912
1913 // Now reference the 8-bit subreg of the result.
1914 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1915 DestReg: DstReg)
1916 .addReg(RegNo: ResultSuperReg, Flags: {}, SubReg: X86::sub_8bit);
1917 } else {
1918 BuildMI(BB&: *I.getParent(), I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: TargetOpcode::COPY),
1919 DestReg: DstReg)
1920 .addReg(RegNo: OpEntry.ResultReg);
1921 }
1922 I.eraseFromParent();
1923
1924 return true;
1925}
1926
1927bool X86InstructionSelector::selectSelect(MachineInstr &I,
1928 MachineRegisterInfo &MRI,
1929 MachineFunction &MF) const {
1930 GSelect &Sel = cast<GSelect>(Val&: I);
1931 Register DstReg = Sel.getReg(Idx: 0);
1932 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::TEST32rr))
1933 .addReg(RegNo: Sel.getCondReg())
1934 .addReg(RegNo: Sel.getCondReg());
1935
1936 unsigned OpCmp;
1937 LLT Ty = MRI.getType(Reg: DstReg);
1938 if (Ty.getSizeInBits() == 80) {
1939 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: X86::CMOVE_Fp80),
1940 DestReg: DstReg)
1941 .addReg(RegNo: Sel.getTrueReg())
1942 .addReg(RegNo: Sel.getFalseReg());
1943 } else {
1944 switch (Ty.getSizeInBits()) {
1945 default:
1946 return false;
1947 case 8:
1948 OpCmp = X86::CMOV_GR8;
1949 break;
1950 case 16:
1951 OpCmp = STI.canUseCMOV() ? X86::CMOV16rr : X86::CMOV_GR16;
1952 break;
1953 case 32:
1954 OpCmp = STI.canUseCMOV() ? X86::CMOV32rr : X86::CMOV_GR32;
1955 break;
1956 case 64:
1957 assert(STI.is64Bit() && STI.canUseCMOV());
1958 OpCmp = X86::CMOV64rr;
1959 break;
1960 }
1961 BuildMI(BB&: *Sel.getParent(), I&: Sel, MIMD: Sel.getDebugLoc(), MCID: TII.get(Opcode: OpCmp), DestReg: DstReg)
1962 .addReg(RegNo: Sel.getTrueReg())
1963 .addReg(RegNo: Sel.getFalseReg())
1964 .addImm(Val: X86::COND_E);
1965 }
1966 const TargetRegisterClass *DstRC = getRegClass(Ty, Reg: DstReg, MRI);
1967 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI)) {
1968 LLVM_DEBUG(dbgs() << "Failed to constrain CMOV\n");
1969 return false;
1970 }
1971
1972 Sel.eraseFromParent();
1973 return true;
1974}
1975
1976InstructionSelector::ComplexRendererFns
1977X86InstructionSelector::selectAddr(MachineOperand &Root) const {
1978 MachineInstr *MI = Root.getParent();
1979 MachineIRBuilder MIRBuilder(*MI);
1980
1981 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
1982 MachineInstr *Ptr = MRI.getVRegDef(Reg: Root.getReg());
1983 X86AddressMode AM;
1984 if (!X86SelectAddress(I&: *Ptr, TM, MRI, STI, AM))
1985 return std::nullopt;
1986
1987 if (AM.IndexReg)
1988 return std::nullopt;
1989
1990 return {// Base
1991 {[=](MachineInstrBuilder &MIB) {
1992 if (AM.BaseType == X86AddressMode::RegBase)
1993 MIB.addUse(RegNo: AM.Base.Reg);
1994 else {
1995 assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
1996 "Unknown type of address base");
1997 MIB.addFrameIndex(Idx: AM.Base.FrameIndex);
1998 }
1999 },
2000 // Scale
2001 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: AM.Scale); },
2002 // Index
2003 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); },
2004 // Disp
2005 [=](MachineInstrBuilder &MIB) {
2006 if (AM.GV)
2007 MIB.addGlobalAddress(GV: AM.GV, Offset: AM.Disp, TargetFlags: AM.GVOpFlags);
2008 else if (AM.CP)
2009 MIB.addConstantPoolIndex(Idx: AM.Disp, Offset: 0, TargetFlags: AM.GVOpFlags);
2010 else
2011 MIB.addImm(Val: AM.Disp);
2012 },
2013 // Segment
2014 [=](MachineInstrBuilder &MIB) { MIB.addUse(RegNo: 0); }}};
2015}
2016
2017InstructionSelector *
2018llvm::createX86InstructionSelector(const X86TargetMachine &TM,
2019 const X86Subtarget &Subtarget,
2020 const X86RegisterBankInfo &RBI) {
2021 return new X86InstructionSelector(TM, Subtarget, RBI);
2022}
2023