1//===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the X86-specific support for the FastISel class. Much
10// of the target-specific code is generated by tablegen in the file
11// X86GenFastISel.inc, which is #included here.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86CallingConv.h"
17#include "X86InstrBuilder.h"
18#include "X86InstrInfo.h"
19#include "X86MachineFunctionInfo.h"
20#include "X86RegisterInfo.h"
21#include "X86Subtarget.h"
22#include "X86TargetMachine.h"
23#include "llvm/Analysis/BranchProbabilityInfo.h"
24#include "llvm/CodeGen/FastISel.h"
25#include "llvm/CodeGen/FunctionLoweringInfo.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/IR/CallingConv.h"
30#include "llvm/IR/DebugInfo.h"
31#include "llvm/IR/DerivedTypes.h"
32#include "llvm/IR/GetElementPtrTypeIterator.h"
33#include "llvm/IR/GlobalVariable.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/IntrinsicsX86.h"
37#include "llvm/IR/Module.h"
38#include "llvm/IR/Operator.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCSymbol.h"
41#include "llvm/Support/ErrorHandling.h"
42#include "llvm/Target/TargetOptions.h"
43using namespace llvm;
44
45namespace {
46
47class X86FastISel final : public FastISel {
48 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
49 /// make the right decision when generating code for different targets.
50 const X86Subtarget *Subtarget;
51
52public:
53 explicit X86FastISel(FunctionLoweringInfo &funcInfo,
54 const TargetLibraryInfo *libInfo,
55 const LibcallLoweringInfo *libcallLowering)
56 : FastISel(funcInfo, libInfo, libcallLowering) {
57 Subtarget = &funcInfo.MF->getSubtarget<X86Subtarget>();
58 }
59
60 bool fastSelectInstruction(const Instruction *I) override;
61
62 /// The specified machine instr operand is a vreg, and that
63 /// vreg is being provided by the specified load instruction. If possible,
64 /// try to fold the load as an operand to the instruction, returning true if
65 /// possible.
66 bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
67 const LoadInst *LI) override;
68
69 bool fastLowerArguments() override;
70 bool fastLowerCall(CallLoweringInfo &CLI) override;
71 bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;
72
73#include "X86GenFastISel.inc"
74
75private:
76 bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT,
77 const DebugLoc &DL);
78
79 bool X86FastEmitLoad(MVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
80 Register &ResultReg, unsigned Alignment = 1);
81
82 bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
83 MachineMemOperand *MMO = nullptr, bool Aligned = false);
84 bool X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
85 MachineMemOperand *MMO = nullptr, bool Aligned = false);
86
87 bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, Register Src, EVT SrcVT,
88 Register &ResultReg);
89
90 bool X86SelectAddress(const Value *V, X86AddressMode &AM);
91 bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);
92
93 bool X86SelectLoad(const Instruction *I);
94
95 bool X86SelectStore(const Instruction *I);
96
97 bool X86SelectRet(const Instruction *I);
98
99 bool X86SelectCmp(const Instruction *I);
100
101 bool X86SelectZExt(const Instruction *I);
102
103 bool X86SelectSExt(const Instruction *I);
104
105 bool X86SelectBranch(const Instruction *I);
106
107 bool X86SelectShift(const Instruction *I);
108
109 bool X86SelectDivRem(const Instruction *I);
110
111 bool X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I);
112
113 bool X86FastEmitSSESelect(MVT RetVT, const Instruction *I);
114
115 bool X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I);
116
117 bool X86SelectSelect(const Instruction *I);
118
119 bool X86SelectTrunc(const Instruction *I);
120
121 bool X86SelectFPExtOrFPTrunc(const Instruction *I, unsigned Opc,
122 const TargetRegisterClass *RC);
123
124 bool X86SelectFPExt(const Instruction *I);
125 bool X86SelectFPTrunc(const Instruction *I);
126 bool X86SelectSIToFP(const Instruction *I);
127 bool X86SelectUIToFP(const Instruction *I);
128 bool X86SelectIntToFP(const Instruction *I, bool IsSigned);
129 bool X86SelectBitCast(const Instruction *I);
130
131 const X86InstrInfo *getInstrInfo() const {
132 return Subtarget->getInstrInfo();
133 }
134 const X86TargetMachine *getTargetMachine() const {
135 return static_cast<const X86TargetMachine *>(&TM);
136 }
137
138 bool handleConstantAddresses(const Value *V, X86AddressMode &AM);
139
140 Register X86MaterializeInt(const ConstantInt *CI, MVT VT);
141 Register X86MaterializeFP(const ConstantFP *CFP, MVT VT);
142 Register X86MaterializeGV(const GlobalValue *GV, MVT VT);
143 Register fastMaterializeConstant(const Constant *C) override;
144
145 Register fastMaterializeAlloca(const AllocaInst *C) override;
146
147 Register fastMaterializeFloatZero(const ConstantFP *CF) override;
148
149 /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is
150 /// computed in an SSE register, not on the X87 floating point stack.
151 bool isScalarFPTypeInSSEReg(EVT VT) const {
152 return (VT == MVT::f64 && Subtarget->hasSSE2()) ||
153 (VT == MVT::f32 && Subtarget->hasSSE1()) || VT == MVT::f16;
154 }
155
156 bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);
157
158 bool IsMemcpySmall(uint64_t Len);
159
160 bool TryEmitSmallMemcpy(X86AddressMode DestAM,
161 X86AddressMode SrcAM, uint64_t Len);
162
163 bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
164 const Value *Cond);
165
166 const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
167 X86AddressMode &AM);
168
169 Register fastEmitInst_rrrr(unsigned MachineInstOpcode,
170 const TargetRegisterClass *RC, Register Op0,
171 Register Op1, Register Op2, Register Op3);
172};
173
174} // end anonymous namespace.
175
176static std::pair<unsigned, bool>
177getX86SSEConditionCode(CmpInst::Predicate Predicate) {
178 unsigned CC;
179 bool NeedSwap = false;
180
181 // SSE Condition code mapping:
182 // 0 - EQ
183 // 1 - LT
184 // 2 - LE
185 // 3 - UNORD
186 // 4 - NEQ
187 // 5 - NLT
188 // 6 - NLE
189 // 7 - ORD
190 switch (Predicate) {
191 default: llvm_unreachable("Unexpected predicate");
192 case CmpInst::FCMP_OEQ: CC = 0; break;
193 case CmpInst::FCMP_OGT: NeedSwap = true; [[fallthrough]];
194 case CmpInst::FCMP_OLT: CC = 1; break;
195 case CmpInst::FCMP_OGE: NeedSwap = true; [[fallthrough]];
196 case CmpInst::FCMP_OLE: CC = 2; break;
197 case CmpInst::FCMP_UNO: CC = 3; break;
198 case CmpInst::FCMP_UNE: CC = 4; break;
199 case CmpInst::FCMP_ULE: NeedSwap = true; [[fallthrough]];
200 case CmpInst::FCMP_UGE: CC = 5; break;
201 case CmpInst::FCMP_ULT: NeedSwap = true; [[fallthrough]];
202 case CmpInst::FCMP_UGT: CC = 6; break;
203 case CmpInst::FCMP_ORD: CC = 7; break;
204 case CmpInst::FCMP_UEQ: CC = 8; break;
205 case CmpInst::FCMP_ONE: CC = 12; break;
206 }
207
208 return std::make_pair(x&: CC, y&: NeedSwap);
209}
210
211/// Adds a complex addressing mode to the given machine instr builder.
212/// Note, this will constrain the index register. If its not possible to
213/// constrain the given index register, then a new one will be created. The
214/// IndexReg field of the addressing mode will be updated to match in this case.
215const MachineInstrBuilder &
216X86FastISel::addFullAddress(const MachineInstrBuilder &MIB,
217 X86AddressMode &AM) {
218 // First constrain the index register. It needs to be a GR64_NOSP.
219 AM.IndexReg = constrainOperandRegClass(II: MIB->getDesc(), Op: AM.IndexReg,
220 OpNum: MIB->getNumOperands() +
221 X86::AddrIndexReg);
222 return ::addFullAddress(MIB, AM);
223}
224
225/// Check if it is possible to fold the condition from the XALU intrinsic
226/// into the user. The condition code will only be updated on success.
227bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
228 const Value *Cond) {
229 if (!isa<ExtractValueInst>(Val: Cond))
230 return false;
231
232 const auto *EV = cast<ExtractValueInst>(Val: Cond);
233 if (!isa<IntrinsicInst>(Val: EV->getAggregateOperand()))
234 return false;
235
236 const auto *II = cast<IntrinsicInst>(Val: EV->getAggregateOperand());
237 MVT RetVT;
238 const Function *Callee = II->getCalledFunction();
239 Type *RetTy =
240 cast<StructType>(Val: Callee->getReturnType())->getTypeAtIndex(N: 0U);
241 if (!isTypeLegal(Ty: RetTy, VT&: RetVT))
242 return false;
243
244 if (RetVT != MVT::i32 && RetVT != MVT::i64)
245 return false;
246
247 X86::CondCode TmpCC;
248 switch (II->getIntrinsicID()) {
249 default: return false;
250 case Intrinsic::sadd_with_overflow:
251 case Intrinsic::ssub_with_overflow:
252 case Intrinsic::smul_with_overflow:
253 case Intrinsic::umul_with_overflow: TmpCC = X86::COND_O; break;
254 case Intrinsic::uadd_with_overflow:
255 case Intrinsic::usub_with_overflow: TmpCC = X86::COND_B; break;
256 }
257
258 // Check if both instructions are in the same basic block.
259 if (II->getParent() != I->getParent())
260 return false;
261
262 // Make sure nothing is in the way
263 BasicBlock::const_iterator Start(I);
264 BasicBlock::const_iterator End(II);
265 for (auto Itr = std::prev(x: Start); Itr != End; --Itr) {
266 // We only expect extractvalue instructions between the intrinsic and the
267 // instruction to be selected.
268 if (!isa<ExtractValueInst>(Val: Itr))
269 return false;
270
271 // Check that the extractvalue operand comes from the intrinsic.
272 const auto *EVI = cast<ExtractValueInst>(Val&: Itr);
273 if (EVI->getAggregateOperand() != II)
274 return false;
275 }
276
277 // Make sure no potentially eflags clobbering phi moves can be inserted in
278 // between.
279 auto HasPhis = [](const BasicBlock *Succ) { return !Succ->phis().empty(); };
280 if (I->isTerminator() && llvm::any_of(Range: successors(I), P: HasPhis))
281 return false;
282
283 // Make sure there are no potentially eflags clobbering constant
284 // materializations in between.
285 if (llvm::any_of(Range: I->operands(), P: [](Value *V) { return isa<Constant>(Val: V); }))
286 return false;
287
288 CC = TmpCC;
289 return true;
290}
291
292bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
293 EVT evt = TLI.getValueType(DL, Ty, /*AllowUnknown=*/true);
294 if (evt == MVT::Other || !evt.isSimple())
295 // Unhandled type. Halt "fast" selection and bail.
296 return false;
297
298 VT = evt.getSimpleVT();
299 // For now, require SSE/SSE2 for performing floating-point operations,
300 // since x87 requires additional work.
301 if (VT == MVT::f64 && !Subtarget->hasSSE2())
302 return false;
303 if (VT == MVT::f32 && !Subtarget->hasSSE1())
304 return false;
305 // Similarly, no f80 support yet.
306 if (VT == MVT::f80)
307 return false;
308 // We only handle legal types. For example, on x86-32 the instruction
309 // selector contains all of the 64-bit instructions from x86-64,
310 // under the assumption that i64 won't be used if the target doesn't
311 // support it.
312 return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);
313}
314
315/// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
316/// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
317/// Return true and the result register by reference if it is possible.
318bool X86FastISel::X86FastEmitLoad(MVT VT, X86AddressMode &AM,
319 MachineMemOperand *MMO, Register &ResultReg,
320 unsigned Alignment) {
321 bool HasSSE1 = Subtarget->hasSSE1();
322 bool HasSSE2 = Subtarget->hasSSE2();
323 bool HasSSE41 = Subtarget->hasSSE41();
324 bool HasAVX = Subtarget->hasAVX();
325 bool HasAVX2 = Subtarget->hasAVX2();
326 bool HasAVX512 = Subtarget->hasAVX512();
327 bool HasVLX = Subtarget->hasVLX();
328 bool IsNonTemporal = MMO && MMO->isNonTemporal();
329
330 // Treat i1 loads the same as i8 loads. Masking will be done when storing.
331 if (VT == MVT::i1)
332 VT = MVT::i8;
333
334 // Get opcode and regclass of the output for the given load instruction.
335 unsigned Opc = 0;
336 switch (VT.SimpleTy) {
337 default: return false;
338 case MVT::i8:
339 Opc = X86::MOV8rm;
340 break;
341 case MVT::i16:
342 Opc = X86::MOV16rm;
343 break;
344 case MVT::i32:
345 Opc = X86::MOV32rm;
346 break;
347 case MVT::i64:
348 // Must be in x86-64 mode.
349 Opc = X86::MOV64rm;
350 break;
351 case MVT::f32:
352 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
353 : HasAVX ? X86::VMOVSSrm_alt
354 : HasSSE1 ? X86::MOVSSrm_alt
355 : X86::LD_Fp32m;
356 break;
357 case MVT::f64:
358 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
359 : HasAVX ? X86::VMOVSDrm_alt
360 : HasSSE2 ? X86::MOVSDrm_alt
361 : X86::LD_Fp64m;
362 break;
363 case MVT::f80:
364 // No f80 support yet.
365 return false;
366 case MVT::v4f32:
367 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
368 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
369 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
370 else if (Alignment >= 16)
371 Opc = HasVLX ? X86::VMOVAPSZ128rm :
372 HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm;
373 else
374 Opc = HasVLX ? X86::VMOVUPSZ128rm :
375 HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm;
376 break;
377 case MVT::v2f64:
378 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
379 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
380 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
381 else if (Alignment >= 16)
382 Opc = HasVLX ? X86::VMOVAPDZ128rm :
383 HasAVX ? X86::VMOVAPDrm : X86::MOVAPDrm;
384 else
385 Opc = HasVLX ? X86::VMOVUPDZ128rm :
386 HasAVX ? X86::VMOVUPDrm : X86::MOVUPDrm;
387 break;
388 case MVT::v4i32:
389 case MVT::v2i64:
390 case MVT::v8i16:
391 case MVT::v16i8:
392 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
393 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
394 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
395 else if (Alignment >= 16)
396 Opc = HasVLX ? X86::VMOVDQA64Z128rm :
397 HasAVX ? X86::VMOVDQArm : X86::MOVDQArm;
398 else
399 Opc = HasVLX ? X86::VMOVDQU64Z128rm :
400 HasAVX ? X86::VMOVDQUrm : X86::MOVDQUrm;
401 break;
402 case MVT::v8f32:
403 assert(HasAVX);
404 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
405 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
406 else if (IsNonTemporal && Alignment >= 16)
407 return false; // Force split for X86::VMOVNTDQArm
408 else if (Alignment >= 32)
409 Opc = HasVLX ? X86::VMOVAPSZ256rm : X86::VMOVAPSYrm;
410 else
411 Opc = HasVLX ? X86::VMOVUPSZ256rm : X86::VMOVUPSYrm;
412 break;
413 case MVT::v4f64:
414 assert(HasAVX);
415 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
416 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
417 else if (IsNonTemporal && Alignment >= 16)
418 return false; // Force split for X86::VMOVNTDQArm
419 else if (Alignment >= 32)
420 Opc = HasVLX ? X86::VMOVAPDZ256rm : X86::VMOVAPDYrm;
421 else
422 Opc = HasVLX ? X86::VMOVUPDZ256rm : X86::VMOVUPDYrm;
423 break;
424 case MVT::v8i32:
425 case MVT::v4i64:
426 case MVT::v16i16:
427 case MVT::v32i8:
428 assert(HasAVX);
429 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
430 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
431 else if (IsNonTemporal && Alignment >= 16)
432 return false; // Force split for X86::VMOVNTDQArm
433 else if (Alignment >= 32)
434 Opc = HasVLX ? X86::VMOVDQA64Z256rm : X86::VMOVDQAYrm;
435 else
436 Opc = HasVLX ? X86::VMOVDQU64Z256rm : X86::VMOVDQUYrm;
437 break;
438 case MVT::v16f32:
439 assert(HasAVX512);
440 if (IsNonTemporal && Alignment >= 64)
441 Opc = X86::VMOVNTDQAZrm;
442 else
443 Opc = (Alignment >= 64) ? X86::VMOVAPSZrm : X86::VMOVUPSZrm;
444 break;
445 case MVT::v8f64:
446 assert(HasAVX512);
447 if (IsNonTemporal && Alignment >= 64)
448 Opc = X86::VMOVNTDQAZrm;
449 else
450 Opc = (Alignment >= 64) ? X86::VMOVAPDZrm : X86::VMOVUPDZrm;
451 break;
452 case MVT::v8i64:
453 case MVT::v16i32:
454 case MVT::v32i16:
455 case MVT::v64i8:
456 assert(HasAVX512);
457 // Note: There are a lot more choices based on type with AVX-512, but
458 // there's really no advantage when the load isn't masked.
459 if (IsNonTemporal && Alignment >= 64)
460 Opc = X86::VMOVNTDQAZrm;
461 else
462 Opc = (Alignment >= 64) ? X86::VMOVDQA64Zrm : X86::VMOVDQU64Zrm;
463 break;
464 }
465
466 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
467
468 ResultReg = createResultReg(RC);
469 MachineInstrBuilder MIB =
470 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: ResultReg);
471 addFullAddress(MIB, AM);
472 if (MMO)
473 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
474 return true;
475}
476
477/// X86FastEmitStore - Emit a machine instruction to store a value Val of
478/// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
479/// and a displacement offset, or a GlobalAddress,
480/// i.e. V. Return true if it is possible.
481bool X86FastISel::X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
482 MachineMemOperand *MMO, bool Aligned) {
483 bool HasSSE1 = Subtarget->hasSSE1();
484 bool HasSSE2 = Subtarget->hasSSE2();
485 bool HasSSE4A = Subtarget->hasSSE4A();
486 bool HasAVX = Subtarget->hasAVX();
487 bool HasAVX512 = Subtarget->hasAVX512();
488 bool HasVLX = Subtarget->hasVLX();
489 bool IsNonTemporal = MMO && MMO->isNonTemporal();
490
491 // Get opcode and regclass of the output for the given store instruction.
492 unsigned Opc = 0;
493 switch (VT.getSimpleVT().SimpleTy) {
494 case MVT::f80: // No f80 support yet.
495 default: return false;
496 case MVT::i1: {
497 // Mask out all but lowest bit.
498 Register AndResult = createResultReg(RC: &X86::GR8RegClass);
499 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
500 MCID: TII.get(Opcode: X86::AND8ri), DestReg: AndResult)
501 .addReg(RegNo: ValReg).addImm(Val: 1);
502 ValReg = AndResult;
503 [[fallthrough]]; // handle i1 as i8.
504 }
505 case MVT::i8: Opc = X86::MOV8mr; break;
506 case MVT::i16: Opc = X86::MOV16mr; break;
507 case MVT::i32:
508 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTImr : X86::MOV32mr;
509 break;
510 case MVT::i64:
511 // Must be in x86-64 mode.
512 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTI_64mr : X86::MOV64mr;
513 break;
514 case MVT::f32:
515 if (HasSSE1) {
516 if (IsNonTemporal && HasSSE4A)
517 Opc = X86::MOVNTSS;
518 else
519 Opc = HasAVX512 ? X86::VMOVSSZmr :
520 HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
521 } else
522 Opc = X86::ST_Fp32m;
523 break;
524 case MVT::f64:
525 if (HasSSE2) {
526 if (IsNonTemporal && HasSSE4A)
527 Opc = X86::MOVNTSD;
528 else
529 Opc = HasAVX512 ? X86::VMOVSDZmr :
530 HasAVX ? X86::VMOVSDmr : X86::MOVSDmr;
531 } else
532 Opc = X86::ST_Fp64m;
533 break;
534 case MVT::x86mmx:
535 Opc = (IsNonTemporal && HasSSE1) ? X86::MMX_MOVNTQmr : X86::MMX_MOVQ64mr;
536 break;
537 case MVT::v4f32:
538 if (Aligned) {
539 if (IsNonTemporal)
540 Opc = HasVLX ? X86::VMOVNTPSZ128mr :
541 HasAVX ? X86::VMOVNTPSmr : X86::MOVNTPSmr;
542 else
543 Opc = HasVLX ? X86::VMOVAPSZ128mr :
544 HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr;
545 } else
546 Opc = HasVLX ? X86::VMOVUPSZ128mr :
547 HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr;
548 break;
549 case MVT::v2f64:
550 if (Aligned) {
551 if (IsNonTemporal)
552 Opc = HasVLX ? X86::VMOVNTPDZ128mr :
553 HasAVX ? X86::VMOVNTPDmr : X86::MOVNTPDmr;
554 else
555 Opc = HasVLX ? X86::VMOVAPDZ128mr :
556 HasAVX ? X86::VMOVAPDmr : X86::MOVAPDmr;
557 } else
558 Opc = HasVLX ? X86::VMOVUPDZ128mr :
559 HasAVX ? X86::VMOVUPDmr : X86::MOVUPDmr;
560 break;
561 case MVT::v4i32:
562 case MVT::v2i64:
563 case MVT::v8i16:
564 case MVT::v16i8:
565 if (Aligned) {
566 if (IsNonTemporal)
567 Opc = HasVLX ? X86::VMOVNTDQZ128mr :
568 HasAVX ? X86::VMOVNTDQmr : X86::MOVNTDQmr;
569 else
570 Opc = HasVLX ? X86::VMOVDQA64Z128mr :
571 HasAVX ? X86::VMOVDQAmr : X86::MOVDQAmr;
572 } else
573 Opc = HasVLX ? X86::VMOVDQU64Z128mr :
574 HasAVX ? X86::VMOVDQUmr : X86::MOVDQUmr;
575 break;
576 case MVT::v8f32:
577 assert(HasAVX);
578 if (Aligned) {
579 if (IsNonTemporal)
580 Opc = HasVLX ? X86::VMOVNTPSZ256mr : X86::VMOVNTPSYmr;
581 else
582 Opc = HasVLX ? X86::VMOVAPSZ256mr : X86::VMOVAPSYmr;
583 } else
584 Opc = HasVLX ? X86::VMOVUPSZ256mr : X86::VMOVUPSYmr;
585 break;
586 case MVT::v4f64:
587 assert(HasAVX);
588 if (Aligned) {
589 if (IsNonTemporal)
590 Opc = HasVLX ? X86::VMOVNTPDZ256mr : X86::VMOVNTPDYmr;
591 else
592 Opc = HasVLX ? X86::VMOVAPDZ256mr : X86::VMOVAPDYmr;
593 } else
594 Opc = HasVLX ? X86::VMOVUPDZ256mr : X86::VMOVUPDYmr;
595 break;
596 case MVT::v8i32:
597 case MVT::v4i64:
598 case MVT::v16i16:
599 case MVT::v32i8:
600 assert(HasAVX);
601 if (Aligned) {
602 if (IsNonTemporal)
603 Opc = HasVLX ? X86::VMOVNTDQZ256mr : X86::VMOVNTDQYmr;
604 else
605 Opc = HasVLX ? X86::VMOVDQA64Z256mr : X86::VMOVDQAYmr;
606 } else
607 Opc = HasVLX ? X86::VMOVDQU64Z256mr : X86::VMOVDQUYmr;
608 break;
609 case MVT::v16f32:
610 assert(HasAVX512);
611 if (Aligned)
612 Opc = IsNonTemporal ? X86::VMOVNTPSZmr : X86::VMOVAPSZmr;
613 else
614 Opc = X86::VMOVUPSZmr;
615 break;
616 case MVT::v8f64:
617 assert(HasAVX512);
618 if (Aligned) {
619 Opc = IsNonTemporal ? X86::VMOVNTPDZmr : X86::VMOVAPDZmr;
620 } else
621 Opc = X86::VMOVUPDZmr;
622 break;
623 case MVT::v8i64:
624 case MVT::v16i32:
625 case MVT::v32i16:
626 case MVT::v64i8:
627 assert(HasAVX512);
628 // Note: There are a lot more choices based on type with AVX-512, but
629 // there's really no advantage when the store isn't masked.
630 if (Aligned)
631 Opc = IsNonTemporal ? X86::VMOVNTDQZmr : X86::VMOVDQA64Zmr;
632 else
633 Opc = X86::VMOVDQU64Zmr;
634 break;
635 }
636
637 const MCInstrDesc &Desc = TII.get(Opcode: Opc);
638 // Some of the instructions in the previous switch use FR128 instead
639 // of FR32 for ValReg. Make sure the register we feed the instruction
640 // matches its register class constraints.
641 // Note: This is fine to do a copy from FR32 to FR128, this is the
642 // same registers behind the scene and actually why it did not trigger
643 // any bugs before.
644 ValReg = constrainOperandRegClass(II: Desc, Op: ValReg, OpNum: Desc.getNumOperands() - 1);
645 MachineInstrBuilder MIB =
646 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: Desc);
647 addFullAddress(MIB, AM).addReg(RegNo: ValReg);
648 if (MMO)
649 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
650
651 return true;
652}
653
654bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
655 X86AddressMode &AM,
656 MachineMemOperand *MMO, bool Aligned) {
657 // Handle 'null' like i32/i64 0.
658 if (isa<ConstantPointerNull>(Val))
659 Val = Constant::getNullValue(Ty: DL.getIntPtrType(C&: Val->getContext()));
660
661 // If this is a store of a simple constant, fold the constant into the store.
662 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
663 unsigned Opc = 0;
664 bool Signed = true;
665 switch (VT.getSimpleVT().SimpleTy) {
666 default: break;
667 case MVT::i1:
668 Signed = false;
669 [[fallthrough]]; // Handle as i8.
670 case MVT::i8: Opc = X86::MOV8mi; break;
671 case MVT::i16: Opc = X86::MOV16mi; break;
672 case MVT::i32: Opc = X86::MOV32mi; break;
673 case MVT::i64:
674 // Must be a 32-bit sign extended value.
675 if (isInt<32>(x: CI->getSExtValue()))
676 Opc = X86::MOV64mi32;
677 break;
678 }
679
680 if (Opc) {
681 MachineInstrBuilder MIB =
682 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc));
683 addFullAddress(MIB, AM).addImm(Val: Signed ? (uint64_t) CI->getSExtValue()
684 : CI->getZExtValue());
685 if (MMO)
686 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
687 return true;
688 }
689 }
690
691 Register ValReg = getRegForValue(V: Val);
692 if (!ValReg)
693 return false;
694
695 return X86FastEmitStore(VT, ValReg, AM, MMO, Aligned);
696}
697
698/// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
699/// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.
700/// ISD::SIGN_EXTEND).
701bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, Register Src,
702 EVT SrcVT, Register &ResultReg) {
703 Register RR = fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(), Opcode: Opc, Op0: Src);
704 if (!RR)
705 return false;
706
707 ResultReg = RR;
708 return true;
709}
710
711bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
712 // Handle constant address.
713 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
714 // Can't handle alternate code models yet.
715 if (TM.getCodeModel() != CodeModel::Small &&
716 TM.getCodeModel() != CodeModel::Medium)
717 return false;
718
719 // Can't handle large objects yet.
720 if (TM.isLargeGlobalValue(GV))
721 return false;
722
723 // Can't handle TLS yet.
724 if (GV->isThreadLocal())
725 return false;
726
727 // Can't handle !absolute_symbol references yet.
728 if (GV->isAbsoluteSymbolRef())
729 return false;
730
731 // RIP-relative addresses can't have additional register operands, so if
732 // we've already folded stuff into the addressing mode, just force the
733 // global value into its own register, which we can use as the basereg.
734 if (!Subtarget->isPICStyleRIPRel() ||
735 (AM.Base.Reg == 0 && AM.IndexReg == 0)) {
736 // Okay, we've committed to selecting this global. Set up the address.
737 AM.GV = GV;
738
739 // Allow the subtarget to classify the global.
740 unsigned char GVFlags = Subtarget->classifyGlobalReference(GV);
741
742 // If this reference is relative to the pic base, set it now.
743 if (isGlobalRelativeToPICBase(TargetFlag: GVFlags)) {
744 // FIXME: How do we know Base.Reg is free??
745 AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
746 }
747
748 // Unless the ABI requires an extra load, return a direct reference to
749 // the global.
750 if (!isGlobalStubReference(TargetFlag: GVFlags)) {
751 if (Subtarget->isPICStyleRIPRel()) {
752 // Use rip-relative addressing if we can. Above we verified that the
753 // base and index registers are unused.
754 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
755 AM.Base.Reg = X86::RIP;
756 }
757 AM.GVOpFlags = GVFlags;
758 return true;
759 }
760
761 // Ok, we need to do a load from a stub. If we've already loaded from
762 // this stub, reuse the loaded pointer, otherwise emit the load now.
763 DenseMap<const Value *, Register>::iterator I = LocalValueMap.find(Val: V);
764 Register LoadReg;
765 if (I != LocalValueMap.end() && I->second) {
766 LoadReg = I->second;
767 } else {
768 // Issue load from stub.
769 unsigned Opc = 0;
770 const TargetRegisterClass *RC = nullptr;
771 X86AddressMode StubAM;
772 StubAM.Base.Reg = AM.Base.Reg;
773 StubAM.GV = GV;
774 StubAM.GVOpFlags = GVFlags;
775
776 // Prepare for inserting code in the local-value area.
777 SavePoint SaveInsertPt = enterLocalValueArea();
778
779 if (TLI.getPointerTy(DL) == MVT::i64) {
780 Opc = X86::MOV64rm;
781 RC = &X86::GR64RegClass;
782 } else {
783 Opc = X86::MOV32rm;
784 RC = &X86::GR32RegClass;
785 }
786
787 if (Subtarget->isPICStyleRIPRel() || GVFlags == X86II::MO_GOTPCREL ||
788 GVFlags == X86II::MO_GOTPCREL_NORELAX)
789 StubAM.Base.Reg = X86::RIP;
790
791 LoadReg = createResultReg(RC);
792 MachineInstrBuilder LoadMI =
793 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: LoadReg);
794 addFullAddress(MIB: LoadMI, AM&: StubAM);
795
796 // Ok, back to normal mode.
797 leaveLocalValueArea(Old: SaveInsertPt);
798
799 // Prevent loading GV stub multiple times in same MBB.
800 LocalValueMap[V] = LoadReg;
801 }
802
803 // Now construct the final address. Note that the Disp, Scale,
804 // and Index values may already be set here.
805 AM.Base.Reg = LoadReg;
806 AM.GV = nullptr;
807 return true;
808 }
809 }
810
811 // If all else fails, try to materialize the value in a register.
812 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
813 if (AM.Base.Reg == 0) {
814 AM.Base.Reg = getRegForValue(V);
815 return AM.Base.Reg != 0;
816 }
817 if (AM.IndexReg == 0) {
818 assert(AM.Scale == 1 && "Scale with no index!");
819 AM.IndexReg = getRegForValue(V);
820 return AM.IndexReg != 0;
821 }
822 }
823
824 return false;
825}
826
827/// X86SelectAddress - Attempt to fill in an address from the given value.
828///
829bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
830 SmallVector<const Value *, 32> GEPs;
831redo_gep:
832 const User *U = nullptr;
833 unsigned Opcode = Instruction::UserOp1;
834 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
835 // Don't walk into other basic blocks; it's possible we haven't
836 // visited them yet, so the instructions may not yet be assigned
837 // virtual registers.
838 if (FuncInfo.StaticAllocaMap.count(Val: static_cast<const AllocaInst *>(V)) ||
839 FuncInfo.getMBB(BB: I->getParent()) == FuncInfo.MBB) {
840 Opcode = I->getOpcode();
841 U = I;
842 }
843 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(Val: V)) {
844 Opcode = C->getOpcode();
845 U = C;
846 }
847
848 if (PointerType *Ty = dyn_cast<PointerType>(Val: V->getType()))
849 if (Ty->getAddressSpace() > 255)
850 // Fast instruction selection doesn't support the special
851 // address spaces.
852 return false;
853
854 switch (Opcode) {
855 default: break;
856 case Instruction::BitCast:
857 // Look past bitcasts.
858 return X86SelectAddress(V: U->getOperand(i: 0), AM);
859
860 case Instruction::IntToPtr:
861 // Look past no-op inttoptrs.
862 if (TLI.getValueType(DL, Ty: U->getOperand(i: 0)->getType()) ==
863 TLI.getPointerTy(DL))
864 return X86SelectAddress(V: U->getOperand(i: 0), AM);
865 break;
866
867 case Instruction::PtrToInt:
868 // Look past no-op ptrtoints.
869 if (TLI.getValueType(DL, Ty: U->getType()) == TLI.getPointerTy(DL))
870 return X86SelectAddress(V: U->getOperand(i: 0), AM);
871 break;
872
873 case Instruction::Alloca: {
874 // Do static allocas.
875 const AllocaInst *A = cast<AllocaInst>(Val: V);
876 DenseMap<const AllocaInst *, int>::iterator SI =
877 FuncInfo.StaticAllocaMap.find(Val: A);
878 if (SI != FuncInfo.StaticAllocaMap.end()) {
879 AM.BaseType = X86AddressMode::FrameIndexBase;
880 AM.Base.FrameIndex = SI->second;
881 return true;
882 }
883 break;
884 }
885
886 case Instruction::Add: {
887 // Adds of constants are common and easy enough.
888 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: U->getOperand(i: 1))) {
889 uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue();
890 // They have to fit in the 32-bit signed displacement field though.
891 if (isInt<32>(x: Disp)) {
892 AM.Disp = (uint32_t)Disp;
893 return X86SelectAddress(V: U->getOperand(i: 0), AM);
894 }
895 }
896 break;
897 }
898
899 case Instruction::GetElementPtr: {
900 X86AddressMode SavedAM = AM;
901
902 // Pattern-match simple GEPs.
903 uint64_t Disp = (int32_t)AM.Disp;
904 Register IndexReg = AM.IndexReg;
905 unsigned Scale = AM.Scale;
906 MVT PtrVT = TLI.getValueType(DL, Ty: U->getType()).getSimpleVT();
907
908 gep_type_iterator GTI = gep_type_begin(GEP: U);
909 // Iterate through the indices, folding what we can. Constants can be
910 // folded, and one dynamic index can be handled, if the scale is supported.
911 for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();
912 i != e; ++i, ++GTI) {
913 const Value *Op = *i;
914 if (StructType *STy = GTI.getStructTypeOrNull()) {
915 const StructLayout *SL = DL.getStructLayout(Ty: STy);
916 Disp += SL->getElementOffset(Idx: cast<ConstantInt>(Val: Op)->getZExtValue());
917 continue;
918 }
919
920 // A array/variable index is always of the form i*S where S is the
921 // constant scale size. See if we can push the scale into immediates.
922 uint64_t S = GTI.getSequentialElementStride(DL);
923 for (;;) {
924 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Op)) {
925 // Constant-offset addressing.
926 Disp += CI->getSExtValue() * S;
927 break;
928 }
929 if (canFoldAddIntoGEP(GEP: U, Add: Op)) {
930 // A compatible add with a constant operand. Fold the constant.
931 ConstantInt *CI =
932 cast<ConstantInt>(Val: cast<AddOperator>(Val: Op)->getOperand(i_nocapture: 1));
933 Disp += CI->getSExtValue() * S;
934 // Iterate on the other operand.
935 Op = cast<AddOperator>(Val: Op)->getOperand(i_nocapture: 0);
936 continue;
937 }
938 if (!IndexReg && (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
939 (S == 1 || S == 2 || S == 4 || S == 8)) {
940 // Scaled-index addressing.
941 Scale = S;
942 IndexReg = getRegForGEPIndex(PtrVT, Idx: Op);
943 if (!IndexReg)
944 return false;
945 break;
946 }
947 // Unsupported.
948 goto unsupported_gep;
949 }
950 }
951
952 // Check for displacement overflow.
953 if (!isInt<32>(x: Disp))
954 break;
955
956 AM.IndexReg = IndexReg;
957 AM.Scale = Scale;
958 AM.Disp = (uint32_t)Disp;
959 GEPs.push_back(Elt: V);
960
961 if (const GetElementPtrInst *GEP =
962 dyn_cast<GetElementPtrInst>(Val: U->getOperand(i: 0))) {
963 // Ok, the GEP indices were covered by constant-offset and scaled-index
964 // addressing. Update the address state and move on to examining the base.
965 V = GEP;
966 goto redo_gep;
967 } else if (X86SelectAddress(V: U->getOperand(i: 0), AM)) {
968 return true;
969 }
970
971 // If we couldn't merge the gep value into this addr mode, revert back to
972 // our address and just match the value instead of completely failing.
973 AM = SavedAM;
974
975 for (const Value *I : reverse(C&: GEPs))
976 if (handleConstantAddresses(V: I, AM))
977 return true;
978
979 return false;
980 unsupported_gep:
981 // Ok, the GEP indices weren't all covered.
982 break;
983 }
984 }
985
986 return handleConstantAddresses(V, AM);
987}
988
989/// X86SelectCallAddress - Attempt to fill in an address from the given value.
990///
991bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
992 const User *U = nullptr;
993 unsigned Opcode = Instruction::UserOp1;
994 const Instruction *I = dyn_cast<Instruction>(Val: V);
995 // Record if the value is defined in the same basic block.
996 //
997 // This information is crucial to know whether or not folding an
998 // operand is valid.
999 // Indeed, FastISel generates or reuses a virtual register for all
1000 // operands of all instructions it selects. Obviously, the definition and
1001 // its uses must use the same virtual register otherwise the produced
1002 // code is incorrect.
1003 // Before instruction selection, FunctionLoweringInfo::set sets the virtual
1004 // registers for values that are alive across basic blocks. This ensures
1005 // that the values are consistently set between across basic block, even
1006 // if different instruction selection mechanisms are used (e.g., a mix of
1007 // SDISel and FastISel).
1008 // For values local to a basic block, the instruction selection process
1009 // generates these virtual registers with whatever method is appropriate
1010 // for its needs. In particular, FastISel and SDISel do not share the way
1011 // local virtual registers are set.
1012 // Therefore, this is impossible (or at least unsafe) to share values
1013 // between basic blocks unless they use the same instruction selection
1014 // method, which is not guarantee for X86.
1015 // Moreover, things like hasOneUse could not be used accurately, if we
1016 // allow to reference values across basic blocks whereas they are not
1017 // alive across basic blocks initially.
1018 bool InMBB = true;
1019 if (I) {
1020 Opcode = I->getOpcode();
1021 U = I;
1022 InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock();
1023 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(Val: V)) {
1024 Opcode = C->getOpcode();
1025 U = C;
1026 }
1027
1028 switch (Opcode) {
1029 default: break;
1030 case Instruction::BitCast:
1031 // Look past bitcasts if its operand is in the same BB.
1032 if (InMBB)
1033 return X86SelectCallAddress(V: U->getOperand(i: 0), AM);
1034 break;
1035
1036 case Instruction::IntToPtr:
1037 // Look past no-op inttoptrs if its operand is in the same BB.
1038 if (InMBB &&
1039 TLI.getValueType(DL, Ty: U->getOperand(i: 0)->getType()) ==
1040 TLI.getPointerTy(DL))
1041 return X86SelectCallAddress(V: U->getOperand(i: 0), AM);
1042 break;
1043
1044 case Instruction::PtrToInt:
1045 // Look past no-op ptrtoints if its operand is in the same BB.
1046 if (InMBB && TLI.getValueType(DL, Ty: U->getType()) == TLI.getPointerTy(DL))
1047 return X86SelectCallAddress(V: U->getOperand(i: 0), AM);
1048 break;
1049 }
1050
1051 // Handle constant address.
1052 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
1053 // Can't handle alternate code models yet.
1054 if (TM.getCodeModel() != CodeModel::Small &&
1055 TM.getCodeModel() != CodeModel::Medium)
1056 return false;
1057
1058 // RIP-relative addresses can't have additional register operands.
1059 if (Subtarget->isPICStyleRIPRel() &&
1060 (AM.Base.Reg != 0 || AM.IndexReg != 0))
1061 return false;
1062
1063 // Can't handle TLS.
1064 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(Val: GV))
1065 if (GVar->isThreadLocal())
1066 return false;
1067
1068 // Okay, we've committed to selecting this global. Set up the basic address.
1069 AM.GV = GV;
1070
1071 // Return a direct reference to the global. Fastisel can handle calls to
1072 // functions that require loads, such as dllimport and nonlazybind
1073 // functions.
1074 if (Subtarget->isPICStyleRIPRel()) {
1075 // Use rip-relative addressing if we can. Above we verified that the
1076 // base and index registers are unused.
1077 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
1078 AM.Base.Reg = X86::RIP;
1079 } else {
1080 AM.GVOpFlags = Subtarget->classifyLocalReference(GV: nullptr);
1081 }
1082
1083 return true;
1084 }
1085
1086 // If all else fails, try to materialize the value in a register.
1087 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
1088 auto GetCallRegForValue = [this](const Value *V) {
1089 Register Reg = getRegForValue(V);
1090
1091 // In 64-bit mode, we need a 64-bit register even if pointers are 32 bits.
1092 if (Reg && Subtarget->isTarget64BitILP32()) {
1093 Register CopyReg = createResultReg(RC: &X86::GR32RegClass);
1094 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV32rr),
1095 DestReg: CopyReg)
1096 .addReg(RegNo: Reg);
1097
1098 Register ExtReg = createResultReg(RC: &X86::GR64RegClass);
1099 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1100 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: ExtReg)
1101 .addReg(RegNo: CopyReg)
1102 .addImm(Val: X86::sub_32bit);
1103 Reg = ExtReg;
1104 }
1105
1106 return Reg;
1107 };
1108
1109 if (AM.Base.Reg == 0) {
1110 AM.Base.Reg = GetCallRegForValue(V);
1111 return AM.Base.Reg != 0;
1112 }
1113 if (AM.IndexReg == 0) {
1114 assert(AM.Scale == 1 && "Scale with no index!");
1115 AM.IndexReg = GetCallRegForValue(V);
1116 return AM.IndexReg != 0;
1117 }
1118 }
1119
1120 return false;
1121}
1122
1123
1124/// X86SelectStore - Select and emit code to implement store instructions.
1125bool X86FastISel::X86SelectStore(const Instruction *I) {
1126 // Atomic stores need special handling.
1127 const StoreInst *S = cast<StoreInst>(Val: I);
1128
1129 if (S->isAtomic())
1130 return false;
1131
1132 const Value *PtrV = I->getOperand(i: 1);
1133 if (TLI.supportSwiftError()) {
1134 // Swifterror values can come from either a function parameter with
1135 // swifterror attribute or an alloca with swifterror attribute.
1136 if (const Argument *Arg = dyn_cast<Argument>(Val: PtrV)) {
1137 if (Arg->hasSwiftErrorAttr())
1138 return false;
1139 }
1140
1141 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: PtrV)) {
1142 if (Alloca->isSwiftError())
1143 return false;
1144 }
1145 }
1146
1147 const Value *Val = S->getValueOperand();
1148 const Value *Ptr = S->getPointerOperand();
1149
1150 MVT VT;
1151 if (!isTypeLegal(Ty: Val->getType(), VT, /*AllowI1=*/true))
1152 return false;
1153
1154 Align Alignment = S->getAlign();
1155 Align ABIAlignment = DL.getABITypeAlign(Ty: Val->getType());
1156 bool Aligned = Alignment >= ABIAlignment;
1157
1158 X86AddressMode AM;
1159 if (!X86SelectAddress(V: Ptr, AM))
1160 return false;
1161
1162 return X86FastEmitStore(VT, Val, AM, MMO: createMachineMemOperandFor(I), Aligned);
1163}
1164
1165/// X86SelectRet - Select and emit code to implement ret instructions.
1166bool X86FastISel::X86SelectRet(const Instruction *I) {
1167 const ReturnInst *Ret = cast<ReturnInst>(Val: I);
1168 const Function &F = *I->getParent()->getParent();
1169 const X86MachineFunctionInfo *X86MFInfo =
1170 FuncInfo.MF->getInfo<X86MachineFunctionInfo>();
1171
1172 if (!FuncInfo.CanLowerReturn)
1173 return false;
1174
1175 if (TLI.supportSwiftError() &&
1176 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
1177 return false;
1178
1179 if (TLI.supportSplitCSR(MF: FuncInfo.MF))
1180 return false;
1181
1182 CallingConv::ID CC = F.getCallingConv();
1183 if (CC != CallingConv::C &&
1184 CC != CallingConv::Fast &&
1185 CC != CallingConv::Tail &&
1186 CC != CallingConv::SwiftTail &&
1187 CC != CallingConv::X86_FastCall &&
1188 CC != CallingConv::X86_StdCall &&
1189 CC != CallingConv::X86_ThisCall &&
1190 CC != CallingConv::X86_64_SysV &&
1191 CC != CallingConv::Win64)
1192 return false;
1193
1194 // Don't handle popping bytes if they don't fit the ret's immediate.
1195 if (!isUInt<16>(x: X86MFInfo->getBytesToPopOnReturn()))
1196 return false;
1197
1198 // fastcc with -tailcallopt is intended to provide a guaranteed
1199 // tail call optimization. Fastisel doesn't know how to do that.
1200 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
1201 CC == CallingConv::Tail || CC == CallingConv::SwiftTail)
1202 return false;
1203
1204 // Let SDISel handle vararg functions.
1205 if (F.isVarArg())
1206 return false;
1207
1208 // Build a list of return value registers.
1209 SmallVector<Register, 4> RetRegs;
1210
1211 if (Ret->getNumOperands() > 0) {
1212 SmallVector<ISD::OutputArg, 4> Outs;
1213 GetReturnInfo(CC, ReturnType: F.getReturnType(), attr: F.getAttributes(), Outs, TLI, DL);
1214
1215 // Analyze operands of the call, assigning locations to each operand.
1216 SmallVector<CCValAssign, 16> ValLocs;
1217 CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext());
1218 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_X86);
1219
1220 const Value *RV = Ret->getOperand(i_nocapture: 0);
1221 Register Reg = getRegForValue(V: RV);
1222 if (!Reg)
1223 return false;
1224
1225 // Only handle a single return value for now.
1226 if (ValLocs.size() != 1)
1227 return false;
1228
1229 CCValAssign &VA = ValLocs[0];
1230
1231 // Don't bother handling odd stuff for now.
1232 if (VA.getLocInfo() != CCValAssign::Full)
1233 return false;
1234 // Only handle register returns for now.
1235 if (!VA.isRegLoc())
1236 return false;
1237
1238 // The calling-convention tables for x87 returns don't tell
1239 // the whole story.
1240 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
1241 return false;
1242
1243 Register SrcReg = Reg + VA.getValNo();
1244 EVT SrcVT = TLI.getValueType(DL, Ty: RV->getType());
1245 EVT DstVT = VA.getValVT();
1246 // Special handling for extended integers.
1247 if (SrcVT != DstVT) {
1248 if (SrcVT != MVT::i1 && SrcVT != MVT::i8 && SrcVT != MVT::i16)
1249 return false;
1250
1251 if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
1252 return false;
1253
1254 if (SrcVT == MVT::i1) {
1255 if (Outs[0].Flags.isSExt())
1256 return false;
1257 SrcReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: SrcReg);
1258 SrcVT = MVT::i8;
1259 }
1260 if (SrcVT != DstVT) {
1261 unsigned Op =
1262 Outs[0].Flags.isZExt() ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
1263 SrcReg =
1264 fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(), Opcode: Op, Op0: SrcReg);
1265 }
1266 }
1267
1268 // Make the copy.
1269 Register DstReg = VA.getLocReg();
1270 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
1271 // Avoid a cross-class copy. This is very unlikely.
1272 if (!SrcRC->contains(Reg: DstReg))
1273 return false;
1274 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1275 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: SrcReg);
1276
1277 // Add register to return instruction.
1278 RetRegs.push_back(Elt: VA.getLocReg());
1279 }
1280
1281 // Swift calling convention does not require we copy the sret argument
1282 // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
1283
1284 // All x86 ABIs require that for returning structs by value we copy
1285 // the sret argument into %rax/%eax (depending on ABI) for the return.
1286 // We saved the argument into a virtual register in the entry block,
1287 // so now we copy the value out and into %rax/%eax.
1288 if (F.hasStructRetAttr() && CC != CallingConv::Swift &&
1289 CC != CallingConv::SwiftTail) {
1290 Register Reg = X86MFInfo->getSRetReturnReg();
1291 assert(Reg &&
1292 "SRetReturnReg should have been set in LowerFormalArguments()!");
1293 Register RetReg = Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX;
1294 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1295 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: RetReg).addReg(RegNo: Reg);
1296 RetRegs.push_back(Elt: RetReg);
1297 }
1298
1299 // Now emit the RET.
1300 MachineInstrBuilder MIB;
1301 if (X86MFInfo->getBytesToPopOnReturn()) {
1302 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1303 MCID: TII.get(Opcode: Subtarget->is64Bit() ? X86::RETI64 : X86::RETI32))
1304 .addImm(Val: X86MFInfo->getBytesToPopOnReturn());
1305 } else {
1306 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1307 MCID: TII.get(Opcode: Subtarget->is64Bit() ? X86::RET64 : X86::RET32));
1308 }
1309 for (Register Reg : RetRegs)
1310 MIB.addReg(RegNo: Reg, Flags: RegState::Implicit);
1311 return true;
1312}
1313
1314/// X86SelectLoad - Select and emit code to implement load instructions.
1315///
1316bool X86FastISel::X86SelectLoad(const Instruction *I) {
1317 const LoadInst *LI = cast<LoadInst>(Val: I);
1318
1319 // Atomic loads need special handling.
1320 if (LI->isAtomic())
1321 return false;
1322
1323 const Value *SV = I->getOperand(i: 0);
1324 if (TLI.supportSwiftError()) {
1325 // Swifterror values can come from either a function parameter with
1326 // swifterror attribute or an alloca with swifterror attribute.
1327 if (const Argument *Arg = dyn_cast<Argument>(Val: SV)) {
1328 if (Arg->hasSwiftErrorAttr())
1329 return false;
1330 }
1331
1332 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: SV)) {
1333 if (Alloca->isSwiftError())
1334 return false;
1335 }
1336 }
1337
1338 MVT VT;
1339 if (!isTypeLegal(Ty: LI->getType(), VT, /*AllowI1=*/true))
1340 return false;
1341
1342 const Value *Ptr = LI->getPointerOperand();
1343
1344 X86AddressMode AM;
1345 if (!X86SelectAddress(V: Ptr, AM))
1346 return false;
1347
1348 Register ResultReg;
1349 if (!X86FastEmitLoad(VT, AM, MMO: createMachineMemOperandFor(I: LI), ResultReg,
1350 Alignment: LI->getAlign().value()))
1351 return false;
1352
1353 updateValueMap(I, Reg: ResultReg);
1354 return true;
1355}
1356
1357static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget) {
1358 bool HasAVX512 = Subtarget->hasAVX512();
1359 bool HasAVX = Subtarget->hasAVX();
1360 bool HasSSE1 = Subtarget->hasSSE1();
1361 bool HasSSE2 = Subtarget->hasSSE2();
1362
1363 switch (VT.getSimpleVT().SimpleTy) {
1364 default: return 0;
1365 case MVT::i8: return X86::CMP8rr;
1366 case MVT::i16: return X86::CMP16rr;
1367 case MVT::i32: return X86::CMP32rr;
1368 case MVT::i64: return X86::CMP64rr;
1369 case MVT::f32:
1370 return HasAVX512 ? X86::VUCOMISSZrr
1371 : HasAVX ? X86::VUCOMISSrr
1372 : HasSSE1 ? X86::UCOMISSrr
1373 : 0;
1374 case MVT::f64:
1375 return HasAVX512 ? X86::VUCOMISDZrr
1376 : HasAVX ? X86::VUCOMISDrr
1377 : HasSSE2 ? X86::UCOMISDrr
1378 : 0;
1379 }
1380}
1381
1382/// If we have a comparison with RHS as the RHS of the comparison, return an
1383/// opcode that works for the compare (e.g. CMP32ri) otherwise return 0.
1384static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC) {
1385 switch (VT.getSimpleVT().SimpleTy) {
1386 // Otherwise, we can't fold the immediate into this comparison.
1387 default:
1388 return 0;
1389 case MVT::i8:
1390 return X86::CMP8ri;
1391 case MVT::i16:
1392 return X86::CMP16ri;
1393 case MVT::i32:
1394 return X86::CMP32ri;
1395 case MVT::i64:
1396 // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext
1397 // field.
1398 return isInt<32>(x: RHSC->getSExtValue()) ? X86::CMP64ri32 : 0;
1399 }
1400}
1401
1402bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, EVT VT,
1403 const DebugLoc &CurMIMD) {
1404 Register Op0Reg = getRegForValue(V: Op0);
1405 if (!Op0Reg)
1406 return false;
1407
1408 // Handle 'null' like i32/i64 0.
1409 if (isa<ConstantPointerNull>(Val: Op1))
1410 Op1 = Constant::getNullValue(Ty: DL.getIntPtrType(C&: Op0->getContext()));
1411
1412 // We have two options: compare with register or immediate. If the RHS of
1413 // the compare is an immediate that we can fold into this compare, use
1414 // CMPri, otherwise use CMPrr.
1415 if (const ConstantInt *Op1C = dyn_cast<ConstantInt>(Val: Op1)) {
1416 if (unsigned CompareImmOpc = X86ChooseCmpImmediateOpcode(VT, RHSC: Op1C)) {
1417 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: CurMIMD, MCID: TII.get(Opcode: CompareImmOpc))
1418 .addReg(RegNo: Op0Reg)
1419 .addImm(Val: Op1C->getSExtValue());
1420 return true;
1421 }
1422 }
1423
1424 unsigned CompareOpc = X86ChooseCmpOpcode(VT, Subtarget);
1425 if (CompareOpc == 0) return false;
1426
1427 Register Op1Reg = getRegForValue(V: Op1);
1428 if (!Op1Reg)
1429 return false;
1430 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: CurMIMD, MCID: TII.get(Opcode: CompareOpc))
1431 .addReg(RegNo: Op0Reg)
1432 .addReg(RegNo: Op1Reg);
1433
1434 return true;
1435}
1436
1437#define GET_SETCC \
1438 ((!Subtarget->hasZU() || Subtarget->preferLegacySetCC()) ? X86::SETCCr \
1439 : X86::SETZUCCr)
1440
1441bool X86FastISel::X86SelectCmp(const Instruction *I) {
1442 const CmpInst *CI = cast<CmpInst>(Val: I);
1443
1444 MVT VT;
1445 if (!isTypeLegal(Ty: I->getOperand(i: 0)->getType(), VT))
1446 return false;
1447
1448 // Below code only works for scalars.
1449 if (VT.isVector())
1450 return false;
1451
1452 // Try to optimize or fold the cmp.
1453 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1454 Register ResultReg;
1455 switch (Predicate) {
1456 default: break;
1457 case CmpInst::FCMP_FALSE: {
1458 ResultReg = createResultReg(RC: &X86::GR32RegClass);
1459 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV32r0),
1460 DestReg: ResultReg);
1461 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: ResultReg, Idx: X86::sub_8bit);
1462 if (!ResultReg)
1463 return false;
1464 break;
1465 }
1466 case CmpInst::FCMP_TRUE: {
1467 ResultReg = createResultReg(RC: &X86::GR8RegClass);
1468 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV8ri),
1469 DestReg: ResultReg).addImm(Val: 1);
1470 break;
1471 }
1472 }
1473
1474 if (ResultReg) {
1475 updateValueMap(I, Reg: ResultReg);
1476 return true;
1477 }
1478
1479 const Value *LHS = CI->getOperand(i_nocapture: 0);
1480 const Value *RHS = CI->getOperand(i_nocapture: 1);
1481
1482 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
1483 // We don't have to materialize a zero constant for this case and can just use
1484 // %x again on the RHS.
1485 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1486 const auto *RHSC = dyn_cast<ConstantFP>(Val: RHS);
1487 if (RHSC && RHSC->isNullValue())
1488 RHS = LHS;
1489 }
1490
1491 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1492 static const uint16_t SETFOpcTable[2][3] = {
1493 { X86::COND_E, X86::COND_NP, X86::AND8rr },
1494 { X86::COND_NE, X86::COND_P, X86::OR8rr }
1495 };
1496 const uint16_t *SETFOpc = nullptr;
1497 switch (Predicate) {
1498 default: break;
1499 case CmpInst::FCMP_OEQ: SETFOpc = &SETFOpcTable[0][0]; break;
1500 case CmpInst::FCMP_UNE: SETFOpc = &SETFOpcTable[1][0]; break;
1501 }
1502
1503 ResultReg = createResultReg(RC: &X86::GR8RegClass);
1504 if (SETFOpc) {
1505 if (!X86FastEmitCompare(Op0: LHS, Op1: RHS, VT, CurMIMD: I->getDebugLoc()))
1506 return false;
1507
1508 Register FlagReg1 = createResultReg(RC: &X86::GR8RegClass);
1509 Register FlagReg2 = createResultReg(RC: &X86::GR8RegClass);
1510 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
1511 DestReg: FlagReg1)
1512 .addImm(Val: SETFOpc[0]);
1513 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
1514 DestReg: FlagReg2)
1515 .addImm(Val: SETFOpc[1]);
1516 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: SETFOpc[2]),
1517 DestReg: ResultReg).addReg(RegNo: FlagReg1).addReg(RegNo: FlagReg2);
1518 updateValueMap(I, Reg: ResultReg);
1519 return true;
1520 }
1521
1522 X86::CondCode CC;
1523 bool SwapArgs;
1524 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1525 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1526
1527 if (SwapArgs)
1528 std::swap(a&: LHS, b&: RHS);
1529
1530 // Emit a compare of LHS/RHS.
1531 if (!X86FastEmitCompare(Op0: LHS, Op1: RHS, VT, CurMIMD: I->getDebugLoc()))
1532 return false;
1533
1534 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC), DestReg: ResultReg)
1535 .addImm(Val: CC);
1536 updateValueMap(I, Reg: ResultReg);
1537 return true;
1538}
1539
1540bool X86FastISel::X86SelectZExt(const Instruction *I) {
1541 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1542 if (!TLI.isTypeLegal(VT: DstVT))
1543 return false;
1544
1545 Register ResultReg = getRegForValue(V: I->getOperand(i: 0));
1546 if (!ResultReg)
1547 return false;
1548
1549 // Handle zero-extension from i1 to i8, which is common.
1550 MVT SrcVT = TLI.getSimpleValueType(DL, Ty: I->getOperand(i: 0)->getType());
1551 if (SrcVT == MVT::i1) {
1552 // Set the high bits to zero.
1553 ResultReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: ResultReg);
1554 SrcVT = MVT::i8;
1555
1556 if (!ResultReg)
1557 return false;
1558 }
1559
1560 if (DstVT == MVT::i64) {
1561 // Handle extension to 64-bits via sub-register shenanigans.
1562 unsigned MovInst;
1563
1564 switch (SrcVT.SimpleTy) {
1565 case MVT::i8: MovInst = X86::MOVZX32rr8; break;
1566 case MVT::i16: MovInst = X86::MOVZX32rr16; break;
1567 case MVT::i32: MovInst = X86::MOV32rr; break;
1568 default: llvm_unreachable("Unexpected zext to i64 source type");
1569 }
1570
1571 Register Result32 = createResultReg(RC: &X86::GR32RegClass);
1572 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: MovInst), DestReg: Result32)
1573 .addReg(RegNo: ResultReg);
1574
1575 ResultReg = createResultReg(RC: &X86::GR64RegClass);
1576 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1577 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: ResultReg)
1578 .addReg(RegNo: Result32)
1579 .addImm(Val: X86::sub_32bit);
1580 } else if (DstVT == MVT::i16) {
1581 // i8->i16 doesn't exist in the autogenerated isel table. Need to zero
1582 // extend to 32-bits and then extract down to 16-bits.
1583 Register Result32 = createResultReg(RC: &X86::GR32RegClass);
1584 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOVZX32rr8),
1585 DestReg: Result32).addReg(RegNo: ResultReg);
1586
1587 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0: Result32, Idx: X86::sub_16bit);
1588 } else if (DstVT != MVT::i8) {
1589 ResultReg = fastEmit_r(VT: MVT::i8, RetVT: DstVT.getSimpleVT(), Opcode: ISD::ZERO_EXTEND,
1590 Op0: ResultReg);
1591 if (!ResultReg)
1592 return false;
1593 }
1594
1595 updateValueMap(I, Reg: ResultReg);
1596 return true;
1597}
1598
1599bool X86FastISel::X86SelectSExt(const Instruction *I) {
1600 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1601 if (!TLI.isTypeLegal(VT: DstVT))
1602 return false;
1603
1604 Register ResultReg = getRegForValue(V: I->getOperand(i: 0));
1605 if (!ResultReg)
1606 return false;
1607
1608 // Handle sign-extension from i1 to i8.
1609 MVT SrcVT = TLI.getSimpleValueType(DL, Ty: I->getOperand(i: 0)->getType());
1610 if (SrcVT == MVT::i1) {
1611 // Set the high bits to zero.
1612 Register ZExtReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: ResultReg);
1613 if (!ZExtReg)
1614 return false;
1615
1616 // Negate the result to make an 8-bit sign extended value.
1617 ResultReg = createResultReg(RC: &X86::GR8RegClass);
1618 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::NEG8r),
1619 DestReg: ResultReg).addReg(RegNo: ZExtReg);
1620
1621 SrcVT = MVT::i8;
1622 }
1623
1624 if (DstVT == MVT::i16) {
1625 // i8->i16 doesn't exist in the autogenerated isel table. Need to sign
1626 // extend to 32-bits and then extract down to 16-bits.
1627 Register Result32 = createResultReg(RC: &X86::GR32RegClass);
1628 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOVSX32rr8),
1629 DestReg: Result32).addReg(RegNo: ResultReg);
1630
1631 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0: Result32, Idx: X86::sub_16bit);
1632 } else if (DstVT != MVT::i8) {
1633 ResultReg = fastEmit_r(VT: MVT::i8, RetVT: DstVT.getSimpleVT(), Opcode: ISD::SIGN_EXTEND,
1634 Op0: ResultReg);
1635 if (!ResultReg)
1636 return false;
1637 }
1638
1639 updateValueMap(I, Reg: ResultReg);
1640 return true;
1641}
1642
1643bool X86FastISel::X86SelectBranch(const Instruction *I) {
1644 // Unconditional branches are selected by tablegen-generated code.
1645 // Handle a conditional branch.
1646 const BranchInst *BI = cast<BranchInst>(Val: I);
1647 MachineBasicBlock *TrueMBB = FuncInfo.getMBB(BB: BI->getSuccessor(i: 0));
1648 MachineBasicBlock *FalseMBB = FuncInfo.getMBB(BB: BI->getSuccessor(i: 1));
1649
1650 // Fold the common case of a conditional branch with a comparison
1651 // in the same block (values defined on other blocks may not have
1652 // initialized registers).
1653 X86::CondCode CC;
1654 if (const CmpInst *CI = dyn_cast<CmpInst>(Val: BI->getCondition())) {
1655 if (CI->hasOneUse() && CI->getParent() == I->getParent()) {
1656 EVT VT = TLI.getValueType(DL, Ty: CI->getOperand(i_nocapture: 0)->getType());
1657
1658 // Try to optimize or fold the cmp.
1659 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1660 switch (Predicate) {
1661 default: break;
1662 case CmpInst::FCMP_FALSE: fastEmitBranch(MSucc: FalseMBB, DbgLoc: MIMD.getDL()); return true;
1663 case CmpInst::FCMP_TRUE: fastEmitBranch(MSucc: TrueMBB, DbgLoc: MIMD.getDL()); return true;
1664 }
1665
1666 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
1667 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
1668
1669 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x,
1670 // 0.0.
1671 // We don't have to materialize a zero constant for this case and can just
1672 // use %x again on the RHS.
1673 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1674 const auto *CmpRHSC = dyn_cast<ConstantFP>(Val: CmpRHS);
1675 if (CmpRHSC && CmpRHSC->isNullValue())
1676 CmpRHS = CmpLHS;
1677 }
1678
1679 // Try to take advantage of fallthrough opportunities.
1680 if (FuncInfo.MBB->isLayoutSuccessor(MBB: TrueMBB)) {
1681 std::swap(a&: TrueMBB, b&: FalseMBB);
1682 Predicate = CmpInst::getInversePredicate(pred: Predicate);
1683 }
1684
1685 // FCMP_OEQ and FCMP_UNE cannot be expressed with a single flag/condition
1686 // code check. Instead two branch instructions are required to check all
1687 // the flags. First we change the predicate to a supported condition code,
1688 // which will be the first branch. Later one we will emit the second
1689 // branch.
1690 bool NeedExtraBranch = false;
1691 switch (Predicate) {
1692 default: break;
1693 case CmpInst::FCMP_OEQ:
1694 std::swap(a&: TrueMBB, b&: FalseMBB);
1695 [[fallthrough]];
1696 case CmpInst::FCMP_UNE:
1697 NeedExtraBranch = true;
1698 Predicate = CmpInst::FCMP_ONE;
1699 break;
1700 }
1701
1702 bool SwapArgs;
1703 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1704 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1705
1706 if (SwapArgs)
1707 std::swap(a&: CmpLHS, b&: CmpRHS);
1708
1709 // Emit a compare of the LHS and RHS, setting the flags.
1710 if (!X86FastEmitCompare(Op0: CmpLHS, Op1: CmpRHS, VT, CurMIMD: CI->getDebugLoc()))
1711 return false;
1712
1713 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1714 .addMBB(MBB: TrueMBB).addImm(Val: CC);
1715
1716 // X86 requires a second branch to handle UNE (and OEQ, which is mapped
1717 // to UNE above).
1718 if (NeedExtraBranch) {
1719 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1720 .addMBB(MBB: TrueMBB).addImm(Val: X86::COND_P);
1721 }
1722
1723 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1724 return true;
1725 }
1726 } else if (TruncInst *TI = dyn_cast<TruncInst>(Val: BI->getCondition())) {
1727 // Handle things like "%cond = trunc i32 %X to i1 / br i1 %cond", which
1728 // typically happen for _Bool and C++ bools.
1729 MVT SourceVT;
1730 if (TI->hasOneUse() && TI->getParent() == I->getParent() &&
1731 isTypeLegal(Ty: TI->getOperand(i_nocapture: 0)->getType(), VT&: SourceVT)) {
1732 unsigned TestOpc = 0;
1733 switch (SourceVT.SimpleTy) {
1734 default: break;
1735 case MVT::i8: TestOpc = X86::TEST8ri; break;
1736 case MVT::i16: TestOpc = X86::TEST16ri; break;
1737 case MVT::i32: TestOpc = X86::TEST32ri; break;
1738 case MVT::i64: TestOpc = X86::TEST64ri32; break;
1739 }
1740 if (TestOpc) {
1741 Register OpReg = getRegForValue(V: TI->getOperand(i_nocapture: 0));
1742 if (!OpReg)
1743 return false;
1744
1745 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TestOpc))
1746 .addReg(RegNo: OpReg).addImm(Val: 1);
1747
1748 unsigned JmpCond = X86::COND_NE;
1749 if (FuncInfo.MBB->isLayoutSuccessor(MBB: TrueMBB)) {
1750 std::swap(a&: TrueMBB, b&: FalseMBB);
1751 JmpCond = X86::COND_E;
1752 }
1753
1754 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1755 .addMBB(MBB: TrueMBB).addImm(Val: JmpCond);
1756
1757 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1758 return true;
1759 }
1760 }
1761 } else if (foldX86XALUIntrinsic(CC, I: BI, Cond: BI->getCondition())) {
1762 // Fake request the condition, otherwise the intrinsic might be completely
1763 // optimized away.
1764 Register TmpReg = getRegForValue(V: BI->getCondition());
1765 if (!TmpReg)
1766 return false;
1767
1768 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1769 .addMBB(MBB: TrueMBB).addImm(Val: CC);
1770 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1771 return true;
1772 }
1773
1774 // Otherwise do a clumsy setcc and re-test it.
1775 // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used
1776 // in an explicit cast, so make sure to handle that correctly.
1777 Register OpReg = getRegForValue(V: BI->getCondition());
1778 if (!OpReg)
1779 return false;
1780
1781 // In case OpReg is a K register, COPY to a GPR
1782 if (MRI.getRegClass(Reg: OpReg) == &X86::VK1RegClass) {
1783 Register KOpReg = OpReg;
1784 OpReg = createResultReg(RC: &X86::GR32RegClass);
1785 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1786 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: OpReg)
1787 .addReg(RegNo: KOpReg);
1788 OpReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: OpReg, Idx: X86::sub_8bit);
1789 }
1790 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TEST8ri))
1791 .addReg(RegNo: OpReg)
1792 .addImm(Val: 1);
1793 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1794 .addMBB(MBB: TrueMBB).addImm(Val: X86::COND_NE);
1795 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1796 return true;
1797}
1798
1799bool X86FastISel::X86SelectShift(const Instruction *I) {
1800 Register CReg;
1801 unsigned OpReg;
1802 const TargetRegisterClass *RC = nullptr;
1803 if (I->getType()->isIntegerTy(Bitwidth: 8)) {
1804 CReg = X86::CL;
1805 RC = &X86::GR8RegClass;
1806 switch (I->getOpcode()) {
1807 case Instruction::LShr: OpReg = X86::SHR8rCL; break;
1808 case Instruction::AShr: OpReg = X86::SAR8rCL; break;
1809 case Instruction::Shl: OpReg = X86::SHL8rCL; break;
1810 default: return false;
1811 }
1812 } else if (I->getType()->isIntegerTy(Bitwidth: 16)) {
1813 CReg = X86::CX;
1814 RC = &X86::GR16RegClass;
1815 switch (I->getOpcode()) {
1816 default: llvm_unreachable("Unexpected shift opcode");
1817 case Instruction::LShr: OpReg = X86::SHR16rCL; break;
1818 case Instruction::AShr: OpReg = X86::SAR16rCL; break;
1819 case Instruction::Shl: OpReg = X86::SHL16rCL; break;
1820 }
1821 } else if (I->getType()->isIntegerTy(Bitwidth: 32)) {
1822 CReg = X86::ECX;
1823 RC = &X86::GR32RegClass;
1824 switch (I->getOpcode()) {
1825 default: llvm_unreachable("Unexpected shift opcode");
1826 case Instruction::LShr: OpReg = X86::SHR32rCL; break;
1827 case Instruction::AShr: OpReg = X86::SAR32rCL; break;
1828 case Instruction::Shl: OpReg = X86::SHL32rCL; break;
1829 }
1830 } else if (I->getType()->isIntegerTy(Bitwidth: 64)) {
1831 CReg = X86::RCX;
1832 RC = &X86::GR64RegClass;
1833 switch (I->getOpcode()) {
1834 default: llvm_unreachable("Unexpected shift opcode");
1835 case Instruction::LShr: OpReg = X86::SHR64rCL; break;
1836 case Instruction::AShr: OpReg = X86::SAR64rCL; break;
1837 case Instruction::Shl: OpReg = X86::SHL64rCL; break;
1838 }
1839 } else {
1840 return false;
1841 }
1842
1843 MVT VT;
1844 if (!isTypeLegal(Ty: I->getType(), VT))
1845 return false;
1846
1847 Register Op0Reg = getRegForValue(V: I->getOperand(i: 0));
1848 if (!Op0Reg)
1849 return false;
1850
1851 Register Op1Reg = getRegForValue(V: I->getOperand(i: 1));
1852 if (!Op1Reg)
1853 return false;
1854 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
1855 DestReg: CReg).addReg(RegNo: Op1Reg);
1856
1857 // The shift instruction uses X86::CL. If we defined a super-register
1858 // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.
1859 if (CReg != X86::CL)
1860 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1861 MCID: TII.get(Opcode: TargetOpcode::KILL), DestReg: X86::CL)
1862 .addReg(RegNo: CReg, Flags: RegState::Kill);
1863
1864 Register ResultReg = createResultReg(RC);
1865 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: OpReg), DestReg: ResultReg)
1866 .addReg(RegNo: Op0Reg);
1867 updateValueMap(I, Reg: ResultReg);
1868 return true;
1869}
1870
1871bool X86FastISel::X86SelectDivRem(const Instruction *I) {
1872 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1873 const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem
1874 const static bool S = true; // IsSigned
1875 const static bool U = false; // !IsSigned
1876 const static unsigned Copy = TargetOpcode::COPY;
1877 // For the X86 DIV/IDIV instruction, in most cases the dividend
1878 // (numerator) must be in a specific register pair highreg:lowreg,
1879 // producing the quotient in lowreg and the remainder in highreg.
1880 // For most data types, to set up the instruction, the dividend is
1881 // copied into lowreg, and lowreg is sign-extended or zero-extended
1882 // into highreg. The exception is i8, where the dividend is defined
1883 // as a single register rather than a register pair, and we
1884 // therefore directly sign-extend or zero-extend the dividend into
1885 // lowreg, instead of copying, and ignore the highreg.
1886 const static struct DivRemEntry {
1887 // The following portion depends only on the data type.
1888 const TargetRegisterClass *RC;
1889 unsigned LowInReg; // low part of the register pair
1890 unsigned HighInReg; // high part of the register pair
1891 // The following portion depends on both the data type and the operation.
1892 struct DivRemResult {
1893 unsigned OpDivRem; // The specific DIV/IDIV opcode to use.
1894 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1895 // highreg, or copying a zero into highreg.
1896 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1897 // zero/sign-extending into lowreg for i8.
1898 unsigned DivRemResultReg; // Register containing the desired result.
1899 bool IsOpSigned; // Whether to use signed or unsigned form.
1900 } ResultTable[NumOps];
1901 } OpTable[NumTypes] = {
1902 { .RC: &X86::GR8RegClass, .LowInReg: X86::AX, .HighInReg: 0, .ResultTable: {
1903 { .OpDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .DivRemResultReg: X86::AL, .IsOpSigned: S }, // SDiv
1904 { .OpDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .DivRemResultReg: X86::AH, .IsOpSigned: S }, // SRem
1905 { .OpDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .DivRemResultReg: X86::AL, .IsOpSigned: U }, // UDiv
1906 { .OpDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .DivRemResultReg: X86::AH, .IsOpSigned: U }, // URem
1907 }
1908 }, // i8
1909 { .RC: &X86::GR16RegClass, .LowInReg: X86::AX, .HighInReg: X86::DX, .ResultTable: {
1910 { .OpDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .DivRemResultReg: X86::AX, .IsOpSigned: S }, // SDiv
1911 { .OpDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .DivRemResultReg: X86::DX, .IsOpSigned: S }, // SRem
1912 { .OpDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::AX, .IsOpSigned: U }, // UDiv
1913 { .OpDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::DX, .IsOpSigned: U }, // URem
1914 }
1915 }, // i16
1916 { .RC: &X86::GR32RegClass, .LowInReg: X86::EAX, .HighInReg: X86::EDX, .ResultTable: {
1917 { .OpDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .DivRemResultReg: X86::EAX, .IsOpSigned: S }, // SDiv
1918 { .OpDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .DivRemResultReg: X86::EDX, .IsOpSigned: S }, // SRem
1919 { .OpDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::EAX, .IsOpSigned: U }, // UDiv
1920 { .OpDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::EDX, .IsOpSigned: U }, // URem
1921 }
1922 }, // i32
1923 { .RC: &X86::GR64RegClass, .LowInReg: X86::RAX, .HighInReg: X86::RDX, .ResultTable: {
1924 { .OpDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .DivRemResultReg: X86::RAX, .IsOpSigned: S }, // SDiv
1925 { .OpDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .DivRemResultReg: X86::RDX, .IsOpSigned: S }, // SRem
1926 { .OpDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::RAX, .IsOpSigned: U }, // UDiv
1927 { .OpDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::RDX, .IsOpSigned: U }, // URem
1928 }
1929 }, // i64
1930 };
1931
1932 MVT VT;
1933 if (!isTypeLegal(Ty: I->getType(), VT))
1934 return false;
1935
1936 unsigned TypeIndex, OpIndex;
1937 switch (VT.SimpleTy) {
1938 default: return false;
1939 case MVT::i8: TypeIndex = 0; break;
1940 case MVT::i16: TypeIndex = 1; break;
1941 case MVT::i32: TypeIndex = 2; break;
1942 case MVT::i64: TypeIndex = 3;
1943 if (!Subtarget->is64Bit())
1944 return false;
1945 break;
1946 }
1947
1948 switch (I->getOpcode()) {
1949 default: llvm_unreachable("Unexpected div/rem opcode");
1950 case Instruction::SDiv: OpIndex = 0; break;
1951 case Instruction::SRem: OpIndex = 1; break;
1952 case Instruction::UDiv: OpIndex = 2; break;
1953 case Instruction::URem: OpIndex = 3; break;
1954 }
1955
1956 const DivRemEntry &TypeEntry = OpTable[TypeIndex];
1957 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1958 Register Op0Reg = getRegForValue(V: I->getOperand(i: 0));
1959 if (!Op0Reg)
1960 return false;
1961 Register Op1Reg = getRegForValue(V: I->getOperand(i: 1));
1962 if (!Op1Reg)
1963 return false;
1964
1965 // Move op0 into low-order input register.
1966 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1967 MCID: TII.get(Opcode: OpEntry.OpCopy), DestReg: TypeEntry.LowInReg).addReg(RegNo: Op0Reg);
1968 // Zero-extend or sign-extend into high-order input register.
1969 if (OpEntry.OpSignExtend) {
1970 if (OpEntry.IsOpSigned)
1971 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1972 MCID: TII.get(Opcode: OpEntry.OpSignExtend));
1973 else {
1974 Register Zero32 = createResultReg(RC: &X86::GR32RegClass);
1975 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1976 MCID: TII.get(Opcode: X86::MOV32r0), DestReg: Zero32);
1977
1978 // Copy the zero into the appropriate sub/super/identical physical
1979 // register. Unfortunately the operations needed are not uniform enough
1980 // to fit neatly into the table above.
1981 if (VT == MVT::i16) {
1982 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Copy),
1983 DestReg: TypeEntry.HighInReg)
1984 .addReg(RegNo: Zero32, Flags: {}, SubReg: X86::sub_16bit);
1985 } else if (VT == MVT::i32) {
1986 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1987 MCID: TII.get(Opcode: Copy), DestReg: TypeEntry.HighInReg)
1988 .addReg(RegNo: Zero32);
1989 } else if (VT == MVT::i64) {
1990 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1991 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: TypeEntry.HighInReg)
1992 .addReg(RegNo: Zero32)
1993 .addImm(Val: X86::sub_32bit);
1994 }
1995 }
1996 }
1997 // Generate the DIV/IDIV instruction.
1998 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1999 MCID: TII.get(Opcode: OpEntry.OpDivRem)).addReg(RegNo: Op1Reg);
2000 // For i8 remainder, we can't reference ah directly, as we'll end
2001 // up with bogus copies like %r9b = COPY %ah. Reference ax
2002 // instead to prevent ah references in a rex instruction.
2003 //
2004 // The current assumption of the fast register allocator is that isel
2005 // won't generate explicit references to the GR8_NOREX registers. If
2006 // the allocator and/or the backend get enhanced to be more robust in
2007 // that regard, this can be, and should be, removed.
2008 Register ResultReg;
2009 if ((I->getOpcode() == Instruction::SRem ||
2010 I->getOpcode() == Instruction::URem) &&
2011 OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
2012 Register SourceSuperReg = createResultReg(RC: &X86::GR16RegClass);
2013 Register ResultSuperReg = createResultReg(RC: &X86::GR16RegClass);
2014 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2015 MCID: TII.get(Opcode: Copy), DestReg: SourceSuperReg).addReg(RegNo: X86::AX);
2016
2017 // Shift AX right by 8 bits instead of using AH.
2018 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::SHR16ri),
2019 DestReg: ResultSuperReg).addReg(RegNo: SourceSuperReg).addImm(Val: 8);
2020
2021 // Now reference the 8-bit subreg of the result.
2022 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: ResultSuperReg,
2023 Idx: X86::sub_8bit);
2024 }
2025 // Copy the result out of the physreg if we haven't already.
2026 if (!ResultReg) {
2027 ResultReg = createResultReg(RC: TypeEntry.RC);
2028 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Copy), DestReg: ResultReg)
2029 .addReg(RegNo: OpEntry.DivRemResultReg);
2030 }
2031 updateValueMap(I, Reg: ResultReg);
2032
2033 return true;
2034}
2035
2036/// Emit a conditional move instruction (if the are supported) to lower
2037/// the select.
2038bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
2039 // Check if the subtarget supports these instructions.
2040 if (!Subtarget->canUseCMOV())
2041 return false;
2042
2043 // FIXME: Add support for i8.
2044 if (RetVT < MVT::i16 || RetVT > MVT::i64)
2045 return false;
2046
2047 const Value *Cond = I->getOperand(i: 0);
2048 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2049 bool NeedTest = true;
2050 X86::CondCode CC = X86::COND_NE;
2051
2052 // Optimize conditions coming from a compare if both instructions are in the
2053 // same basic block (values defined in other basic blocks may not have
2054 // initialized registers).
2055 const auto *CI = dyn_cast<CmpInst>(Val: Cond);
2056 if (CI && (CI->getParent() == I->getParent())) {
2057 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2058
2059 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
2060 static const uint16_t SETFOpcTable[2][3] = {
2061 { X86::COND_NP, X86::COND_E, X86::TEST8rr },
2062 { X86::COND_P, X86::COND_NE, X86::OR8rr }
2063 };
2064 const uint16_t *SETFOpc = nullptr;
2065 switch (Predicate) {
2066 default: break;
2067 case CmpInst::FCMP_OEQ:
2068 SETFOpc = &SETFOpcTable[0][0];
2069 Predicate = CmpInst::ICMP_NE;
2070 break;
2071 case CmpInst::FCMP_UNE:
2072 SETFOpc = &SETFOpcTable[1][0];
2073 Predicate = CmpInst::ICMP_NE;
2074 break;
2075 }
2076
2077 bool NeedSwap;
2078 std::tie(args&: CC, args&: NeedSwap) = X86::getX86ConditionCode(Predicate);
2079 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
2080
2081 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
2082 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
2083 if (NeedSwap)
2084 std::swap(a&: CmpLHS, b&: CmpRHS);
2085
2086 EVT CmpVT = TLI.getValueType(DL, Ty: CmpLHS->getType());
2087 // Emit a compare of the LHS and RHS, setting the flags.
2088 if (!X86FastEmitCompare(Op0: CmpLHS, Op1: CmpRHS, VT: CmpVT, CurMIMD: CI->getDebugLoc()))
2089 return false;
2090
2091 if (SETFOpc) {
2092 Register FlagReg1 = createResultReg(RC: &X86::GR8RegClass);
2093 Register FlagReg2 = createResultReg(RC: &X86::GR8RegClass);
2094 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
2095 DestReg: FlagReg1)
2096 .addImm(Val: SETFOpc[0]);
2097 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
2098 DestReg: FlagReg2)
2099 .addImm(Val: SETFOpc[1]);
2100 auto const &II = TII.get(Opcode: SETFOpc[2]);
2101 if (II.getNumDefs()) {
2102 Register TmpReg = createResultReg(RC: &X86::GR8RegClass);
2103 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: TmpReg)
2104 .addReg(RegNo: FlagReg2).addReg(RegNo: FlagReg1);
2105 } else {
2106 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2107 .addReg(RegNo: FlagReg2).addReg(RegNo: FlagReg1);
2108 }
2109 }
2110 NeedTest = false;
2111 } else if (foldX86XALUIntrinsic(CC, I, Cond)) {
2112 // Fake request the condition, otherwise the intrinsic might be completely
2113 // optimized away.
2114 Register TmpReg = getRegForValue(V: Cond);
2115 if (!TmpReg)
2116 return false;
2117
2118 NeedTest = false;
2119 }
2120
2121 if (NeedTest) {
2122 // Selects operate on i1, however, CondReg is 8 bits width and may contain
2123 // garbage. Indeed, only the less significant bit is supposed to be
2124 // accurate. If we read more than the lsb, we may see non-zero values
2125 // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for
2126 // the select. This is achieved by performing TEST against 1.
2127 Register CondReg = getRegForValue(V: Cond);
2128 if (!CondReg)
2129 return false;
2130
2131 // In case OpReg is a K register, COPY to a GPR
2132 if (MRI.getRegClass(Reg: CondReg) == &X86::VK1RegClass) {
2133 Register KCondReg = CondReg;
2134 CondReg = createResultReg(RC: &X86::GR32RegClass);
2135 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2136 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CondReg)
2137 .addReg(RegNo: KCondReg);
2138 CondReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: CondReg, Idx: X86::sub_8bit);
2139 }
2140 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TEST8ri))
2141 .addReg(RegNo: CondReg)
2142 .addImm(Val: 1);
2143 }
2144
2145 const Value *LHS = I->getOperand(i: 1);
2146 const Value *RHS = I->getOperand(i: 2);
2147
2148 Register RHSReg = getRegForValue(V: RHS);
2149 Register LHSReg = getRegForValue(V: LHS);
2150 if (!LHSReg || !RHSReg)
2151 return false;
2152
2153 const TargetRegisterInfo &TRI = *Subtarget->getRegisterInfo();
2154 unsigned Opc = X86::getCMovOpcode(RegBytes: TRI.getRegSizeInBits(RC: *RC) / 8, HasMemoryOperand: false,
2155 HasNDD: Subtarget->hasNDD());
2156 Register ResultReg = fastEmitInst_rri(MachineInstOpcode: Opc, RC, Op0: RHSReg, Op1: LHSReg, Imm: CC);
2157 updateValueMap(I, Reg: ResultReg);
2158 return true;
2159}
2160
2161/// Emit SSE or AVX instructions to lower the select.
2162///
2163/// Try to use SSE1/SSE2 instructions to simulate a select without branches.
2164/// This lowers fp selects into a CMP/AND/ANDN/OR sequence when the necessary
2165/// SSE instructions are available. If AVX is available, try to use a VBLENDV.
2166bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
2167 // Optimize conditions coming from a compare if both instructions are in the
2168 // same basic block (values defined in other basic blocks may not have
2169 // initialized registers).
2170 const auto *CI = dyn_cast<FCmpInst>(Val: I->getOperand(i: 0));
2171 if (!CI || (CI->getParent() != I->getParent()))
2172 return false;
2173
2174 if (I->getType() != CI->getOperand(i_nocapture: 0)->getType() ||
2175 !((Subtarget->hasSSE1() && RetVT == MVT::f32) ||
2176 (Subtarget->hasSSE2() && RetVT == MVT::f64)))
2177 return false;
2178
2179 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
2180 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
2181 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2182
2183 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
2184 // We don't have to materialize a zero constant for this case and can just use
2185 // %x again on the RHS.
2186 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
2187 const auto *CmpRHSC = dyn_cast<ConstantFP>(Val: CmpRHS);
2188 if (CmpRHSC && CmpRHSC->isNullValue())
2189 CmpRHS = CmpLHS;
2190 }
2191
2192 unsigned CC;
2193 bool NeedSwap;
2194 std::tie(args&: CC, args&: NeedSwap) = getX86SSEConditionCode(Predicate);
2195 if (CC > 7 && !Subtarget->hasAVX())
2196 return false;
2197
2198 if (NeedSwap)
2199 std::swap(a&: CmpLHS, b&: CmpRHS);
2200
2201 const Value *LHS = I->getOperand(i: 1);
2202 const Value *RHS = I->getOperand(i: 2);
2203
2204 Register LHSReg = getRegForValue(V: LHS);
2205 Register RHSReg = getRegForValue(V: RHS);
2206 Register CmpLHSReg = getRegForValue(V: CmpLHS);
2207 Register CmpRHSReg = getRegForValue(V: CmpRHS);
2208 if (!LHSReg || !RHSReg || !CmpLHSReg || !CmpRHSReg)
2209 return false;
2210
2211 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2212 Register ResultReg;
2213
2214 if (Subtarget->hasAVX512()) {
2215 // If we have AVX512 we can use a mask compare and masked movss/sd.
2216 const TargetRegisterClass *VR128X = &X86::VR128XRegClass;
2217 const TargetRegisterClass *VK1 = &X86::VK1RegClass;
2218
2219 unsigned CmpOpcode =
2220 (RetVT == MVT::f32) ? X86::VCMPSSZrri : X86::VCMPSDZrri;
2221 Register CmpReg = fastEmitInst_rri(MachineInstOpcode: CmpOpcode, RC: VK1, Op0: CmpLHSReg, Op1: CmpRHSReg,
2222 Imm: CC);
2223
2224 // Need an IMPLICIT_DEF for the input that is used to generate the upper
2225 // bits of the result register since its not based on any of the inputs.
2226 Register ImplicitDefReg = createResultReg(RC: VR128X);
2227 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2228 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2229
2230 // Place RHSReg is the passthru of the masked movss/sd operation and put
2231 // LHS in the input. The mask input comes from the compare.
2232 unsigned MovOpcode =
2233 (RetVT == MVT::f32) ? X86::VMOVSSZrrk : X86::VMOVSDZrrk;
2234 Register MovReg = fastEmitInst_rrrr(MachineInstOpcode: MovOpcode, RC: VR128X, Op0: RHSReg, Op1: CmpReg,
2235 Op2: ImplicitDefReg, Op3: LHSReg);
2236
2237 ResultReg = createResultReg(RC);
2238 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2239 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: MovReg);
2240
2241 } else if (Subtarget->hasAVX()) {
2242 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2243
2244 // If we have AVX, create 1 blendv instead of 3 logic instructions.
2245 // Blendv was introduced with SSE 4.1, but the 2 register form implicitly
2246 // uses XMM0 as the selection register. That may need just as many
2247 // instructions as the AND/ANDN/OR sequence due to register moves, so
2248 // don't bother.
2249 unsigned CmpOpcode =
2250 (RetVT == MVT::f32) ? X86::VCMPSSrri : X86::VCMPSDrri;
2251 unsigned BlendOpcode =
2252 (RetVT == MVT::f32) ? X86::VBLENDVPSrrr : X86::VBLENDVPDrrr;
2253
2254 Register CmpReg = fastEmitInst_rri(MachineInstOpcode: CmpOpcode, RC, Op0: CmpLHSReg, Op1: CmpRHSReg,
2255 Imm: CC);
2256 Register VBlendReg = fastEmitInst_rrr(MachineInstOpcode: BlendOpcode, RC: VR128, Op0: RHSReg, Op1: LHSReg,
2257 Op2: CmpReg);
2258 ResultReg = createResultReg(RC);
2259 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2260 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: VBlendReg);
2261 } else {
2262 // Choose the SSE instruction sequence based on data type (float or double).
2263 static const uint16_t OpcTable[2][4] = {
2264 { X86::CMPSSrri, X86::ANDPSrr, X86::ANDNPSrr, X86::ORPSrr },
2265 { X86::CMPSDrri, X86::ANDPDrr, X86::ANDNPDrr, X86::ORPDrr }
2266 };
2267
2268 const uint16_t *Opc = nullptr;
2269 switch (RetVT.SimpleTy) {
2270 default: return false;
2271 case MVT::f32: Opc = &OpcTable[0][0]; break;
2272 case MVT::f64: Opc = &OpcTable[1][0]; break;
2273 }
2274
2275 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2276 Register CmpReg = fastEmitInst_rri(MachineInstOpcode: Opc[0], RC, Op0: CmpLHSReg, Op1: CmpRHSReg, Imm: CC);
2277 Register AndReg = fastEmitInst_rr(MachineInstOpcode: Opc[1], RC: VR128, Op0: CmpReg, Op1: LHSReg);
2278 Register AndNReg = fastEmitInst_rr(MachineInstOpcode: Opc[2], RC: VR128, Op0: CmpReg, Op1: RHSReg);
2279 Register OrReg = fastEmitInst_rr(MachineInstOpcode: Opc[3], RC: VR128, Op0: AndNReg, Op1: AndReg);
2280 ResultReg = createResultReg(RC);
2281 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2282 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: OrReg);
2283 }
2284 updateValueMap(I, Reg: ResultReg);
2285 return true;
2286}
2287
2288bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
2289 // These are pseudo CMOV instructions and will be later expanded into control-
2290 // flow.
2291 unsigned Opc;
2292 switch (RetVT.SimpleTy) {
2293 default: return false;
2294 case MVT::i8: Opc = X86::CMOV_GR8; break;
2295 case MVT::i16: Opc = X86::CMOV_GR16; break;
2296 case MVT::i32: Opc = X86::CMOV_GR32; break;
2297 case MVT::f16:
2298 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR16X : X86::CMOV_FR16; break;
2299 case MVT::f32:
2300 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR32X : X86::CMOV_FR32; break;
2301 case MVT::f64:
2302 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR64X : X86::CMOV_FR64; break;
2303 }
2304
2305 const Value *Cond = I->getOperand(i: 0);
2306 X86::CondCode CC = X86::COND_NE;
2307
2308 // Optimize conditions coming from a compare if both instructions are in the
2309 // same basic block (values defined in other basic blocks may not have
2310 // initialized registers).
2311 const auto *CI = dyn_cast<CmpInst>(Val: Cond);
2312 if (CI && (CI->getParent() == I->getParent())) {
2313 bool NeedSwap;
2314 std::tie(args&: CC, args&: NeedSwap) = X86::getX86ConditionCode(Predicate: CI->getPredicate());
2315 if (CC > X86::LAST_VALID_COND)
2316 return false;
2317
2318 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
2319 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
2320
2321 if (NeedSwap)
2322 std::swap(a&: CmpLHS, b&: CmpRHS);
2323
2324 EVT CmpVT = TLI.getValueType(DL, Ty: CmpLHS->getType());
2325 if (!X86FastEmitCompare(Op0: CmpLHS, Op1: CmpRHS, VT: CmpVT, CurMIMD: CI->getDebugLoc()))
2326 return false;
2327 } else {
2328 Register CondReg = getRegForValue(V: Cond);
2329 if (!CondReg)
2330 return false;
2331
2332 // In case OpReg is a K register, COPY to a GPR
2333 if (MRI.getRegClass(Reg: CondReg) == &X86::VK1RegClass) {
2334 Register KCondReg = CondReg;
2335 CondReg = createResultReg(RC: &X86::GR32RegClass);
2336 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2337 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CondReg)
2338 .addReg(RegNo: KCondReg);
2339 CondReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: CondReg, Idx: X86::sub_8bit);
2340 }
2341 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TEST8ri))
2342 .addReg(RegNo: CondReg)
2343 .addImm(Val: 1);
2344 }
2345
2346 const Value *LHS = I->getOperand(i: 1);
2347 const Value *RHS = I->getOperand(i: 2);
2348
2349 Register LHSReg = getRegForValue(V: LHS);
2350 Register RHSReg = getRegForValue(V: RHS);
2351 if (!LHSReg || !RHSReg)
2352 return false;
2353
2354 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2355
2356 Register ResultReg =
2357 fastEmitInst_rri(MachineInstOpcode: Opc, RC, Op0: RHSReg, Op1: LHSReg, Imm: CC);
2358 updateValueMap(I, Reg: ResultReg);
2359 return true;
2360}
2361
2362bool X86FastISel::X86SelectSelect(const Instruction *I) {
2363 MVT RetVT;
2364 if (!isTypeLegal(Ty: I->getType(), VT&: RetVT))
2365 return false;
2366
2367 // Check if we can fold the select.
2368 if (const auto *CI = dyn_cast<CmpInst>(Val: I->getOperand(i: 0))) {
2369 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2370 const Value *Opnd = nullptr;
2371 switch (Predicate) {
2372 default: break;
2373 case CmpInst::FCMP_FALSE: Opnd = I->getOperand(i: 2); break;
2374 case CmpInst::FCMP_TRUE: Opnd = I->getOperand(i: 1); break;
2375 }
2376 // No need for a select anymore - this is an unconditional move.
2377 if (Opnd) {
2378 Register OpReg = getRegForValue(V: Opnd);
2379 if (!OpReg)
2380 return false;
2381 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2382 Register ResultReg = createResultReg(RC);
2383 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2384 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg)
2385 .addReg(RegNo: OpReg);
2386 updateValueMap(I, Reg: ResultReg);
2387 return true;
2388 }
2389 }
2390
2391 // First try to use real conditional move instructions.
2392 if (X86FastEmitCMoveSelect(RetVT, I))
2393 return true;
2394
2395 // Try to use a sequence of SSE instructions to simulate a conditional move.
2396 if (X86FastEmitSSESelect(RetVT, I))
2397 return true;
2398
2399 // Fall-back to pseudo conditional move instructions, which will be later
2400 // converted to control-flow.
2401 if (X86FastEmitPseudoSelect(RetVT, I))
2402 return true;
2403
2404 return false;
2405}
2406
2407// Common code for X86SelectSIToFP and X86SelectUIToFP.
2408bool X86FastISel::X86SelectIntToFP(const Instruction *I, bool IsSigned) {
2409 // The target-independent selection algorithm in FastISel already knows how
2410 // to select a SINT_TO_FP if the target is SSE but not AVX.
2411 // Early exit if the subtarget doesn't have AVX.
2412 // Unsigned conversion requires avx512.
2413 bool HasAVX512 = Subtarget->hasAVX512();
2414 if (!Subtarget->hasAVX() || (!IsSigned && !HasAVX512))
2415 return false;
2416
2417 // TODO: We could sign extend narrower types.
2418 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
2419 if (SrcVT != MVT::i32 && SrcVT != MVT::i64)
2420 return false;
2421
2422 // Select integer to float/double conversion.
2423 Register OpReg = getRegForValue(V: I->getOperand(i: 0));
2424 if (!OpReg)
2425 return false;
2426
2427 unsigned Opcode;
2428
2429 static const uint16_t SCvtOpc[2][2][2] = {
2430 { { X86::VCVTSI2SSrr, X86::VCVTSI642SSrr },
2431 { X86::VCVTSI2SDrr, X86::VCVTSI642SDrr } },
2432 { { X86::VCVTSI2SSZrr, X86::VCVTSI642SSZrr },
2433 { X86::VCVTSI2SDZrr, X86::VCVTSI642SDZrr } },
2434 };
2435 static const uint16_t UCvtOpc[2][2] = {
2436 { X86::VCVTUSI2SSZrr, X86::VCVTUSI642SSZrr },
2437 { X86::VCVTUSI2SDZrr, X86::VCVTUSI642SDZrr },
2438 };
2439 bool Is64Bit = SrcVT == MVT::i64;
2440
2441 if (I->getType()->isDoubleTy()) {
2442 // s/uitofp int -> double
2443 Opcode = IsSigned ? SCvtOpc[HasAVX512][1][Is64Bit] : UCvtOpc[1][Is64Bit];
2444 } else if (I->getType()->isFloatTy()) {
2445 // s/uitofp int -> float
2446 Opcode = IsSigned ? SCvtOpc[HasAVX512][0][Is64Bit] : UCvtOpc[0][Is64Bit];
2447 } else
2448 return false;
2449
2450 MVT DstVT = TLI.getValueType(DL, Ty: I->getType()).getSimpleVT();
2451 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: DstVT);
2452 Register ImplicitDefReg = createResultReg(RC);
2453 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2454 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2455 Register ResultReg = fastEmitInst_rr(MachineInstOpcode: Opcode, RC, Op0: ImplicitDefReg, Op1: OpReg);
2456 updateValueMap(I, Reg: ResultReg);
2457 return true;
2458}
2459
2460bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
2461 return X86SelectIntToFP(I, /*IsSigned*/true);
2462}
2463
2464bool X86FastISel::X86SelectUIToFP(const Instruction *I) {
2465 return X86SelectIntToFP(I, /*IsSigned*/false);
2466}
2467
2468// Helper method used by X86SelectFPExt and X86SelectFPTrunc.
2469bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction *I,
2470 unsigned TargetOpc,
2471 const TargetRegisterClass *RC) {
2472 assert((I->getOpcode() == Instruction::FPExt ||
2473 I->getOpcode() == Instruction::FPTrunc) &&
2474 "Instruction must be an FPExt or FPTrunc!");
2475 bool HasAVX = Subtarget->hasAVX();
2476
2477 Register OpReg = getRegForValue(V: I->getOperand(i: 0));
2478 if (!OpReg)
2479 return false;
2480
2481 Register ImplicitDefReg;
2482 if (HasAVX) {
2483 ImplicitDefReg = createResultReg(RC);
2484 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2485 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2486
2487 }
2488
2489 Register ResultReg = createResultReg(RC);
2490 MachineInstrBuilder MIB;
2491 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpc),
2492 DestReg: ResultReg);
2493
2494 if (HasAVX)
2495 MIB.addReg(RegNo: ImplicitDefReg);
2496
2497 MIB.addReg(RegNo: OpReg);
2498 updateValueMap(I, Reg: ResultReg);
2499 return true;
2500}
2501
2502bool X86FastISel::X86SelectFPExt(const Instruction *I) {
2503 if (Subtarget->hasSSE2() && I->getType()->isDoubleTy() &&
2504 I->getOperand(i: 0)->getType()->isFloatTy()) {
2505 bool HasAVX512 = Subtarget->hasAVX512();
2506 // fpext from float to double.
2507 unsigned Opc =
2508 HasAVX512 ? X86::VCVTSS2SDZrr
2509 : Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr;
2510 return X86SelectFPExtOrFPTrunc(I, TargetOpc: Opc, RC: TLI.getRegClassFor(VT: MVT::f64));
2511 }
2512
2513 return false;
2514}
2515
2516bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
2517 if (Subtarget->hasSSE2() && I->getType()->isFloatTy() &&
2518 I->getOperand(i: 0)->getType()->isDoubleTy()) {
2519 bool HasAVX512 = Subtarget->hasAVX512();
2520 // fptrunc from double to float.
2521 unsigned Opc =
2522 HasAVX512 ? X86::VCVTSD2SSZrr
2523 : Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr;
2524 return X86SelectFPExtOrFPTrunc(I, TargetOpc: Opc, RC: TLI.getRegClassFor(VT: MVT::f32));
2525 }
2526
2527 return false;
2528}
2529
2530bool X86FastISel::X86SelectTrunc(const Instruction *I) {
2531 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
2532 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
2533
2534 // This code only handles truncation to byte.
2535 if (DstVT != MVT::i8 && DstVT != MVT::i1)
2536 return false;
2537 if (!TLI.isTypeLegal(VT: SrcVT))
2538 return false;
2539
2540 Register InputReg = getRegForValue(V: I->getOperand(i: 0));
2541 if (!InputReg)
2542 // Unhandled operand. Halt "fast" selection and bail.
2543 return false;
2544
2545 if (SrcVT == MVT::i8) {
2546 // Truncate from i8 to i1; no code needed.
2547 updateValueMap(I, Reg: InputReg);
2548 return true;
2549 }
2550
2551 // Issue an extract_subreg.
2552 Register ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: InputReg,
2553 Idx: X86::sub_8bit);
2554 if (!ResultReg)
2555 return false;
2556
2557 updateValueMap(I, Reg: ResultReg);
2558 return true;
2559}
2560
2561bool X86FastISel::X86SelectBitCast(const Instruction *I) {
2562 // Select SSE2/AVX bitcasts between 128/256/512 bit vector types.
2563 MVT SrcVT, DstVT;
2564 if (!Subtarget->hasSSE2() ||
2565 !isTypeLegal(Ty: I->getOperand(i: 0)->getType(), VT&: SrcVT) ||
2566 !isTypeLegal(Ty: I->getType(), VT&: DstVT))
2567 return false;
2568
2569 // Only allow vectors that use xmm/ymm/zmm.
2570 if (!SrcVT.isVector() || !DstVT.isVector() ||
2571 SrcVT.getVectorElementType() == MVT::i1 ||
2572 DstVT.getVectorElementType() == MVT::i1)
2573 return false;
2574
2575 Register Reg = getRegForValue(V: I->getOperand(i: 0));
2576 if (!Reg)
2577 return false;
2578
2579 // Emit a reg-reg copy so we don't propagate cached known bits information
2580 // with the wrong VT if we fall out of fast isel after selecting this.
2581 const TargetRegisterClass *DstClass = TLI.getRegClassFor(VT: DstVT);
2582 Register ResultReg = createResultReg(RC: DstClass);
2583 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2584 DestReg: ResultReg)
2585 .addReg(RegNo: Reg);
2586
2587 updateValueMap(I, Reg: ResultReg);
2588 return true;
2589}
2590
2591bool X86FastISel::IsMemcpySmall(uint64_t Len) {
2592 return Len <= (Subtarget->is64Bit() ? 32 : 16);
2593}
2594
2595bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,
2596 X86AddressMode SrcAM, uint64_t Len) {
2597
2598 // Make sure we don't bloat code by inlining very large memcpy's.
2599 if (!IsMemcpySmall(Len))
2600 return false;
2601
2602 bool i64Legal = Subtarget->is64Bit();
2603
2604 // We don't care about alignment here since we just emit integer accesses.
2605 while (Len) {
2606 MVT VT;
2607 if (Len >= 8 && i64Legal)
2608 VT = MVT::i64;
2609 else if (Len >= 4)
2610 VT = MVT::i32;
2611 else if (Len >= 2)
2612 VT = MVT::i16;
2613 else
2614 VT = MVT::i8;
2615
2616 Register Reg;
2617 bool RV = X86FastEmitLoad(VT, AM&: SrcAM, MMO: nullptr, ResultReg&: Reg);
2618 RV &= X86FastEmitStore(VT, ValReg: Reg, AM&: DestAM);
2619 assert(RV && "Failed to emit load or store??");
2620 (void)RV;
2621
2622 unsigned Size = VT.getSizeInBits()/8;
2623 Len -= Size;
2624 DestAM.Disp += Size;
2625 SrcAM.Disp += Size;
2626 }
2627
2628 return true;
2629}
2630
2631bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
2632 // FIXME: Handle more intrinsics.
2633 switch (II->getIntrinsicID()) {
2634 default:
2635 return false;
2636 case Intrinsic::frameaddress: {
2637 MachineFunction *MF = FuncInfo.MF;
2638 if (MF->getTarget().getMCAsmInfo()->usesWindowsCFI())
2639 return false;
2640
2641 Type *RetTy = II->getCalledFunction()->getReturnType();
2642
2643 MVT VT;
2644 if (!isTypeLegal(Ty: RetTy, VT))
2645 return false;
2646
2647 unsigned Opc;
2648 const TargetRegisterClass *RC = nullptr;
2649
2650 switch (VT.SimpleTy) {
2651 default: llvm_unreachable("Invalid result type for frameaddress.");
2652 case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass; break;
2653 case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass; break;
2654 }
2655
2656 // This needs to be set before we call getPtrSizedFrameRegister, otherwise
2657 // we get the wrong frame register.
2658 MachineFrameInfo &MFI = MF->getFrameInfo();
2659 MFI.setFrameAddressIsTaken(true);
2660
2661 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2662 Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF: *MF);
2663 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
2664 (FrameReg == X86::EBP && VT == MVT::i32)) &&
2665 "Invalid Frame Register!");
2666
2667 // Always make a copy of the frame register to a vreg first, so that we
2668 // never directly reference the frame register (the TwoAddressInstruction-
2669 // Pass doesn't like that).
2670 Register SrcReg = createResultReg(RC);
2671 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2672 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: SrcReg).addReg(RegNo: FrameReg);
2673
2674 // Now recursively load from the frame address.
2675 // movq (%rbp), %rax
2676 // movq (%rax), %rax
2677 // movq (%rax), %rax
2678 // ...
2679 unsigned Depth = cast<ConstantInt>(Val: II->getOperand(i_nocapture: 0))->getZExtValue();
2680 while (Depth--) {
2681 Register DestReg = createResultReg(RC);
2682 addDirectMem(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2683 MCID: TII.get(Opcode: Opc), DestReg), Reg: SrcReg);
2684 SrcReg = DestReg;
2685 }
2686
2687 updateValueMap(I: II, Reg: SrcReg);
2688 return true;
2689 }
2690 case Intrinsic::memcpy: {
2691 const MemCpyInst *MCI = cast<MemCpyInst>(Val: II);
2692 // Don't handle volatile or variable length memcpys.
2693 if (MCI->isVolatile())
2694 return false;
2695
2696 if (isa<ConstantInt>(Val: MCI->getLength())) {
2697 // Small memcpy's are common enough that we want to do them
2698 // without a call if possible.
2699 uint64_t Len = cast<ConstantInt>(Val: MCI->getLength())->getZExtValue();
2700 if (IsMemcpySmall(Len)) {
2701 X86AddressMode DestAM, SrcAM;
2702 if (!X86SelectAddress(V: MCI->getRawDest(), AM&: DestAM) ||
2703 !X86SelectAddress(V: MCI->getRawSource(), AM&: SrcAM))
2704 return false;
2705 TryEmitSmallMemcpy(DestAM, SrcAM, Len);
2706 return true;
2707 }
2708 }
2709
2710 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2711 if (!MCI->getLength()->getType()->isIntegerTy(Bitwidth: SizeWidth))
2712 return false;
2713
2714 if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255)
2715 return false;
2716
2717 return lowerCallTo(CI: II, SymName: "memcpy", NumArgs: II->arg_size() - 1);
2718 }
2719 case Intrinsic::memset: {
2720 const MemSetInst *MSI = cast<MemSetInst>(Val: II);
2721
2722 if (MSI->isVolatile())
2723 return false;
2724
2725 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2726 if (!MSI->getLength()->getType()->isIntegerTy(Bitwidth: SizeWidth))
2727 return false;
2728
2729 if (MSI->getDestAddressSpace() > 255)
2730 return false;
2731
2732 return lowerCallTo(CI: II, SymName: "memset", NumArgs: II->arg_size() - 1);
2733 }
2734 case Intrinsic::stackprotector: {
2735 // Emit code to store the stack guard onto the stack.
2736 EVT PtrTy = TLI.getPointerTy(DL);
2737
2738 const Value *Op1 = II->getArgOperand(i: 0); // The guard's value.
2739 const AllocaInst *Slot = cast<AllocaInst>(Val: II->getArgOperand(i: 1));
2740
2741 MFI.setStackProtectorIndex(FuncInfo.StaticAllocaMap[Slot]);
2742
2743 // Grab the frame index.
2744 X86AddressMode AM;
2745 if (!X86SelectAddress(V: Slot, AM)) return false;
2746 if (!X86FastEmitStore(VT: PtrTy, Val: Op1, AM)) return false;
2747 return true;
2748 }
2749 case Intrinsic::dbg_declare: {
2750 const DbgDeclareInst *DI = cast<DbgDeclareInst>(Val: II);
2751 X86AddressMode AM;
2752 assert(DI->getAddress() && "Null address should be checked earlier!");
2753 if (!X86SelectAddress(V: DI->getAddress(), AM))
2754 return false;
2755 const MCInstrDesc &II = TII.get(Opcode: TargetOpcode::DBG_VALUE);
2756 assert(DI->getVariable()->isValidLocationForIntrinsic(MIMD.getDL()) &&
2757 "Expected inlined-at fields to agree");
2758 addFullAddress(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II), AM)
2759 .addImm(Val: 0)
2760 .addMetadata(MD: DI->getVariable())
2761 .addMetadata(MD: DI->getExpression());
2762 return true;
2763 }
2764 case Intrinsic::trap: {
2765 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TRAP));
2766 return true;
2767 }
2768 case Intrinsic::sqrt: {
2769 if (!Subtarget->hasSSE1())
2770 return false;
2771
2772 Type *RetTy = II->getCalledFunction()->getReturnType();
2773
2774 MVT VT;
2775 if (!isTypeLegal(Ty: RetTy, VT))
2776 return false;
2777
2778 // Unfortunately we can't use fastEmit_r, because the AVX version of FSQRT
2779 // is not generated by FastISel yet.
2780 // FIXME: Update this code once tablegen can handle it.
2781 static const uint16_t SqrtOpc[3][2] = {
2782 { X86::SQRTSSr, X86::SQRTSDr },
2783 { X86::VSQRTSSr, X86::VSQRTSDr },
2784 { X86::VSQRTSSZr, X86::VSQRTSDZr },
2785 };
2786 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
2787 Subtarget->hasAVX() ? 1 :
2788 0;
2789 unsigned Opc;
2790 switch (VT.SimpleTy) {
2791 default: return false;
2792 case MVT::f32: Opc = SqrtOpc[AVXLevel][0]; break;
2793 case MVT::f64: Opc = SqrtOpc[AVXLevel][1]; break;
2794 }
2795
2796 const Value *SrcVal = II->getArgOperand(i: 0);
2797 Register SrcReg = getRegForValue(V: SrcVal);
2798
2799 if (!SrcReg)
2800 return false;
2801
2802 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
2803 Register ImplicitDefReg;
2804 if (AVXLevel > 0) {
2805 ImplicitDefReg = createResultReg(RC);
2806 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2807 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2808 }
2809
2810 Register ResultReg = createResultReg(RC);
2811 MachineInstrBuilder MIB;
2812 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc),
2813 DestReg: ResultReg);
2814
2815 if (ImplicitDefReg)
2816 MIB.addReg(RegNo: ImplicitDefReg);
2817
2818 MIB.addReg(RegNo: SrcReg);
2819
2820 updateValueMap(I: II, Reg: ResultReg);
2821 return true;
2822 }
2823 case Intrinsic::sadd_with_overflow:
2824 case Intrinsic::uadd_with_overflow:
2825 case Intrinsic::ssub_with_overflow:
2826 case Intrinsic::usub_with_overflow:
2827 case Intrinsic::smul_with_overflow:
2828 case Intrinsic::umul_with_overflow: {
2829 // This implements the basic lowering of the xalu with overflow intrinsics
2830 // into add/sub/mul followed by either seto or setb.
2831 const Function *Callee = II->getCalledFunction();
2832 auto *Ty = cast<StructType>(Val: Callee->getReturnType());
2833 Type *RetTy = Ty->getTypeAtIndex(N: 0U);
2834 assert(Ty->getTypeAtIndex(1)->isIntegerTy() &&
2835 Ty->getTypeAtIndex(1)->getScalarSizeInBits() == 1 &&
2836 "Overflow value expected to be an i1");
2837
2838 MVT VT;
2839 if (!isTypeLegal(Ty: RetTy, VT))
2840 return false;
2841
2842 if (VT < MVT::i8 || VT > MVT::i64)
2843 return false;
2844
2845 const Value *LHS = II->getArgOperand(i: 0);
2846 const Value *RHS = II->getArgOperand(i: 1);
2847
2848 // Canonicalize immediate to the RHS.
2849 if (isa<ConstantInt>(Val: LHS) && !isa<ConstantInt>(Val: RHS) && II->isCommutative())
2850 std::swap(a&: LHS, b&: RHS);
2851
2852 unsigned BaseOpc, CondCode;
2853 switch (II->getIntrinsicID()) {
2854 default: llvm_unreachable("Unexpected intrinsic!");
2855 case Intrinsic::sadd_with_overflow:
2856 BaseOpc = ISD::ADD; CondCode = X86::COND_O; break;
2857 case Intrinsic::uadd_with_overflow:
2858 BaseOpc = ISD::ADD; CondCode = X86::COND_B; break;
2859 case Intrinsic::ssub_with_overflow:
2860 BaseOpc = ISD::SUB; CondCode = X86::COND_O; break;
2861 case Intrinsic::usub_with_overflow:
2862 BaseOpc = ISD::SUB; CondCode = X86::COND_B; break;
2863 case Intrinsic::smul_with_overflow:
2864 BaseOpc = X86ISD::SMUL; CondCode = X86::COND_O; break;
2865 case Intrinsic::umul_with_overflow:
2866 BaseOpc = X86ISD::UMUL; CondCode = X86::COND_O; break;
2867 }
2868
2869 Register LHSReg = getRegForValue(V: LHS);
2870 if (!LHSReg)
2871 return false;
2872
2873 Register ResultReg;
2874 // Check if we have an immediate version.
2875 if (const auto *CI = dyn_cast<ConstantInt>(Val: RHS)) {
2876 static const uint16_t Opc[2][4] = {
2877 { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },
2878 { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
2879 };
2880
2881 if (CI->isOne() && (BaseOpc == ISD::ADD || BaseOpc == ISD::SUB) &&
2882 CondCode == X86::COND_O) {
2883 // We can use INC/DEC.
2884 ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
2885 bool IsDec = BaseOpc == ISD::SUB;
2886 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2887 MCID: TII.get(Opcode: Opc[IsDec][VT.SimpleTy-MVT::i8]), DestReg: ResultReg)
2888 .addReg(RegNo: LHSReg);
2889 } else
2890 ResultReg = fastEmit_ri(VT, RetVT: VT, Opcode: BaseOpc, Op0: LHSReg, imm1: CI->getZExtValue());
2891 }
2892
2893 Register RHSReg;
2894 if (!ResultReg) {
2895 RHSReg = getRegForValue(V: RHS);
2896 if (!RHSReg)
2897 return false;
2898 ResultReg = fastEmit_rr(VT, RetVT: VT, Opcode: BaseOpc, Op0: LHSReg, Op1: RHSReg);
2899 }
2900
2901 // FastISel doesn't have a pattern for all X86::MUL*r and X86::IMUL*r. Emit
2902 // it manually.
2903 if (BaseOpc == X86ISD::UMUL && !ResultReg) {
2904 static const uint16_t MULOpc[] =
2905 { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };
2906 static const MCPhysReg Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };
2907 // First copy the first operand into RAX, which is an implicit input to
2908 // the X86::MUL*r instruction.
2909 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2910 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg[VT.SimpleTy-MVT::i8])
2911 .addReg(RegNo: LHSReg);
2912 ResultReg = fastEmitInst_r(MachineInstOpcode: MULOpc[VT.SimpleTy-MVT::i8],
2913 RC: TLI.getRegClassFor(VT), Op0: RHSReg);
2914 } else if (BaseOpc == X86ISD::SMUL && !ResultReg) {
2915 static const uint16_t MULOpc[] =
2916 { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };
2917 if (VT == MVT::i8) {
2918 // Copy the first operand into AL, which is an implicit input to the
2919 // X86::IMUL8r instruction.
2920 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2921 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::AL)
2922 .addReg(RegNo: LHSReg);
2923 ResultReg = fastEmitInst_r(MachineInstOpcode: MULOpc[0], RC: TLI.getRegClassFor(VT), Op0: RHSReg);
2924 } else
2925 ResultReg = fastEmitInst_rr(MachineInstOpcode: MULOpc[VT.SimpleTy-MVT::i8],
2926 RC: TLI.getRegClassFor(VT), Op0: LHSReg, Op1: RHSReg);
2927 }
2928
2929 if (!ResultReg)
2930 return false;
2931
2932 // Assign to a GPR since the overflow return value is lowered to a SETcc.
2933 Register ResultReg2 = createResultReg(RC: &X86::GR8RegClass);
2934 assert((ResultReg+1) == ResultReg2 && "Nonconsecutive result registers.");
2935 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
2936 DestReg: ResultReg2)
2937 .addImm(Val: CondCode);
2938
2939 updateValueMap(I: II, Reg: ResultReg, NumRegs: 2);
2940 return true;
2941 }
2942 case Intrinsic::x86_sse_cvttss2si:
2943 case Intrinsic::x86_sse_cvttss2si64:
2944 case Intrinsic::x86_sse2_cvttsd2si:
2945 case Intrinsic::x86_sse2_cvttsd2si64: {
2946 bool IsInputDouble;
2947 switch (II->getIntrinsicID()) {
2948 default: llvm_unreachable("Unexpected intrinsic.");
2949 case Intrinsic::x86_sse_cvttss2si:
2950 case Intrinsic::x86_sse_cvttss2si64:
2951 if (!Subtarget->hasSSE1())
2952 return false;
2953 IsInputDouble = false;
2954 break;
2955 case Intrinsic::x86_sse2_cvttsd2si:
2956 case Intrinsic::x86_sse2_cvttsd2si64:
2957 if (!Subtarget->hasSSE2())
2958 return false;
2959 IsInputDouble = true;
2960 break;
2961 }
2962
2963 Type *RetTy = II->getCalledFunction()->getReturnType();
2964 MVT VT;
2965 if (!isTypeLegal(Ty: RetTy, VT))
2966 return false;
2967
2968 static const uint16_t CvtOpc[3][2][2] = {
2969 { { X86::CVTTSS2SIrr, X86::CVTTSS2SI64rr },
2970 { X86::CVTTSD2SIrr, X86::CVTTSD2SI64rr } },
2971 { { X86::VCVTTSS2SIrr, X86::VCVTTSS2SI64rr },
2972 { X86::VCVTTSD2SIrr, X86::VCVTTSD2SI64rr } },
2973 { { X86::VCVTTSS2SIZrr, X86::VCVTTSS2SI64Zrr },
2974 { X86::VCVTTSD2SIZrr, X86::VCVTTSD2SI64Zrr } },
2975 };
2976 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
2977 Subtarget->hasAVX() ? 1 :
2978 0;
2979 unsigned Opc;
2980 switch (VT.SimpleTy) {
2981 default: llvm_unreachable("Unexpected result type.");
2982 case MVT::i32: Opc = CvtOpc[AVXLevel][IsInputDouble][0]; break;
2983 case MVT::i64: Opc = CvtOpc[AVXLevel][IsInputDouble][1]; break;
2984 }
2985
2986 // Check if we can fold insertelement instructions into the convert.
2987 const Value *Op = II->getArgOperand(i: 0);
2988 while (auto *IE = dyn_cast<InsertElementInst>(Val: Op)) {
2989 const Value *Index = IE->getOperand(i_nocapture: 2);
2990 if (!isa<ConstantInt>(Val: Index))
2991 break;
2992 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
2993
2994 if (!Idx) {
2995 Op = IE->getOperand(i_nocapture: 1);
2996 break;
2997 }
2998 Op = IE->getOperand(i_nocapture: 0);
2999 }
3000
3001 Register Reg = getRegForValue(V: Op);
3002 if (!Reg)
3003 return false;
3004
3005 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3006 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: ResultReg)
3007 .addReg(RegNo: Reg);
3008
3009 updateValueMap(I: II, Reg: ResultReg);
3010 return true;
3011 }
3012 case Intrinsic::x86_sse42_crc32_32_8:
3013 case Intrinsic::x86_sse42_crc32_32_16:
3014 case Intrinsic::x86_sse42_crc32_32_32:
3015 case Intrinsic::x86_sse42_crc32_64_64: {
3016 if (!Subtarget->hasCRC32())
3017 return false;
3018
3019 Type *RetTy = II->getCalledFunction()->getReturnType();
3020
3021 MVT VT;
3022 if (!isTypeLegal(Ty: RetTy, VT))
3023 return false;
3024
3025 unsigned Opc;
3026 const TargetRegisterClass *RC = nullptr;
3027
3028 switch (II->getIntrinsicID()) {
3029 default:
3030 llvm_unreachable("Unexpected intrinsic.");
3031#define GET_EGPR_IF_ENABLED(OPC) Subtarget->hasEGPR() ? OPC##_EVEX : OPC
3032 case Intrinsic::x86_sse42_crc32_32_8:
3033 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r8);
3034 RC = &X86::GR32RegClass;
3035 break;
3036 case Intrinsic::x86_sse42_crc32_32_16:
3037 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r16);
3038 RC = &X86::GR32RegClass;
3039 break;
3040 case Intrinsic::x86_sse42_crc32_32_32:
3041 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r32);
3042 RC = &X86::GR32RegClass;
3043 break;
3044 case Intrinsic::x86_sse42_crc32_64_64:
3045 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r64r64);
3046 RC = &X86::GR64RegClass;
3047 break;
3048#undef GET_EGPR_IF_ENABLED
3049 }
3050
3051 const Value *LHS = II->getArgOperand(i: 0);
3052 const Value *RHS = II->getArgOperand(i: 1);
3053
3054 Register LHSReg = getRegForValue(V: LHS);
3055 Register RHSReg = getRegForValue(V: RHS);
3056 if (!LHSReg || !RHSReg)
3057 return false;
3058
3059 Register ResultReg = fastEmitInst_rr(MachineInstOpcode: Opc, RC, Op0: LHSReg, Op1: RHSReg);
3060 if (!ResultReg)
3061 return false;
3062
3063 updateValueMap(I: II, Reg: ResultReg);
3064 return true;
3065 }
3066 }
3067}
3068
3069bool X86FastISel::fastLowerArguments() {
3070 if (!FuncInfo.CanLowerReturn)
3071 return false;
3072
3073 const Function *F = FuncInfo.Fn;
3074 if (F->isVarArg())
3075 return false;
3076
3077 CallingConv::ID CC = F->getCallingConv();
3078 if (CC != CallingConv::C)
3079 return false;
3080
3081 if (Subtarget->isCallingConvWin64(CC))
3082 return false;
3083
3084 if (!Subtarget->is64Bit())
3085 return false;
3086
3087 if (Subtarget->useSoftFloat())
3088 return false;
3089
3090 // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.
3091 unsigned GPRCnt = 0;
3092 unsigned FPRCnt = 0;
3093 for (auto const &Arg : F->args()) {
3094 if (Arg.hasAttribute(Kind: Attribute::ByVal) ||
3095 Arg.hasAttribute(Kind: Attribute::InReg) ||
3096 Arg.hasAttribute(Kind: Attribute::StructRet) ||
3097 Arg.hasAttribute(Kind: Attribute::SwiftSelf) ||
3098 Arg.hasAttribute(Kind: Attribute::SwiftAsync) ||
3099 Arg.hasAttribute(Kind: Attribute::SwiftError) ||
3100 Arg.hasAttribute(Kind: Attribute::Nest))
3101 return false;
3102
3103 Type *ArgTy = Arg.getType();
3104 if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())
3105 return false;
3106
3107 EVT ArgVT = TLI.getValueType(DL, Ty: ArgTy);
3108 if (!ArgVT.isSimple()) return false;
3109 switch (ArgVT.getSimpleVT().SimpleTy) {
3110 default: return false;
3111 case MVT::i32:
3112 case MVT::i64:
3113 ++GPRCnt;
3114 break;
3115 case MVT::f32:
3116 case MVT::f64:
3117 if (!Subtarget->hasSSE1())
3118 return false;
3119 ++FPRCnt;
3120 break;
3121 }
3122
3123 if (GPRCnt > 6)
3124 return false;
3125
3126 if (FPRCnt > 8)
3127 return false;
3128 }
3129
3130 static const MCPhysReg GPR32ArgRegs[] = {
3131 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
3132 };
3133 static const MCPhysReg GPR64ArgRegs[] = {
3134 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9
3135 };
3136 static const MCPhysReg XMMArgRegs[] = {
3137 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3138 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3139 };
3140
3141 unsigned GPRIdx = 0;
3142 unsigned FPRIdx = 0;
3143 for (auto const &Arg : F->args()) {
3144 MVT VT = TLI.getSimpleValueType(DL, Ty: Arg.getType());
3145 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
3146 MCRegister SrcReg;
3147 switch (VT.SimpleTy) {
3148 default: llvm_unreachable("Unexpected value type.");
3149 case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;
3150 case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++]; break;
3151 case MVT::f32: [[fallthrough]];
3152 case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++]; break;
3153 }
3154 Register DstReg = FuncInfo.MF->addLiveIn(PReg: SrcReg, RC);
3155 // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.
3156 // Without this, EmitLiveInCopies may eliminate the livein if its only
3157 // use is a bitcast (which isn't turned into an instruction).
3158 Register ResultReg = createResultReg(RC);
3159 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3160 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg)
3161 .addReg(RegNo: DstReg, Flags: getKillRegState(B: true));
3162 updateValueMap(I: &Arg, Reg: ResultReg);
3163 }
3164 return true;
3165}
3166
3167static unsigned computeBytesPoppedByCalleeForSRet(const X86Subtarget *Subtarget,
3168 CallingConv::ID CC,
3169 const CallBase *CB) {
3170 if (Subtarget->is64Bit())
3171 return 0;
3172 if (Subtarget->getTargetTriple().isOSMSVCRT())
3173 return 0;
3174 if (CC == CallingConv::Fast || CC == CallingConv::GHC ||
3175 CC == CallingConv::HiPE || CC == CallingConv::Tail ||
3176 CC == CallingConv::SwiftTail)
3177 return 0;
3178
3179 if (CB)
3180 if (CB->arg_empty() || !CB->paramHasAttr(ArgNo: 0, Kind: Attribute::StructRet) ||
3181 CB->paramHasAttr(ArgNo: 0, Kind: Attribute::InReg) || Subtarget->isTargetMCU())
3182 return 0;
3183
3184 return 4;
3185}
3186
3187bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3188 auto &OutVals = CLI.OutVals;
3189 auto &OutFlags = CLI.OutFlags;
3190 auto &OutRegs = CLI.OutRegs;
3191 auto &Ins = CLI.Ins;
3192 auto &InRegs = CLI.InRegs;
3193 CallingConv::ID CC = CLI.CallConv;
3194 bool &IsTailCall = CLI.IsTailCall;
3195 bool IsVarArg = CLI.IsVarArg;
3196 const Value *Callee = CLI.Callee;
3197 MCSymbol *Symbol = CLI.Symbol;
3198 const auto *CB = CLI.CB;
3199
3200 bool Is64Bit = Subtarget->is64Bit();
3201 bool IsWin64 = Subtarget->isCallingConvWin64(CC);
3202
3203 // Call / invoke instructions with NoCfCheck attribute require special
3204 // handling.
3205 if (CB && CB->doesNoCfCheck())
3206 return false;
3207
3208 // Functions with no_caller_saved_registers that need special handling.
3209 if ((CB && isa<CallInst>(Val: CB) && CB->hasFnAttr(Kind: "no_caller_saved_registers")))
3210 return false;
3211
3212 // Functions with no_callee_saved_registers that need special handling.
3213 if ((CB && CB->hasFnAttr(Kind: "no_callee_saved_registers")))
3214 return false;
3215
3216 // Indirect calls with CFI checks need special handling.
3217 if (CB && CB->isIndirectCall() && CB->getOperandBundle(ID: LLVMContext::OB_kcfi))
3218 return false;
3219
3220 // Functions using thunks for indirect calls need to use SDISel.
3221 if (Subtarget->useIndirectThunkCalls())
3222 return false;
3223
3224 // Handle only C and fastcc calling conventions for now.
3225 switch (CC) {
3226 default: return false;
3227 case CallingConv::C:
3228 case CallingConv::Fast:
3229 case CallingConv::Tail:
3230 case CallingConv::Swift:
3231 case CallingConv::SwiftTail:
3232 case CallingConv::X86_FastCall:
3233 case CallingConv::X86_StdCall:
3234 case CallingConv::X86_ThisCall:
3235 case CallingConv::Win64:
3236 case CallingConv::X86_64_SysV:
3237 case CallingConv::CFGuard_Check:
3238 break;
3239 }
3240
3241 // Allow SelectionDAG isel to handle tail calls.
3242 if (IsTailCall)
3243 return false;
3244
3245 // fastcc with -tailcallopt is intended to provide a guaranteed
3246 // tail call optimization. Fastisel doesn't know how to do that.
3247 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
3248 CC == CallingConv::Tail || CC == CallingConv::SwiftTail)
3249 return false;
3250
3251 // Don't know how to handle Win64 varargs yet. Nothing special needed for
3252 // x86-32. Special handling for x86-64 is implemented.
3253 if (IsVarArg && IsWin64)
3254 return false;
3255
3256 // Don't know about inalloca yet.
3257 if (CLI.CB && CLI.CB->hasInAllocaArgument())
3258 return false;
3259
3260 for (auto Flag : CLI.OutFlags)
3261 if (Flag.isSwiftError() || Flag.isPreallocated())
3262 return false;
3263
3264 // Can't handle import call optimization.
3265 if (Is64Bit &&
3266 MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization"))
3267 return false;
3268
3269 SmallVector<MVT, 16> OutVTs;
3270 SmallVector<Type *, 16> ArgTys;
3271 SmallVector<Register, 16> ArgRegs;
3272
3273 // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
3274 // instruction. This is safe because it is common to all FastISel supported
3275 // calling conventions on x86.
3276 for (int i = 0, e = OutVals.size(); i != e; ++i) {
3277 Value *&Val = OutVals[i];
3278 ISD::ArgFlagsTy Flags = OutFlags[i];
3279 if (auto *CI = dyn_cast<ConstantInt>(Val)) {
3280 if (CI->getBitWidth() < 32) {
3281 if (Flags.isSExt())
3282 Val = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue().sext(width: 32));
3283 else
3284 Val = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue().zext(width: 32));
3285 }
3286 }
3287
3288 // Passing bools around ends up doing a trunc to i1 and passing it.
3289 // Codegen this as an argument + "and 1".
3290 MVT VT;
3291 auto *TI = dyn_cast<TruncInst>(Val);
3292 Register ResultReg;
3293 if (TI && TI->getType()->isIntegerTy(Bitwidth: 1) && CLI.CB &&
3294 (TI->getParent() == CLI.CB->getParent()) && TI->hasOneUse()) {
3295 Value *PrevVal = TI->getOperand(i_nocapture: 0);
3296 ResultReg = getRegForValue(V: PrevVal);
3297
3298 if (!ResultReg)
3299 return false;
3300
3301 if (!isTypeLegal(Ty: PrevVal->getType(), VT))
3302 return false;
3303
3304 ResultReg = fastEmit_ri(VT, RetVT: VT, Opcode: ISD::AND, Op0: ResultReg, imm1: 1);
3305 } else {
3306 if (!isTypeLegal(Ty: Val->getType(), VT) ||
3307 (VT.isVector() && VT.getVectorElementType() == MVT::i1))
3308 return false;
3309 ResultReg = getRegForValue(V: Val);
3310 }
3311
3312 if (!ResultReg)
3313 return false;
3314
3315 ArgRegs.push_back(Elt: ResultReg);
3316 OutVTs.push_back(Elt: VT);
3317 ArgTys.push_back(Elt: Val->getType());
3318 }
3319
3320 // Analyze operands of the call, assigning locations to each operand.
3321 SmallVector<CCValAssign, 16> ArgLocs;
3322 CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());
3323
3324 // Allocate shadow area for Win64
3325 if (IsWin64)
3326 CCInfo.AllocateStack(Size: 32, Alignment: Align(8));
3327
3328 CCInfo.AnalyzeCallOperands(ArgVTs&: OutVTs, Flags&: OutFlags, OrigTys&: ArgTys, Fn: CC_X86);
3329
3330 // Get a count of how many bytes are to be pushed on the stack.
3331 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
3332
3333 // Issue CALLSEQ_START
3334 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
3335 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackDown))
3336 .addImm(Val: NumBytes).addImm(Val: 0).addImm(Val: 0);
3337
3338 // Walk the register/memloc assignments, inserting copies/loads.
3339 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3340 for (const CCValAssign &VA : ArgLocs) {
3341 const Value *ArgVal = OutVals[VA.getValNo()];
3342 MVT ArgVT = OutVTs[VA.getValNo()];
3343
3344 if (ArgVT == MVT::x86mmx)
3345 return false;
3346
3347 Register ArgReg = ArgRegs[VA.getValNo()];
3348
3349 // Promote the value if needed.
3350 switch (VA.getLocInfo()) {
3351 case CCValAssign::Full: break;
3352 case CCValAssign::SExt: {
3353 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3354 "Unexpected extend");
3355
3356 if (ArgVT == MVT::i1)
3357 return false;
3358
3359 bool Emitted = X86FastEmitExtend(Opc: ISD::SIGN_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3360 SrcVT: ArgVT, ResultReg&: ArgReg);
3361 assert(Emitted && "Failed to emit a sext!"); (void)Emitted;
3362 ArgVT = VA.getLocVT();
3363 break;
3364 }
3365 case CCValAssign::ZExt: {
3366 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3367 "Unexpected extend");
3368
3369 // Handle zero-extension from i1 to i8, which is common.
3370 if (ArgVT == MVT::i1) {
3371 // Set the high bits to zero.
3372 ArgReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: ArgReg);
3373 ArgVT = MVT::i8;
3374
3375 if (!ArgReg)
3376 return false;
3377 }
3378
3379 bool Emitted = X86FastEmitExtend(Opc: ISD::ZERO_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3380 SrcVT: ArgVT, ResultReg&: ArgReg);
3381 assert(Emitted && "Failed to emit a zext!"); (void)Emitted;
3382 ArgVT = VA.getLocVT();
3383 break;
3384 }
3385 case CCValAssign::AExt: {
3386 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3387 "Unexpected extend");
3388 bool Emitted = X86FastEmitExtend(Opc: ISD::ANY_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3389 SrcVT: ArgVT, ResultReg&: ArgReg);
3390 if (!Emitted)
3391 Emitted = X86FastEmitExtend(Opc: ISD::ZERO_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3392 SrcVT: ArgVT, ResultReg&: ArgReg);
3393 if (!Emitted)
3394 Emitted = X86FastEmitExtend(Opc: ISD::SIGN_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3395 SrcVT: ArgVT, ResultReg&: ArgReg);
3396
3397 assert(Emitted && "Failed to emit a aext!"); (void)Emitted;
3398 ArgVT = VA.getLocVT();
3399 break;
3400 }
3401 case CCValAssign::BCvt: {
3402 ArgReg = fastEmit_r(VT: ArgVT, RetVT: VA.getLocVT(), Opcode: ISD::BITCAST, Op0: ArgReg);
3403 assert(ArgReg && "Failed to emit a bitcast!");
3404 ArgVT = VA.getLocVT();
3405 break;
3406 }
3407 case CCValAssign::VExt:
3408 // VExt has not been implemented, so this should be impossible to reach
3409 // for now. However, fallback to Selection DAG isel once implemented.
3410 return false;
3411 case CCValAssign::AExtUpper:
3412 case CCValAssign::SExtUpper:
3413 case CCValAssign::ZExtUpper:
3414 case CCValAssign::FPExt:
3415 case CCValAssign::Trunc:
3416 llvm_unreachable("Unexpected loc info!");
3417 case CCValAssign::Indirect:
3418 // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully
3419 // support this.
3420 return false;
3421 }
3422
3423 if (VA.isRegLoc()) {
3424 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3425 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: VA.getLocReg()).addReg(RegNo: ArgReg);
3426 OutRegs.push_back(Elt: VA.getLocReg());
3427 } else {
3428 assert(VA.isMemLoc() && "Unknown value location!");
3429
3430 // Don't emit stores for undef values.
3431 if (isa<UndefValue>(Val: ArgVal))
3432 continue;
3433
3434 unsigned LocMemOffset = VA.getLocMemOffset();
3435 X86AddressMode AM;
3436 AM.Base.Reg = RegInfo->getStackRegister();
3437 AM.Disp = LocMemOffset;
3438 ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];
3439 Align Alignment = DL.getABITypeAlign(Ty: ArgVal->getType());
3440 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3441 PtrInfo: MachinePointerInfo::getStack(MF&: *FuncInfo.MF, Offset: LocMemOffset),
3442 F: MachineMemOperand::MOStore, Size: ArgVT.getStoreSize(), BaseAlignment: Alignment);
3443 if (Flags.isByVal()) {
3444 X86AddressMode SrcAM;
3445 SrcAM.Base.Reg = ArgReg;
3446 if (!TryEmitSmallMemcpy(DestAM: AM, SrcAM, Len: Flags.getByValSize()))
3447 return false;
3448 } else if (isa<ConstantInt>(Val: ArgVal) || isa<ConstantPointerNull>(Val: ArgVal)) {
3449 // If this is a really simple value, emit this with the Value* version
3450 // of X86FastEmitStore. If it isn't simple, we don't want to do this,
3451 // as it can cause us to reevaluate the argument.
3452 if (!X86FastEmitStore(VT: ArgVT, Val: ArgVal, AM, MMO))
3453 return false;
3454 } else {
3455 if (!X86FastEmitStore(VT: ArgVT, ValReg: ArgReg, AM, MMO))
3456 return false;
3457 }
3458 }
3459 }
3460
3461 // ELF / PIC requires GOT in the EBX register before function calls via PLT
3462 // GOT pointer.
3463 if (Subtarget->isPICStyleGOT()) {
3464 Register Base = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
3465 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3466 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::EBX).addReg(RegNo: Base);
3467 }
3468
3469 if (Is64Bit && IsVarArg && !IsWin64) {
3470 // From AMD64 ABI document:
3471 // For calls that may call functions that use varargs or stdargs
3472 // (prototype-less calls or calls to functions containing ellipsis (...) in
3473 // the declaration) %al is used as hidden argument to specify the number
3474 // of SSE registers used. The contents of %al do not need to match exactly
3475 // the number of registers, but must be an ubound on the number of SSE
3476 // registers used and is in the range 0 - 8 inclusive.
3477
3478 // Count the number of XMM registers allocated.
3479 static const MCPhysReg XMMArgRegs[] = {
3480 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3481 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3482 };
3483 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(Regs: XMMArgRegs);
3484 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3485 && "SSE registers cannot be used when SSE is disabled");
3486 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV8ri),
3487 DestReg: X86::AL).addImm(Val: NumXMMRegs);
3488 }
3489
3490 // Materialize callee address in a register. FIXME: GV address can be
3491 // handled with a CALLpcrel32 instead.
3492 X86AddressMode CalleeAM;
3493 if (!X86SelectCallAddress(V: Callee, AM&: CalleeAM))
3494 return false;
3495
3496 Register CalleeOp;
3497 const GlobalValue *GV = nullptr;
3498 if (CalleeAM.GV != nullptr) {
3499 GV = CalleeAM.GV;
3500 } else if (CalleeAM.Base.Reg) {
3501 CalleeOp = CalleeAM.Base.Reg;
3502 } else
3503 return false;
3504
3505 // Issue the call.
3506 MachineInstrBuilder MIB;
3507 if (CalleeOp) {
3508 // Register-indirect call.
3509 unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
3510 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: CallOpc))
3511 .addReg(RegNo: CalleeOp);
3512 } else {
3513 // Direct call.
3514 assert(GV && "Not a direct call");
3515 // See if we need any target-specific flags on the GV operand.
3516 unsigned char OpFlags = Subtarget->classifyGlobalFunctionReference(GV);
3517 if (OpFlags == X86II::MO_PLT && !Is64Bit &&
3518 TM.getRelocationModel() == Reloc::Static && isa<Function>(Val: GV) &&
3519 cast<Function>(Val: GV)->isIntrinsic())
3520 OpFlags = X86II::MO_NO_FLAG;
3521
3522 // This will be a direct call, or an indirect call through memory for
3523 // NonLazyBind calls or dllimport calls.
3524 bool NeedLoad = OpFlags == X86II::MO_DLLIMPORT ||
3525 OpFlags == X86II::MO_GOTPCREL ||
3526 OpFlags == X86II::MO_GOTPCREL_NORELAX ||
3527 OpFlags == X86II::MO_COFFSTUB;
3528 unsigned CallOpc = NeedLoad
3529 ? (Is64Bit ? X86::CALL64m : X86::CALL32m)
3530 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32);
3531
3532 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: CallOpc));
3533 if (NeedLoad)
3534 MIB.addReg(RegNo: Is64Bit ? X86::RIP : X86::NoRegister).addImm(Val: 1).addReg(RegNo: 0);
3535 if (Symbol)
3536 MIB.addSym(Sym: Symbol, TargetFlags: OpFlags);
3537 else
3538 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
3539 if (NeedLoad)
3540 MIB.addReg(RegNo: 0);
3541 }
3542
3543 // Add a register mask operand representing the call-preserved registers.
3544 // Proper defs for return values will be added by setPhysRegsDeadExcept().
3545 MIB.addRegMask(Mask: TRI.getCallPreservedMask(MF: *FuncInfo.MF, CC));
3546
3547 // Add an implicit use GOT pointer in EBX.
3548 if (Subtarget->isPICStyleGOT())
3549 MIB.addReg(RegNo: X86::EBX, Flags: RegState::Implicit);
3550
3551 if (Is64Bit && IsVarArg && !IsWin64)
3552 MIB.addReg(RegNo: X86::AL, Flags: RegState::Implicit);
3553
3554 // Add implicit physical register uses to the call.
3555 for (auto Reg : OutRegs)
3556 MIB.addReg(RegNo: Reg, Flags: RegState::Implicit);
3557
3558 // Issue CALLSEQ_END
3559 unsigned NumBytesForCalleeToPop =
3560 X86::isCalleePop(CallingConv: CC, is64Bit: Subtarget->is64Bit(), IsVarArg,
3561 GuaranteeTCO: TM.Options.GuaranteedTailCallOpt)
3562 ? NumBytes // Callee pops everything.
3563 : computeBytesPoppedByCalleeForSRet(Subtarget, CC, CB: CLI.CB);
3564 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
3565 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackUp))
3566 .addImm(Val: NumBytes).addImm(Val: NumBytesForCalleeToPop);
3567
3568 // Now handle call return values.
3569 SmallVector<CCValAssign, 16> RVLocs;
3570 CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,
3571 CLI.RetTy->getContext());
3572 CCRetInfo.AnalyzeCallResult(Ins, Fn: RetCC_X86);
3573
3574 // Copy all of the result registers out of their specified physreg.
3575 Register ResultReg = FuncInfo.CreateRegs(Ty: CLI.RetTy);
3576 for (unsigned i = 0; i != RVLocs.size(); ++i) {
3577 CCValAssign &VA = RVLocs[i];
3578 EVT CopyVT = VA.getValVT();
3579 Register CopyReg = ResultReg + i;
3580 Register SrcReg = VA.getLocReg();
3581
3582 // If this is x86-64, and we disabled SSE, we can't return FP values
3583 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
3584 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
3585 report_fatal_error(reason: "SSE register return with SSE disabled");
3586 }
3587
3588 // If we prefer to use the value in xmm registers, copy it out as f80 and
3589 // use a truncate to move it from fp stack reg to xmm reg.
3590 if ((SrcReg == X86::FP0 || SrcReg == X86::FP1) &&
3591 isScalarFPTypeInSSEReg(VT: VA.getValVT())) {
3592 CopyVT = MVT::f80;
3593 CopyReg = createResultReg(RC: &X86::RFP80RegClass);
3594 }
3595
3596 // Copy out the result.
3597 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3598 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CopyReg).addReg(RegNo: SrcReg);
3599 InRegs.push_back(Elt: VA.getLocReg());
3600
3601 // Round the f80 to the right size, which also moves it to the appropriate
3602 // xmm register. This is accomplished by storing the f80 value in memory
3603 // and then loading it back.
3604 if (CopyVT != VA.getValVT()) {
3605 EVT ResVT = VA.getValVT();
3606 unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
3607 unsigned MemSize = ResVT.getSizeInBits()/8;
3608 int FI = MFI.CreateStackObject(Size: MemSize, Alignment: Align(MemSize), isSpillSlot: false);
3609 addFrameReference(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3610 MCID: TII.get(Opcode: Opc)), FI)
3611 .addReg(RegNo: CopyReg);
3612 Opc = ResVT == MVT::f32 ? X86::MOVSSrm_alt : X86::MOVSDrm_alt;
3613 addFrameReference(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3614 MCID: TII.get(Opcode: Opc), DestReg: ResultReg + i), FI);
3615 }
3616 }
3617
3618 CLI.ResultReg = ResultReg;
3619 CLI.NumResultRegs = RVLocs.size();
3620 CLI.Call = MIB;
3621
3622 // Add call site info for call graph section.
3623 if (TM.Options.EmitCallGraphSection && CB && CB->isIndirectCall()) {
3624 MachineFunction::CallSiteInfo CSInfo(*CB);
3625 MF->addCallSiteInfo(CallI: CLI.Call, CallInfo: std::move(CSInfo));
3626 }
3627
3628 return true;
3629}
3630
3631bool
3632X86FastISel::fastSelectInstruction(const Instruction *I) {
3633 switch (I->getOpcode()) {
3634 default: break;
3635 case Instruction::Load:
3636 return X86SelectLoad(I);
3637 case Instruction::Store:
3638 return X86SelectStore(I);
3639 case Instruction::Ret:
3640 return X86SelectRet(I);
3641 case Instruction::ICmp:
3642 case Instruction::FCmp:
3643 return X86SelectCmp(I);
3644 case Instruction::ZExt:
3645 return X86SelectZExt(I);
3646 case Instruction::SExt:
3647 return X86SelectSExt(I);
3648 case Instruction::Br:
3649 return X86SelectBranch(I);
3650 case Instruction::LShr:
3651 case Instruction::AShr:
3652 case Instruction::Shl:
3653 return X86SelectShift(I);
3654 case Instruction::SDiv:
3655 case Instruction::UDiv:
3656 case Instruction::SRem:
3657 case Instruction::URem:
3658 return X86SelectDivRem(I);
3659 case Instruction::Select:
3660 return X86SelectSelect(I);
3661 case Instruction::Trunc:
3662 return X86SelectTrunc(I);
3663 case Instruction::FPExt:
3664 return X86SelectFPExt(I);
3665 case Instruction::FPTrunc:
3666 return X86SelectFPTrunc(I);
3667 case Instruction::SIToFP:
3668 return X86SelectSIToFP(I);
3669 case Instruction::UIToFP:
3670 return X86SelectUIToFP(I);
3671 case Instruction::IntToPtr: // Deliberate fall-through.
3672 case Instruction::PtrToInt: {
3673 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
3674 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
3675 if (DstVT.bitsGT(VT: SrcVT))
3676 return X86SelectZExt(I);
3677 if (DstVT.bitsLT(VT: SrcVT))
3678 return X86SelectTrunc(I);
3679 Register Reg = getRegForValue(V: I->getOperand(i: 0));
3680 if (!Reg)
3681 return false;
3682 updateValueMap(I, Reg);
3683 return true;
3684 }
3685 case Instruction::BitCast:
3686 return X86SelectBitCast(I);
3687 }
3688
3689 return false;
3690}
3691
3692Register X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
3693 if (VT > MVT::i64)
3694 return Register();
3695
3696 uint64_t Imm = CI->getZExtValue();
3697 if (Imm == 0) {
3698 Register SrcReg = fastEmitInst_(MachineInstOpcode: X86::MOV32r0, RC: &X86::GR32RegClass);
3699 switch (VT.SimpleTy) {
3700 default: llvm_unreachable("Unexpected value type");
3701 case MVT::i1:
3702 case MVT::i8:
3703 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: SrcReg, Idx: X86::sub_8bit);
3704 case MVT::i16:
3705 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0: SrcReg, Idx: X86::sub_16bit);
3706 case MVT::i32:
3707 return SrcReg;
3708 case MVT::i64: {
3709 Register ResultReg = createResultReg(RC: &X86::GR64RegClass);
3710 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3711 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: ResultReg)
3712 .addReg(RegNo: SrcReg)
3713 .addImm(Val: X86::sub_32bit);
3714 return ResultReg;
3715 }
3716 }
3717 }
3718
3719 unsigned Opc = 0;
3720 switch (VT.SimpleTy) {
3721 default: llvm_unreachable("Unexpected value type");
3722 case MVT::i1:
3723 VT = MVT::i8;
3724 [[fallthrough]];
3725 case MVT::i8: Opc = X86::MOV8ri; break;
3726 case MVT::i16: Opc = X86::MOV16ri; break;
3727 case MVT::i32: Opc = X86::MOV32ri; break;
3728 case MVT::i64: {
3729 if (isUInt<32>(x: Imm))
3730 Opc = X86::MOV32ri64;
3731 else if (isInt<32>(x: Imm))
3732 Opc = X86::MOV64ri32;
3733 else
3734 Opc = X86::MOV64ri;
3735 break;
3736 }
3737 }
3738 return fastEmitInst_i(MachineInstOpcode: Opc, RC: TLI.getRegClassFor(VT), Imm);
3739}
3740
3741Register X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
3742 if (CFP->isNullValue())
3743 return fastMaterializeFloatZero(CF: CFP);
3744
3745 // Can't handle alternate code models yet.
3746 CodeModel::Model CM = TM.getCodeModel();
3747 if (CM != CodeModel::Small && CM != CodeModel::Medium &&
3748 CM != CodeModel::Large)
3749 return Register();
3750
3751 // Get opcode and regclass of the output for the given load instruction.
3752 unsigned Opc = 0;
3753 bool HasSSE1 = Subtarget->hasSSE1();
3754 bool HasSSE2 = Subtarget->hasSSE2();
3755 bool HasAVX = Subtarget->hasAVX();
3756 bool HasAVX512 = Subtarget->hasAVX512();
3757 switch (VT.SimpleTy) {
3758 default:
3759 return Register();
3760 case MVT::f32:
3761 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
3762 : HasAVX ? X86::VMOVSSrm_alt
3763 : HasSSE1 ? X86::MOVSSrm_alt
3764 : X86::LD_Fp32m;
3765 break;
3766 case MVT::f64:
3767 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
3768 : HasAVX ? X86::VMOVSDrm_alt
3769 : HasSSE2 ? X86::MOVSDrm_alt
3770 : X86::LD_Fp64m;
3771 break;
3772 case MVT::f80:
3773 // No f80 support yet.
3774 return Register();
3775 }
3776
3777 // MachineConstantPool wants an explicit alignment.
3778 Align Alignment = DL.getPrefTypeAlign(Ty: CFP->getType());
3779
3780 // x86-32 PIC requires a PIC base register for constant pools.
3781 Register PICBase;
3782 unsigned char OpFlag = Subtarget->classifyLocalReference(GV: nullptr);
3783 if (OpFlag == X86II::MO_PIC_BASE_OFFSET)
3784 PICBase = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
3785 else if (OpFlag == X86II::MO_GOTOFF)
3786 PICBase = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
3787 else if (Subtarget->is64Bit() && TM.getCodeModel() != CodeModel::Large)
3788 PICBase = X86::RIP;
3789
3790 // Create the load from the constant pool.
3791 unsigned CPI = MCP.getConstantPoolIndex(C: CFP, Alignment);
3792 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: VT.SimpleTy));
3793
3794 // Large code model only applies to 64-bit mode.
3795 if (Subtarget->is64Bit() && CM == CodeModel::Large) {
3796 Register AddrReg = createResultReg(RC: &X86::GR64RegClass);
3797 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV64ri),
3798 DestReg: AddrReg)
3799 .addConstantPoolIndex(Idx: CPI, Offset: 0, TargetFlags: OpFlag);
3800 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3801 MCID: TII.get(Opcode: Opc), DestReg: ResultReg);
3802 addRegReg(MIB, Reg1: AddrReg, isKill1: false, SubReg1: X86::NoSubRegister, Reg2: PICBase, isKill2: false,
3803 SubReg2: X86::NoSubRegister);
3804 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3805 PtrInfo: MachinePointerInfo::getConstantPool(MF&: *FuncInfo.MF),
3806 F: MachineMemOperand::MOLoad, Size: DL.getPointerSize(), BaseAlignment: Alignment);
3807 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
3808 return ResultReg;
3809 }
3810
3811 addConstantPoolReference(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3812 MCID: TII.get(Opcode: Opc), DestReg: ResultReg),
3813 CPI, GlobalBaseReg: PICBase, OpFlags: OpFlag);
3814 return ResultReg;
3815}
3816
3817Register X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
3818 // Can't handle large GlobalValues yet.
3819 if (TM.getCodeModel() != CodeModel::Small &&
3820 TM.getCodeModel() != CodeModel::Medium)
3821 return Register();
3822 if (TM.isLargeGlobalValue(GV))
3823 return Register();
3824
3825 // Materialize addresses with LEA/MOV instructions.
3826 X86AddressMode AM;
3827 if (X86SelectAddress(V: GV, AM)) {
3828 // If the expression is just a basereg, then we're done, otherwise we need
3829 // to emit an LEA.
3830 if (AM.BaseType == X86AddressMode::RegBase &&
3831 AM.IndexReg == 0 && AM.Disp == 0 && AM.GV == nullptr)
3832 return AM.Base.Reg;
3833
3834 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3835 if (TM.getRelocationModel() == Reloc::Static &&
3836 TLI.getPointerTy(DL) == MVT::i64) {
3837 // The displacement code could be more than 32 bits away so we need to use
3838 // an instruction with a 64 bit immediate
3839 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV64ri),
3840 DestReg: ResultReg)
3841 .addGlobalAddress(GV);
3842 } else {
3843 unsigned Opc =
3844 TLI.getPointerTy(DL) == MVT::i32
3845 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3846 : X86::LEA64r;
3847 addFullAddress(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3848 MCID: TII.get(Opcode: Opc), DestReg: ResultReg), AM);
3849 }
3850 return ResultReg;
3851 }
3852 return Register();
3853}
3854
3855Register X86FastISel::fastMaterializeConstant(const Constant *C) {
3856 EVT CEVT = TLI.getValueType(DL, Ty: C->getType(), AllowUnknown: true);
3857
3858 // Only handle simple types.
3859 if (!CEVT.isSimple())
3860 return Register();
3861 MVT VT = CEVT.getSimpleVT();
3862
3863 if (const auto *CI = dyn_cast<ConstantInt>(Val: C))
3864 return X86MaterializeInt(CI, VT);
3865 if (const auto *CFP = dyn_cast<ConstantFP>(Val: C))
3866 return X86MaterializeFP(CFP, VT);
3867 if (const auto *GV = dyn_cast<GlobalValue>(Val: C))
3868 return X86MaterializeGV(GV, VT);
3869 if (isa<UndefValue>(Val: C)) {
3870 unsigned Opc = 0;
3871 switch (VT.SimpleTy) {
3872 default:
3873 break;
3874 case MVT::f32:
3875 if (!Subtarget->hasSSE1())
3876 Opc = X86::LD_Fp032;
3877 break;
3878 case MVT::f64:
3879 if (!Subtarget->hasSSE2())
3880 Opc = X86::LD_Fp064;
3881 break;
3882 case MVT::f80:
3883 Opc = X86::LD_Fp080;
3884 break;
3885 }
3886
3887 if (Opc) {
3888 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3889 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc),
3890 DestReg: ResultReg);
3891 return ResultReg;
3892 }
3893 }
3894
3895 return Register();
3896}
3897
3898Register X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {
3899 // Fail on dynamic allocas. At this point, getRegForValue has already
3900 // checked its CSE maps, so if we're here trying to handle a dynamic
3901 // alloca, we're not going to succeed. X86SelectAddress has a
3902 // check for dynamic allocas, because it's called directly from
3903 // various places, but targetMaterializeAlloca also needs a check
3904 // in order to avoid recursion between getRegForValue,
3905 // X86SelectAddrss, and targetMaterializeAlloca.
3906 if (!FuncInfo.StaticAllocaMap.count(Val: C))
3907 return Register();
3908 assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?");
3909
3910 X86AddressMode AM;
3911 if (!X86SelectAddress(V: C, AM))
3912 return Register();
3913 unsigned Opc =
3914 TLI.getPointerTy(DL) == MVT::i32
3915 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3916 : X86::LEA64r;
3917 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: TLI.getPointerTy(DL));
3918 Register ResultReg = createResultReg(RC);
3919 addFullAddress(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3920 MCID: TII.get(Opcode: Opc), DestReg: ResultReg), AM);
3921 return ResultReg;
3922}
3923
3924Register X86FastISel::fastMaterializeFloatZero(const ConstantFP *CF) {
3925 MVT VT;
3926 if (!isTypeLegal(Ty: CF->getType(), VT))
3927 return Register();
3928
3929 // Get opcode and regclass for the given zero.
3930 bool HasSSE1 = Subtarget->hasSSE1();
3931 bool HasSSE2 = Subtarget->hasSSE2();
3932 bool HasAVX512 = Subtarget->hasAVX512();
3933 unsigned Opc = 0;
3934 switch (VT.SimpleTy) {
3935 default: return 0;
3936 case MVT::f16:
3937 Opc = HasAVX512 ? X86::AVX512_FsFLD0SH : X86::FsFLD0SH;
3938 break;
3939 case MVT::f32:
3940 Opc = HasAVX512 ? X86::AVX512_FsFLD0SS
3941 : HasSSE1 ? X86::FsFLD0SS
3942 : X86::LD_Fp032;
3943 break;
3944 case MVT::f64:
3945 Opc = HasAVX512 ? X86::AVX512_FsFLD0SD
3946 : HasSSE2 ? X86::FsFLD0SD
3947 : X86::LD_Fp064;
3948 break;
3949 case MVT::f80:
3950 // No f80 support yet.
3951 return Register();
3952 }
3953
3954 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3955 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: ResultReg);
3956 return ResultReg;
3957}
3958
3959bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
3960 const LoadInst *LI) {
3961 const Value *Ptr = LI->getPointerOperand();
3962 X86AddressMode AM;
3963 if (!X86SelectAddress(V: Ptr, AM))
3964 return false;
3965
3966 const X86InstrInfo &XII = (const X86InstrInfo &)TII;
3967
3968 unsigned Size = DL.getTypeAllocSize(Ty: LI->getType());
3969
3970 SmallVector<MachineOperand, 8> AddrOps;
3971 AM.getFullAddress(MO&: AddrOps);
3972
3973 MachineInstr *Result = XII.foldMemoryOperandImpl(
3974 MF&: *FuncInfo.MF, MI&: *MI, OpNum: OpNo, MOs: AddrOps, InsertPt: FuncInfo.InsertPt, Size, Alignment: LI->getAlign(),
3975 /*AllowCommute=*/true);
3976 if (!Result)
3977 return false;
3978
3979 // The index register could be in the wrong register class. Unfortunately,
3980 // foldMemoryOperandImpl could have commuted the instruction so its not enough
3981 // to just look at OpNo + the offset to the index reg. We actually need to
3982 // scan the instruction to find the index reg and see if its the correct reg
3983 // class.
3984 unsigned OperandNo = 0;
3985 for (MachineInstr::mop_iterator I = Result->operands_begin(),
3986 E = Result->operands_end(); I != E; ++I, ++OperandNo) {
3987 MachineOperand &MO = *I;
3988 if (!MO.isReg() || MO.isDef() || MO.getReg() != AM.IndexReg)
3989 continue;
3990 // Found the index reg, now try to rewrite it.
3991 Register IndexReg = constrainOperandRegClass(II: Result->getDesc(),
3992 Op: MO.getReg(), OpNum: OperandNo);
3993 if (IndexReg == MO.getReg())
3994 continue;
3995 MO.setReg(IndexReg);
3996 }
3997
3998 if (MI->isCall())
3999 FuncInfo.MF->moveAdditionalCallInfo(Old: MI, New: Result);
4000 Result->addMemOperand(MF&: *FuncInfo.MF, MO: createMachineMemOperandFor(I: LI));
4001 Result->cloneInstrSymbols(MF&: *FuncInfo.MF, MI: *MI);
4002 MachineBasicBlock::iterator I(MI);
4003 removeDeadCode(I, E: std::next(x: I));
4004 return true;
4005}
4006
4007Register X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode,
4008 const TargetRegisterClass *RC,
4009 Register Op0, Register Op1,
4010 Register Op2, Register Op3) {
4011 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
4012
4013 Register ResultReg = createResultReg(RC);
4014 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
4015 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
4016 Op2 = constrainOperandRegClass(II, Op: Op2, OpNum: II.getNumDefs() + 2);
4017 Op3 = constrainOperandRegClass(II, Op: Op3, OpNum: II.getNumDefs() + 3);
4018
4019 if (II.getNumDefs() >= 1)
4020 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
4021 .addReg(RegNo: Op0)
4022 .addReg(RegNo: Op1)
4023 .addReg(RegNo: Op2)
4024 .addReg(RegNo: Op3);
4025 else {
4026 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
4027 .addReg(RegNo: Op0)
4028 .addReg(RegNo: Op1)
4029 .addReg(RegNo: Op2)
4030 .addReg(RegNo: Op3);
4031 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
4032 DestReg: ResultReg)
4033 .addReg(RegNo: II.implicit_defs()[0]);
4034 }
4035 return ResultReg;
4036}
4037
4038namespace llvm {
4039FastISel *X86::createFastISel(FunctionLoweringInfo &funcInfo,
4040 const TargetLibraryInfo *libInfo,
4041 const LibcallLoweringInfo *libcallLowering) {
4042 return new X86FastISel(funcInfo, libInfo, libcallLowering);
4043}
4044}
4045