1//===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the X86-specific support for the FastISel class. Much
10// of the target-specific code is generated by tablegen in the file
11// X86GenFastISel.inc, which is #included here.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86CallingConv.h"
17#include "X86InstrBuilder.h"
18#include "X86InstrInfo.h"
19#include "X86MachineFunctionInfo.h"
20#include "X86RegisterInfo.h"
21#include "X86Subtarget.h"
22#include "X86TargetMachine.h"
23#include "llvm/Analysis/BranchProbabilityInfo.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/CodeGen/FastISel.h"
26#include "llvm/CodeGen/FunctionLoweringInfo.h"
27#include "llvm/CodeGen/MachineConstantPool.h"
28#include "llvm/CodeGen/MachineFrameInfo.h"
29#include "llvm/CodeGen/MachineRegisterInfo.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/DebugInfo.h"
32#include "llvm/IR/DerivedTypes.h"
33#include "llvm/IR/GetElementPtrTypeIterator.h"
34#include "llvm/IR/GlobalVariable.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/IntrinsicInst.h"
37#include "llvm/IR/IntrinsicsX86.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/MC/MCAsmInfo.h"
41#include "llvm/MC/MCSymbol.h"
42#include "llvm/Support/ErrorHandling.h"
43#include "llvm/Target/TargetOptions.h"
44using namespace llvm;
45
46namespace {
47
48class X86FastISel final : public FastISel {
49 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
50 /// make the right decision when generating code for different targets.
51 const X86Subtarget *Subtarget;
52
53public:
54 explicit X86FastISel(FunctionLoweringInfo &funcInfo,
55 const TargetLibraryInfo *libInfo,
56 const LibcallLoweringInfo *libcallLowering)
57 : FastISel(funcInfo, libInfo, libcallLowering) {
58 Subtarget = &funcInfo.MF->getSubtarget<X86Subtarget>();
59 }
60
61 bool fastSelectInstruction(const Instruction *I) override;
62
63 /// The specified machine instr operand is a vreg, and that
64 /// vreg is being provided by the specified load instruction. If possible,
65 /// try to fold the load as an operand to the instruction, returning true if
66 /// possible.
67 bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
68 const LoadInst *LI) override;
69
70 bool fastLowerArguments() override;
71 bool fastLowerCall(CallLoweringInfo &CLI) override;
72 bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;
73
74#include "X86GenFastISel.inc"
75
76private:
77 bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT,
78 const DebugLoc &DL);
79
80 bool X86FastEmitLoad(MVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
81 Register &ResultReg, unsigned Alignment = 1);
82
83 bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
84 MachineMemOperand *MMO = nullptr, bool Aligned = false);
85 bool X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
86 MachineMemOperand *MMO = nullptr, bool Aligned = false);
87
88 bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, Register Src, EVT SrcVT,
89 Register &ResultReg);
90
91 bool X86SelectAddress(const Value *V, X86AddressMode &AM);
92 bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);
93
94 bool X86SelectLoad(const Instruction *I);
95
96 bool X86SelectStore(const Instruction *I);
97
98 bool X86SelectRet(const Instruction *I);
99
100 bool X86SelectCmp(const Instruction *I);
101
102 bool X86SelectZExt(const Instruction *I);
103
104 bool X86SelectSExt(const Instruction *I);
105
106 bool X86SelectBranch(const Instruction *I);
107
108 bool X86SelectShift(const Instruction *I);
109
110 bool X86SelectDivRem(const Instruction *I);
111
112 bool X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I);
113
114 bool X86FastEmitSSESelect(MVT RetVT, const Instruction *I);
115
116 bool X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I);
117
118 bool X86SelectSelect(const Instruction *I);
119
120 bool X86SelectTrunc(const Instruction *I);
121
122 bool X86SelectFPExtOrFPTrunc(const Instruction *I, unsigned Opc,
123 const TargetRegisterClass *RC);
124
125 bool X86SelectFPExt(const Instruction *I);
126 bool X86SelectFPTrunc(const Instruction *I);
127 bool X86SelectSIToFP(const Instruction *I);
128 bool X86SelectUIToFP(const Instruction *I);
129 bool X86SelectIntToFP(const Instruction *I, bool IsSigned);
130 bool X86SelectBitCast(const Instruction *I);
131
132 const X86InstrInfo *getInstrInfo() const {
133 return Subtarget->getInstrInfo();
134 }
135 const X86TargetMachine *getTargetMachine() const {
136 return static_cast<const X86TargetMachine *>(&TM);
137 }
138
139 bool handleConstantAddresses(const Value *V, X86AddressMode &AM);
140
141 Register emitMOV32r0();
142
143 Register X86MaterializeInt(const ConstantInt *CI, MVT VT);
144 Register X86MaterializeFP(const ConstantFP *CFP, MVT VT);
145 Register X86MaterializeGV(const GlobalValue *GV, MVT VT);
146 Register fastMaterializeConstant(const Constant *C) override;
147
148 Register fastMaterializeAlloca(const AllocaInst *C) override;
149
150 Register fastMaterializeFloatZero(const ConstantFP *CF) override;
151
152 /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is
153 /// computed in an SSE register, not on the X87 floating point stack.
154 bool isScalarFPTypeInSSEReg(EVT VT) const {
155 return (VT == MVT::f64 && Subtarget->hasSSE2()) ||
156 (VT == MVT::f32 && Subtarget->hasSSE1()) || VT == MVT::f16;
157 }
158
159 bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);
160
161 bool IsMemcpySmall(uint64_t Len);
162
163 bool TryEmitSmallMemcpy(X86AddressMode DestAM,
164 X86AddressMode SrcAM, uint64_t Len);
165
166 bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
167 const Value *Cond);
168
169 const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
170 X86AddressMode &AM);
171
172 Register fastEmitInst_rrrr(unsigned MachineInstOpcode,
173 const TargetRegisterClass *RC, Register Op0,
174 Register Op1, Register Op2, Register Op3);
175};
176
177} // end anonymous namespace.
178
179static std::pair<unsigned, bool>
180getX86SSEConditionCode(CmpInst::Predicate Predicate) {
181 unsigned CC;
182 bool NeedSwap = false;
183
184 // SSE Condition code mapping:
185 // 0 - EQ
186 // 1 - LT
187 // 2 - LE
188 // 3 - UNORD
189 // 4 - NEQ
190 // 5 - NLT
191 // 6 - NLE
192 // 7 - ORD
193 switch (Predicate) {
194 default: llvm_unreachable("Unexpected predicate");
195 case CmpInst::FCMP_OEQ: CC = 0; break;
196 case CmpInst::FCMP_OGT: NeedSwap = true; [[fallthrough]];
197 case CmpInst::FCMP_OLT: CC = 1; break;
198 case CmpInst::FCMP_OGE: NeedSwap = true; [[fallthrough]];
199 case CmpInst::FCMP_OLE: CC = 2; break;
200 case CmpInst::FCMP_UNO: CC = 3; break;
201 case CmpInst::FCMP_UNE: CC = 4; break;
202 case CmpInst::FCMP_ULE: NeedSwap = true; [[fallthrough]];
203 case CmpInst::FCMP_UGE: CC = 5; break;
204 case CmpInst::FCMP_ULT: NeedSwap = true; [[fallthrough]];
205 case CmpInst::FCMP_UGT: CC = 6; break;
206 case CmpInst::FCMP_ORD: CC = 7; break;
207 case CmpInst::FCMP_UEQ: CC = 8; break;
208 case CmpInst::FCMP_ONE: CC = 12; break;
209 }
210
211 return std::make_pair(x&: CC, y&: NeedSwap);
212}
213
214/// Adds a complex addressing mode to the given machine instr builder.
215/// Note, this will constrain the index register. If its not possible to
216/// constrain the given index register, then a new one will be created. The
217/// IndexReg field of the addressing mode will be updated to match in this case.
218const MachineInstrBuilder &
219X86FastISel::addFullAddress(const MachineInstrBuilder &MIB,
220 X86AddressMode &AM) {
221 // First constrain the index register. It needs to be a GR64_NOSP.
222 AM.IndexReg = constrainOperandRegClass(II: MIB->getDesc(), Op: AM.IndexReg,
223 OpNum: MIB->getNumOperands() +
224 X86::AddrIndexReg);
225 return ::addFullAddress(MIB, AM);
226}
227
228/// Check if it is possible to fold the condition from the XALU intrinsic
229/// into the user. The condition code will only be updated on success.
230bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
231 const Value *Cond) {
232 if (!isa<ExtractValueInst>(Val: Cond))
233 return false;
234
235 const auto *EV = cast<ExtractValueInst>(Val: Cond);
236 if (!isa<IntrinsicInst>(Val: EV->getAggregateOperand()))
237 return false;
238
239 const auto *II = cast<IntrinsicInst>(Val: EV->getAggregateOperand());
240 MVT RetVT;
241 const Function *Callee = II->getCalledFunction();
242 Type *RetTy =
243 cast<StructType>(Val: Callee->getReturnType())->getTypeAtIndex(N: 0U);
244 if (!isTypeLegal(Ty: RetTy, VT&: RetVT))
245 return false;
246
247 if (RetVT != MVT::i32 && RetVT != MVT::i64)
248 return false;
249
250 X86::CondCode TmpCC;
251 switch (II->getIntrinsicID()) {
252 default: return false;
253 case Intrinsic::sadd_with_overflow:
254 case Intrinsic::ssub_with_overflow: TmpCC = X86::COND_O; break;
255 case Intrinsic::smul_with_overflow:
256 case Intrinsic::umul_with_overflow:
257 case Intrinsic::uadd_with_overflow:
258 case Intrinsic::usub_with_overflow: TmpCC = X86::COND_B; break;
259 }
260
261 // Check if both instructions are in the same basic block.
262 if (II->getParent() != I->getParent())
263 return false;
264
265 // Make sure nothing is in the way
266 BasicBlock::const_iterator Start(I);
267 BasicBlock::const_iterator End(II);
268 for (auto Itr = std::prev(x: Start); Itr != End; --Itr) {
269 // We only expect extractvalue instructions between the intrinsic and the
270 // instruction to be selected.
271 if (!isa<ExtractValueInst>(Val: Itr))
272 return false;
273
274 // Check that the extractvalue operand comes from the intrinsic.
275 const auto *EVI = cast<ExtractValueInst>(Val&: Itr);
276 if (EVI->getAggregateOperand() != II)
277 return false;
278 }
279
280 // Make sure no potentially eflags clobbering phi moves can be inserted in
281 // between.
282 auto HasPhis = [](const BasicBlock *Succ) { return !Succ->phis().empty(); };
283 if (I->isTerminator() && llvm::any_of(Range: successors(I), P: HasPhis))
284 return false;
285
286 // Make sure there are no potentially eflags clobbering constant
287 // materializations in between.
288 if (llvm::any_of(Range: I->operands(), P: [](Value *V) { return isa<Constant>(Val: V); }))
289 return false;
290
291 CC = TmpCC;
292 return true;
293}
294
295bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
296 EVT evt = TLI.getValueType(DL, Ty, /*AllowUnknown=*/true);
297 if (evt == MVT::Other || !evt.isSimple())
298 // Unhandled type. Halt "fast" selection and bail.
299 return false;
300
301 VT = evt.getSimpleVT();
302 // For now, require SSE/SSE2 for performing floating-point operations,
303 // since x87 requires additional work.
304 if (VT == MVT::f64 && !Subtarget->hasSSE2())
305 return false;
306 if (VT == MVT::f32 && !Subtarget->hasSSE1())
307 return false;
308 // Similarly, no f80 support yet.
309 if (VT == MVT::f80)
310 return false;
311 // We only handle legal types. For example, on x86-32 the instruction
312 // selector contains all of the 64-bit instructions from x86-64,
313 // under the assumption that i64 won't be used if the target doesn't
314 // support it.
315 return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);
316}
317
318/// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
319/// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
320/// Return true and the result register by reference if it is possible.
321bool X86FastISel::X86FastEmitLoad(MVT VT, X86AddressMode &AM,
322 MachineMemOperand *MMO, Register &ResultReg,
323 unsigned Alignment) {
324 bool HasSSE1 = Subtarget->hasSSE1();
325 bool HasSSE2 = Subtarget->hasSSE2();
326 bool HasSSE41 = Subtarget->hasSSE41();
327 bool HasAVX = Subtarget->hasAVX();
328 bool HasAVX2 = Subtarget->hasAVX2();
329 bool HasAVX512 = Subtarget->hasAVX512();
330 bool HasVLX = Subtarget->hasVLX();
331 bool IsNonTemporal = MMO && MMO->isNonTemporal();
332
333 // Treat i1 loads the same as i8 loads. Masking will be done when storing.
334 if (VT == MVT::i1)
335 VT = MVT::i8;
336
337 // Get opcode and regclass of the output for the given load instruction.
338 unsigned Opc = 0;
339 switch (VT.SimpleTy) {
340 default: return false;
341 case MVT::i8:
342 Opc = X86::MOV8rm;
343 break;
344 case MVT::i16:
345 Opc = X86::MOV16rm;
346 break;
347 case MVT::i32:
348 Opc = X86::MOV32rm;
349 break;
350 case MVT::i64:
351 // Must be in x86-64 mode.
352 Opc = X86::MOV64rm;
353 break;
354 case MVT::f32:
355 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
356 : HasAVX ? X86::VMOVSSrm_alt
357 : HasSSE1 ? X86::MOVSSrm_alt
358 : X86::LD_Fp32m;
359 break;
360 case MVT::f64:
361 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
362 : HasAVX ? X86::VMOVSDrm_alt
363 : HasSSE2 ? X86::MOVSDrm_alt
364 : X86::LD_Fp64m;
365 break;
366 case MVT::f80:
367 // No f80 support yet.
368 return false;
369 case MVT::v4f32:
370 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
371 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
372 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
373 else if (Alignment >= 16)
374 Opc = HasVLX ? X86::VMOVAPSZ128rm :
375 HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm;
376 else
377 Opc = HasVLX ? X86::VMOVUPSZ128rm :
378 HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm;
379 break;
380 case MVT::v2f64:
381 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
382 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
383 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
384 else if (Alignment >= 16)
385 Opc = HasVLX ? X86::VMOVAPDZ128rm :
386 HasAVX ? X86::VMOVAPDrm : X86::MOVAPDrm;
387 else
388 Opc = HasVLX ? X86::VMOVUPDZ128rm :
389 HasAVX ? X86::VMOVUPDrm : X86::MOVUPDrm;
390 break;
391 case MVT::v4i32:
392 case MVT::v2i64:
393 case MVT::v8i16:
394 case MVT::v16i8:
395 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
396 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
397 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
398 else if (Alignment >= 16)
399 Opc = HasVLX ? X86::VMOVDQA64Z128rm :
400 HasAVX ? X86::VMOVDQArm : X86::MOVDQArm;
401 else
402 Opc = HasVLX ? X86::VMOVDQU64Z128rm :
403 HasAVX ? X86::VMOVDQUrm : X86::MOVDQUrm;
404 break;
405 case MVT::v8f32:
406 assert(HasAVX);
407 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
408 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
409 else if (IsNonTemporal && Alignment >= 16)
410 return false; // Force split for X86::VMOVNTDQArm
411 else if (Alignment >= 32)
412 Opc = HasVLX ? X86::VMOVAPSZ256rm : X86::VMOVAPSYrm;
413 else
414 Opc = HasVLX ? X86::VMOVUPSZ256rm : X86::VMOVUPSYrm;
415 break;
416 case MVT::v4f64:
417 assert(HasAVX);
418 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
419 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
420 else if (IsNonTemporal && Alignment >= 16)
421 return false; // Force split for X86::VMOVNTDQArm
422 else if (Alignment >= 32)
423 Opc = HasVLX ? X86::VMOVAPDZ256rm : X86::VMOVAPDYrm;
424 else
425 Opc = HasVLX ? X86::VMOVUPDZ256rm : X86::VMOVUPDYrm;
426 break;
427 case MVT::v8i32:
428 case MVT::v4i64:
429 case MVT::v16i16:
430 case MVT::v32i8:
431 assert(HasAVX);
432 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
433 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
434 else if (IsNonTemporal && Alignment >= 16)
435 return false; // Force split for X86::VMOVNTDQArm
436 else if (Alignment >= 32)
437 Opc = HasVLX ? X86::VMOVDQA64Z256rm : X86::VMOVDQAYrm;
438 else
439 Opc = HasVLX ? X86::VMOVDQU64Z256rm : X86::VMOVDQUYrm;
440 break;
441 case MVT::v16f32:
442 assert(HasAVX512);
443 if (IsNonTemporal && Alignment >= 64)
444 Opc = X86::VMOVNTDQAZrm;
445 else
446 Opc = (Alignment >= 64) ? X86::VMOVAPSZrm : X86::VMOVUPSZrm;
447 break;
448 case MVT::v8f64:
449 assert(HasAVX512);
450 if (IsNonTemporal && Alignment >= 64)
451 Opc = X86::VMOVNTDQAZrm;
452 else
453 Opc = (Alignment >= 64) ? X86::VMOVAPDZrm : X86::VMOVUPDZrm;
454 break;
455 case MVT::v8i64:
456 case MVT::v16i32:
457 case MVT::v32i16:
458 case MVT::v64i8:
459 assert(HasAVX512);
460 // Note: There are a lot more choices based on type with AVX-512, but
461 // there's really no advantage when the load isn't masked.
462 if (IsNonTemporal && Alignment >= 64)
463 Opc = X86::VMOVNTDQAZrm;
464 else
465 Opc = (Alignment >= 64) ? X86::VMOVDQA64Zrm : X86::VMOVDQU64Zrm;
466 break;
467 }
468
469 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
470
471 ResultReg = createResultReg(RC);
472 MachineInstrBuilder MIB =
473 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: ResultReg);
474 addFullAddress(MIB, AM);
475 if (MMO)
476 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
477 return true;
478}
479
480/// X86FastEmitStore - Emit a machine instruction to store a value Val of
481/// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
482/// and a displacement offset, or a GlobalAddress,
483/// i.e. V. Return true if it is possible.
484bool X86FastISel::X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
485 MachineMemOperand *MMO, bool Aligned) {
486 bool HasSSE1 = Subtarget->hasSSE1();
487 bool HasSSE2 = Subtarget->hasSSE2();
488 bool HasSSE4A = Subtarget->hasSSE4A();
489 bool HasAVX = Subtarget->hasAVX();
490 bool HasAVX512 = Subtarget->hasAVX512();
491 bool HasVLX = Subtarget->hasVLX();
492 bool IsNonTemporal = MMO && MMO->isNonTemporal();
493
494 // Get opcode and regclass of the output for the given store instruction.
495 unsigned Opc = 0;
496 switch (VT.getSimpleVT().SimpleTy) {
497 case MVT::f80: // No f80 support yet.
498 default: return false;
499 case MVT::i1: {
500 // Mask out all but lowest bit.
501 Register AndResult = createResultReg(RC: &X86::GR8RegClass);
502 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
503 MCID: TII.get(Opcode: X86::AND8ri), DestReg: AndResult)
504 .addReg(RegNo: ValReg).addImm(Val: 1);
505 ValReg = AndResult;
506 [[fallthrough]]; // handle i1 as i8.
507 }
508 case MVT::i8: Opc = X86::MOV8mr; break;
509 case MVT::i16: Opc = X86::MOV16mr; break;
510 case MVT::i32:
511 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTImr : X86::MOV32mr;
512 break;
513 case MVT::i64:
514 // Must be in x86-64 mode.
515 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTI_64mr : X86::MOV64mr;
516 break;
517 case MVT::f32:
518 if (HasSSE1) {
519 if (IsNonTemporal && HasSSE4A)
520 Opc = X86::MOVNTSS;
521 else
522 Opc = HasAVX512 ? X86::VMOVSSZmr :
523 HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
524 } else
525 Opc = X86::ST_Fp32m;
526 break;
527 case MVT::f64:
528 if (HasSSE2) {
529 if (IsNonTemporal && HasSSE4A)
530 Opc = X86::MOVNTSD;
531 else
532 Opc = HasAVX512 ? X86::VMOVSDZmr :
533 HasAVX ? X86::VMOVSDmr : X86::MOVSDmr;
534 } else
535 Opc = X86::ST_Fp64m;
536 break;
537 case MVT::x86mmx:
538 Opc = (IsNonTemporal && HasSSE1) ? X86::MMX_MOVNTQmr : X86::MMX_MOVQ64mr;
539 break;
540 case MVT::v4f32:
541 if (Aligned) {
542 if (IsNonTemporal)
543 Opc = HasVLX ? X86::VMOVNTPSZ128mr :
544 HasAVX ? X86::VMOVNTPSmr : X86::MOVNTPSmr;
545 else
546 Opc = HasVLX ? X86::VMOVAPSZ128mr :
547 HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr;
548 } else
549 Opc = HasVLX ? X86::VMOVUPSZ128mr :
550 HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr;
551 break;
552 case MVT::v2f64:
553 if (Aligned) {
554 if (IsNonTemporal)
555 Opc = HasVLX ? X86::VMOVNTPDZ128mr :
556 HasAVX ? X86::VMOVNTPDmr : X86::MOVNTPDmr;
557 else
558 Opc = HasVLX ? X86::VMOVAPDZ128mr :
559 HasAVX ? X86::VMOVAPDmr : X86::MOVAPDmr;
560 } else
561 Opc = HasVLX ? X86::VMOVUPDZ128mr :
562 HasAVX ? X86::VMOVUPDmr : X86::MOVUPDmr;
563 break;
564 case MVT::v4i32:
565 case MVT::v2i64:
566 case MVT::v8i16:
567 case MVT::v16i8:
568 if (Aligned) {
569 if (IsNonTemporal)
570 Opc = HasVLX ? X86::VMOVNTDQZ128mr :
571 HasAVX ? X86::VMOVNTDQmr : X86::MOVNTDQmr;
572 else
573 Opc = HasVLX ? X86::VMOVDQA64Z128mr :
574 HasAVX ? X86::VMOVDQAmr : X86::MOVDQAmr;
575 } else
576 Opc = HasVLX ? X86::VMOVDQU64Z128mr :
577 HasAVX ? X86::VMOVDQUmr : X86::MOVDQUmr;
578 break;
579 case MVT::v8f32:
580 assert(HasAVX);
581 if (Aligned) {
582 if (IsNonTemporal)
583 Opc = HasVLX ? X86::VMOVNTPSZ256mr : X86::VMOVNTPSYmr;
584 else
585 Opc = HasVLX ? X86::VMOVAPSZ256mr : X86::VMOVAPSYmr;
586 } else
587 Opc = HasVLX ? X86::VMOVUPSZ256mr : X86::VMOVUPSYmr;
588 break;
589 case MVT::v4f64:
590 assert(HasAVX);
591 if (Aligned) {
592 if (IsNonTemporal)
593 Opc = HasVLX ? X86::VMOVNTPDZ256mr : X86::VMOVNTPDYmr;
594 else
595 Opc = HasVLX ? X86::VMOVAPDZ256mr : X86::VMOVAPDYmr;
596 } else
597 Opc = HasVLX ? X86::VMOVUPDZ256mr : X86::VMOVUPDYmr;
598 break;
599 case MVT::v8i32:
600 case MVT::v4i64:
601 case MVT::v16i16:
602 case MVT::v32i8:
603 assert(HasAVX);
604 if (Aligned) {
605 if (IsNonTemporal)
606 Opc = HasVLX ? X86::VMOVNTDQZ256mr : X86::VMOVNTDQYmr;
607 else
608 Opc = HasVLX ? X86::VMOVDQA64Z256mr : X86::VMOVDQAYmr;
609 } else
610 Opc = HasVLX ? X86::VMOVDQU64Z256mr : X86::VMOVDQUYmr;
611 break;
612 case MVT::v16f32:
613 assert(HasAVX512);
614 if (Aligned)
615 Opc = IsNonTemporal ? X86::VMOVNTPSZmr : X86::VMOVAPSZmr;
616 else
617 Opc = X86::VMOVUPSZmr;
618 break;
619 case MVT::v8f64:
620 assert(HasAVX512);
621 if (Aligned) {
622 Opc = IsNonTemporal ? X86::VMOVNTPDZmr : X86::VMOVAPDZmr;
623 } else
624 Opc = X86::VMOVUPDZmr;
625 break;
626 case MVT::v8i64:
627 case MVT::v16i32:
628 case MVT::v32i16:
629 case MVT::v64i8:
630 assert(HasAVX512);
631 // Note: There are a lot more choices based on type with AVX-512, but
632 // there's really no advantage when the store isn't masked.
633 if (Aligned)
634 Opc = IsNonTemporal ? X86::VMOVNTDQZmr : X86::VMOVDQA64Zmr;
635 else
636 Opc = X86::VMOVDQU64Zmr;
637 break;
638 }
639
640 const MCInstrDesc &Desc = TII.get(Opcode: Opc);
641 // Some of the instructions in the previous switch use FR128 instead
642 // of FR32 for ValReg. Make sure the register we feed the instruction
643 // matches its register class constraints.
644 // Note: This is fine to do a copy from FR32 to FR128, this is the
645 // same registers behind the scene and actually why it did not trigger
646 // any bugs before.
647 ValReg = constrainOperandRegClass(II: Desc, Op: ValReg, OpNum: Desc.getNumOperands() - 1);
648 MachineInstrBuilder MIB =
649 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: Desc);
650 addFullAddress(MIB, AM).addReg(RegNo: ValReg);
651 if (MMO)
652 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
653
654 return true;
655}
656
657bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
658 X86AddressMode &AM,
659 MachineMemOperand *MMO, bool Aligned) {
660 // Handle 'null' like i32/i64 0.
661 if (isa<ConstantPointerNull>(Val))
662 Val = Constant::getNullValue(Ty: DL.getIntPtrType(C&: Val->getContext()));
663
664 // If this is a store of a simple constant, fold the constant into the store.
665 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
666 unsigned Opc = 0;
667 bool Signed = true;
668 switch (VT.getSimpleVT().SimpleTy) {
669 default: break;
670 case MVT::i1:
671 Signed = false;
672 [[fallthrough]]; // Handle as i8.
673 case MVT::i8: Opc = X86::MOV8mi; break;
674 case MVT::i16: Opc = X86::MOV16mi; break;
675 case MVT::i32: Opc = X86::MOV32mi; break;
676 case MVT::i64:
677 // Must be a 32-bit sign extended value.
678 if (isInt<32>(x: CI->getSExtValue()))
679 Opc = X86::MOV64mi32;
680 break;
681 }
682
683 if (Opc) {
684 MachineInstrBuilder MIB =
685 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc));
686 addFullAddress(MIB, AM).addImm(Val: Signed ? (uint64_t) CI->getSExtValue()
687 : CI->getZExtValue());
688 if (MMO)
689 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
690 return true;
691 }
692 }
693
694 Register ValReg = getRegForValue(V: Val);
695 if (!ValReg)
696 return false;
697
698 return X86FastEmitStore(VT, ValReg, AM, MMO, Aligned);
699}
700
701/// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
702/// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.
703/// ISD::SIGN_EXTEND).
704bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, Register Src,
705 EVT SrcVT, Register &ResultReg) {
706 Register RR = fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(), Opcode: Opc, Op0: Src);
707 if (!RR)
708 return false;
709
710 ResultReg = RR;
711 return true;
712}
713
714bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
715 // Handle constant address.
716 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
717 // Can't handle alternate code models yet.
718 if (TM.getCodeModel() != CodeModel::Small &&
719 TM.getCodeModel() != CodeModel::Medium)
720 return false;
721
722 // Can't handle large objects yet.
723 if (TM.isLargeGlobalValue(GV))
724 return false;
725
726 // Can't handle TLS yet.
727 if (GV->isThreadLocal())
728 return false;
729
730 // Can't handle !absolute_symbol references yet.
731 if (GV->isAbsoluteSymbolRef())
732 return false;
733
734 // RIP-relative addresses can't have additional register operands, so if
735 // we've already folded stuff into the addressing mode, just force the
736 // global value into its own register, which we can use as the basereg.
737 if (!Subtarget->isPICStyleRIPRel() ||
738 (AM.Base.Reg == 0 && AM.IndexReg == 0)) {
739 // Okay, we've committed to selecting this global. Set up the address.
740 AM.GV = GV;
741
742 // Allow the subtarget to classify the global.
743 unsigned char GVFlags = Subtarget->classifyGlobalReference(GV);
744
745 // If this reference is relative to the pic base, set it now.
746 if (isGlobalRelativeToPICBase(TargetFlag: GVFlags)) {
747 // FIXME: How do we know Base.Reg is free??
748 AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
749 }
750
751 // Unless the ABI requires an extra load, return a direct reference to
752 // the global.
753 if (!isGlobalStubReference(TargetFlag: GVFlags)) {
754 if (Subtarget->isPICStyleRIPRel()) {
755 // Use rip-relative addressing if we can. Above we verified that the
756 // base and index registers are unused.
757 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
758 AM.Base.Reg = X86::RIP;
759 }
760 AM.GVOpFlags = GVFlags;
761 return true;
762 }
763
764 // Ok, we need to do a load from a stub. If we've already loaded from
765 // this stub, reuse the loaded pointer, otherwise emit the load now.
766 auto I = LocalValueMap.find(Val: V);
767 Register LoadReg;
768 if (I != LocalValueMap.end() && I->second) {
769 LoadReg = I->second;
770 } else {
771 // Issue load from stub.
772 unsigned Opc = 0;
773 const TargetRegisterClass *RC = nullptr;
774 X86AddressMode StubAM;
775 StubAM.Base.Reg = AM.Base.Reg;
776 StubAM.GV = GV;
777 StubAM.GVOpFlags = GVFlags;
778
779 // Prepare for inserting code in the local-value area.
780 SavePoint SaveInsertPt = enterLocalValueArea();
781
782 if (TLI.getPointerTy(DL) == MVT::i64) {
783 Opc = X86::MOV64rm;
784 RC = &X86::GR64RegClass;
785 } else {
786 Opc = X86::MOV32rm;
787 RC = &X86::GR32RegClass;
788 }
789
790 if (Subtarget->isPICStyleRIPRel() || GVFlags == X86II::MO_GOTPCREL ||
791 GVFlags == X86II::MO_GOTPCREL_NORELAX)
792 StubAM.Base.Reg = X86::RIP;
793
794 LoadReg = createResultReg(RC);
795 MachineInstrBuilder LoadMI =
796 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: LoadReg);
797 addFullAddress(MIB: LoadMI, AM&: StubAM);
798
799 // Ok, back to normal mode.
800 leaveLocalValueArea(Old: SaveInsertPt);
801
802 // Prevent loading GV stub multiple times in same MBB.
803 LocalValueMap[V] = LoadReg;
804 }
805
806 // Now construct the final address. Note that the Disp, Scale,
807 // and Index values may already be set here.
808 AM.Base.Reg = LoadReg;
809 AM.GV = nullptr;
810 return true;
811 }
812 }
813
814 // If all else fails, try to materialize the value in a register.
815 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
816 if (AM.Base.Reg == 0) {
817 AM.Base.Reg = getRegForValue(V);
818 return AM.Base.Reg != 0;
819 }
820 if (AM.IndexReg == 0) {
821 assert(AM.Scale == 1 && "Scale with no index!");
822 AM.IndexReg = getRegForValue(V);
823 return AM.IndexReg != 0;
824 }
825 }
826
827 return false;
828}
829
830/// X86SelectAddress - Attempt to fill in an address from the given value.
831///
832bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
833 SmallVector<const Value *, 32> GEPs;
834redo_gep:
835 const User *U = nullptr;
836 unsigned Opcode = Instruction::UserOp1;
837 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
838 // Don't walk into other basic blocks; it's possible we haven't
839 // visited them yet, so the instructions may not yet be assigned
840 // virtual registers.
841 if (FuncInfo.StaticAllocaMap.count(Val: static_cast<const AllocaInst *>(V)) ||
842 FuncInfo.getMBB(BB: I->getParent()) == FuncInfo.MBB) {
843 Opcode = I->getOpcode();
844 U = I;
845 }
846 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(Val: V)) {
847 Opcode = C->getOpcode();
848 U = C;
849 }
850
851 if (PointerType *Ty = dyn_cast<PointerType>(Val: V->getType()))
852 if (Ty->getAddressSpace() > 255)
853 // Fast instruction selection doesn't support the special
854 // address spaces.
855 return false;
856
857 switch (Opcode) {
858 default: break;
859 case Instruction::BitCast:
860 // Look past bitcasts.
861 return X86SelectAddress(V: U->getOperand(i: 0), AM);
862
863 case Instruction::IntToPtr:
864 // Look past no-op inttoptrs.
865 if (TLI.getValueType(DL, Ty: U->getOperand(i: 0)->getType()) ==
866 TLI.getPointerTy(DL))
867 return X86SelectAddress(V: U->getOperand(i: 0), AM);
868 break;
869
870 case Instruction::PtrToInt:
871 // Look past no-op ptrtoints.
872 if (TLI.getValueType(DL, Ty: U->getType()) == TLI.getPointerTy(DL))
873 return X86SelectAddress(V: U->getOperand(i: 0), AM);
874 break;
875
876 case Instruction::Alloca: {
877 // Do static allocas.
878 const AllocaInst *A = cast<AllocaInst>(Val: V);
879 auto SI = FuncInfo.StaticAllocaMap.find(Val: A);
880 if (SI != FuncInfo.StaticAllocaMap.end()) {
881 AM.BaseType = X86AddressMode::FrameIndexBase;
882 AM.Base.FrameIndex = SI->second;
883 return true;
884 }
885 break;
886 }
887
888 case Instruction::Add: {
889 // Adds of constants are common and easy enough.
890 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: U->getOperand(i: 1))) {
891 uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue();
892 // They have to fit in the 32-bit signed displacement field though.
893 if (isInt<32>(x: Disp)) {
894 AM.Disp = (uint32_t)Disp;
895 return X86SelectAddress(V: U->getOperand(i: 0), AM);
896 }
897 }
898 break;
899 }
900
901 case Instruction::GetElementPtr: {
902 X86AddressMode SavedAM = AM;
903
904 // Pattern-match simple GEPs.
905 uint64_t Disp = (int32_t)AM.Disp;
906 Register IndexReg = AM.IndexReg;
907 unsigned Scale = AM.Scale;
908 MVT PtrVT = TLI.getValueType(DL, Ty: U->getType()).getSimpleVT();
909
910 gep_type_iterator GTI = gep_type_begin(GEP: U);
911 // Iterate through the indices, folding what we can. Constants can be
912 // folded, and one dynamic index can be handled, if the scale is supported.
913 for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();
914 i != e; ++i, ++GTI) {
915 const Value *Op = *i;
916 if (StructType *STy = GTI.getStructTypeOrNull()) {
917 const StructLayout *SL = DL.getStructLayout(Ty: STy);
918 Disp += SL->getElementOffset(Idx: cast<ConstantInt>(Val: Op)->getZExtValue());
919 continue;
920 }
921
922 // A array/variable index is always of the form i*S where S is the
923 // constant scale size. See if we can push the scale into immediates.
924 uint64_t S = GTI.getSequentialElementStride(DL);
925 for (;;) {
926 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Op)) {
927 // Constant-offset addressing. The index may be wider than 64 bits;
928 // it is truncated to the pointer width like any other GEP index.
929 Disp += CI->getValue().sextOrTrunc(width: 64).getSExtValue() * S;
930 break;
931 }
932 if (canFoldAddIntoGEP(GEP: U, Add: Op)) {
933 // A compatible add with a constant operand. Fold the constant.
934 ConstantInt *CI =
935 cast<ConstantInt>(Val: cast<AddOperator>(Val: Op)->getOperand(i_nocapture: 1));
936 Disp += CI->getSExtValue() * S;
937 // Iterate on the other operand.
938 Op = cast<AddOperator>(Val: Op)->getOperand(i_nocapture: 0);
939 continue;
940 }
941 if (!IndexReg && (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
942 (S == 1 || S == 2 || S == 4 || S == 8)) {
943 // Scaled-index addressing.
944 Scale = S;
945 IndexReg = getRegForGEPIndex(PtrVT, Idx: Op);
946 if (!IndexReg)
947 return false;
948 break;
949 }
950 // Unsupported.
951 goto unsupported_gep;
952 }
953 }
954
955 // Check for displacement overflow.
956 if (!isInt<32>(x: Disp))
957 break;
958
959 AM.IndexReg = IndexReg;
960 AM.Scale = Scale;
961 AM.Disp = (uint32_t)Disp;
962 GEPs.push_back(Elt: V);
963
964 if (const GetElementPtrInst *GEP =
965 dyn_cast<GetElementPtrInst>(Val: U->getOperand(i: 0))) {
966 // Ok, the GEP indices were covered by constant-offset and scaled-index
967 // addressing. Update the address state and move on to examining the base.
968 V = GEP;
969 goto redo_gep;
970 } else if (X86SelectAddress(V: U->getOperand(i: 0), AM)) {
971 return true;
972 }
973
974 // If we couldn't merge the gep value into this addr mode, revert back to
975 // our address and just match the value instead of completely failing.
976 AM = SavedAM;
977
978 for (const Value *I : reverse(C&: GEPs))
979 if (handleConstantAddresses(V: I, AM))
980 return true;
981
982 return false;
983 unsupported_gep:
984 // Ok, the GEP indices weren't all covered.
985 break;
986 }
987 }
988
989 return handleConstantAddresses(V, AM);
990}
991
992/// X86SelectCallAddress - Attempt to fill in an address from the given value.
993///
994bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
995 const User *U = nullptr;
996 unsigned Opcode = Instruction::UserOp1;
997 const Instruction *I = dyn_cast<Instruction>(Val: V);
998 // Record if the value is defined in the same basic block.
999 //
1000 // This information is crucial to know whether or not folding an
1001 // operand is valid.
1002 // Indeed, FastISel generates or reuses a virtual register for all
1003 // operands of all instructions it selects. Obviously, the definition and
1004 // its uses must use the same virtual register otherwise the produced
1005 // code is incorrect.
1006 // Before instruction selection, FunctionLoweringInfo::set sets the virtual
1007 // registers for values that are alive across basic blocks. This ensures
1008 // that the values are consistently set between across basic block, even
1009 // if different instruction selection mechanisms are used (e.g., a mix of
1010 // SDISel and FastISel).
1011 // For values local to a basic block, the instruction selection process
1012 // generates these virtual registers with whatever method is appropriate
1013 // for its needs. In particular, FastISel and SDISel do not share the way
1014 // local virtual registers are set.
1015 // Therefore, this is impossible (or at least unsafe) to share values
1016 // between basic blocks unless they use the same instruction selection
1017 // method, which is not guarantee for X86.
1018 // Moreover, things like hasOneUse could not be used accurately, if we
1019 // allow to reference values across basic blocks whereas they are not
1020 // alive across basic blocks initially.
1021 bool InMBB = true;
1022 if (I) {
1023 Opcode = I->getOpcode();
1024 U = I;
1025 InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock();
1026 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(Val: V)) {
1027 Opcode = C->getOpcode();
1028 U = C;
1029 }
1030
1031 switch (Opcode) {
1032 default: break;
1033 case Instruction::BitCast:
1034 // Look past bitcasts if its operand is in the same BB.
1035 if (InMBB)
1036 return X86SelectCallAddress(V: U->getOperand(i: 0), AM);
1037 break;
1038
1039 case Instruction::IntToPtr:
1040 // Look past no-op inttoptrs if its operand is in the same BB.
1041 if (InMBB &&
1042 TLI.getValueType(DL, Ty: U->getOperand(i: 0)->getType()) ==
1043 TLI.getPointerTy(DL))
1044 return X86SelectCallAddress(V: U->getOperand(i: 0), AM);
1045 break;
1046
1047 case Instruction::PtrToInt:
1048 // Look past no-op ptrtoints if its operand is in the same BB.
1049 if (InMBB && TLI.getValueType(DL, Ty: U->getType()) == TLI.getPointerTy(DL))
1050 return X86SelectCallAddress(V: U->getOperand(i: 0), AM);
1051 break;
1052 }
1053
1054 // Handle constant address.
1055 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: V)) {
1056 // Can't handle alternate code models yet.
1057 if (TM.getCodeModel() != CodeModel::Small &&
1058 TM.getCodeModel() != CodeModel::Medium)
1059 return false;
1060
1061 // RIP-relative addresses can't have additional register operands.
1062 if (Subtarget->isPICStyleRIPRel() &&
1063 (AM.Base.Reg != 0 || AM.IndexReg != 0))
1064 return false;
1065
1066 // Can't handle TLS.
1067 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(Val: GV))
1068 if (GVar->isThreadLocal())
1069 return false;
1070
1071 // Okay, we've committed to selecting this global. Set up the basic address.
1072 AM.GV = GV;
1073
1074 // Return a direct reference to the global. Fastisel can handle calls to
1075 // functions that require loads, such as dllimport and nonlazybind
1076 // functions.
1077 if (Subtarget->isPICStyleRIPRel()) {
1078 // Use rip-relative addressing if we can. Above we verified that the
1079 // base and index registers are unused.
1080 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
1081 AM.Base.Reg = X86::RIP;
1082 } else {
1083 AM.GVOpFlags = Subtarget->classifyLocalReference(GV: nullptr);
1084 }
1085
1086 return true;
1087 }
1088
1089 // If all else fails, try to materialize the value in a register.
1090 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
1091 auto GetCallRegForValue = [this](const Value *V) {
1092 Register Reg = getRegForValue(V);
1093
1094 // In 64-bit mode, we need a 64-bit register even if pointers are 32 bits.
1095 if (Reg && Subtarget->isTarget64BitILP32()) {
1096 Register CopyReg = createResultReg(RC: &X86::GR32RegClass);
1097 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV32rr),
1098 DestReg: CopyReg)
1099 .addReg(RegNo: Reg);
1100
1101 Register ExtReg = createResultReg(RC: &X86::GR64RegClass);
1102 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1103 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: ExtReg)
1104 .addReg(RegNo: CopyReg)
1105 .addImm(Val: X86::sub_32bit);
1106 Reg = ExtReg;
1107 }
1108
1109 return Reg;
1110 };
1111
1112 if (AM.Base.Reg == 0) {
1113 AM.Base.Reg = GetCallRegForValue(V);
1114 return AM.Base.Reg != 0;
1115 }
1116 if (AM.IndexReg == 0) {
1117 assert(AM.Scale == 1 && "Scale with no index!");
1118 AM.IndexReg = GetCallRegForValue(V);
1119 return AM.IndexReg != 0;
1120 }
1121 }
1122
1123 return false;
1124}
1125
1126
1127/// X86SelectStore - Select and emit code to implement store instructions.
1128bool X86FastISel::X86SelectStore(const Instruction *I) {
1129 // Atomic stores need special handling.
1130 const StoreInst *S = cast<StoreInst>(Val: I);
1131
1132 if (S->isAtomic())
1133 return false;
1134
1135 const Value *PtrV = I->getOperand(i: 1);
1136 if (TLI.supportSwiftError()) {
1137 // Swifterror values can come from either a function parameter with
1138 // swifterror attribute or an alloca with swifterror attribute.
1139 if (const Argument *Arg = dyn_cast<Argument>(Val: PtrV)) {
1140 if (Arg->hasSwiftErrorAttr())
1141 return false;
1142 }
1143
1144 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: PtrV)) {
1145 if (Alloca->isSwiftError())
1146 return false;
1147 }
1148 }
1149
1150 const Value *Val = S->getValueOperand();
1151 const Value *Ptr = S->getPointerOperand();
1152
1153 MVT VT;
1154 if (!isTypeLegal(Ty: Val->getType(), VT, /*AllowI1=*/true))
1155 return false;
1156
1157 Align Alignment = S->getAlign();
1158 Align ABIAlignment = DL.getABITypeAlign(Ty: Val->getType());
1159 bool Aligned = Alignment >= ABIAlignment;
1160
1161 X86AddressMode AM;
1162 if (!X86SelectAddress(V: Ptr, AM))
1163 return false;
1164
1165 return X86FastEmitStore(VT, Val, AM, MMO: createMachineMemOperandFor(I), Aligned);
1166}
1167
1168/// X86SelectRet - Select and emit code to implement ret instructions.
1169bool X86FastISel::X86SelectRet(const Instruction *I) {
1170 const ReturnInst *Ret = cast<ReturnInst>(Val: I);
1171 const Function &F = *I->getParent()->getParent();
1172 const X86MachineFunctionInfo *X86MFInfo =
1173 FuncInfo.MF->getInfo<X86MachineFunctionInfo>();
1174
1175 if (!FuncInfo.CanLowerReturn)
1176 return false;
1177
1178 if (TLI.supportSwiftError() &&
1179 F.getAttributes().hasAttrSomewhere(Kind: Attribute::SwiftError))
1180 return false;
1181
1182 if (TLI.supportSplitCSR(MF: FuncInfo.MF))
1183 return false;
1184
1185 CallingConv::ID CC = F.getCallingConv();
1186 if (CC != CallingConv::C &&
1187 CC != CallingConv::Fast &&
1188 CC != CallingConv::Tail &&
1189 CC != CallingConv::SwiftTail &&
1190 CC != CallingConv::X86_FastCall &&
1191 CC != CallingConv::X86_StdCall &&
1192 CC != CallingConv::X86_ThisCall &&
1193 CC != CallingConv::X86_64_SysV &&
1194 CC != CallingConv::Win64)
1195 return false;
1196
1197 // Don't handle popping bytes if they don't fit the ret's immediate.
1198 if (!isUInt<16>(x: X86MFInfo->getBytesToPopOnReturn()))
1199 return false;
1200
1201 // fastcc with -tailcallopt is intended to provide a guaranteed
1202 // tail call optimization. Fastisel doesn't know how to do that.
1203 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
1204 CC == CallingConv::Tail || CC == CallingConv::SwiftTail)
1205 return false;
1206
1207 // Let SDISel handle vararg functions.
1208 if (F.isVarArg())
1209 return false;
1210
1211 // Build a list of return value registers.
1212 SmallVector<Register, 4> RetRegs;
1213
1214 if (Ret->getNumOperands() > 0) {
1215 SmallVector<ISD::OutputArg, 4> Outs;
1216 GetReturnInfo(CC, ReturnType: F.getReturnType(), attr: F.getAttributes(), Outs, TLI, DL);
1217
1218 // Analyze operands of the call, assigning locations to each operand.
1219 SmallVector<CCValAssign, 16> ValLocs;
1220 CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext());
1221 CCInfo.AnalyzeReturn(Outs, Fn: RetCC_X86);
1222
1223 const Value *RV = Ret->getOperand(i_nocapture: 0);
1224 Register Reg = getRegForValue(V: RV);
1225 if (!Reg)
1226 return false;
1227
1228 // Only handle a single return value for now.
1229 if (ValLocs.size() != 1)
1230 return false;
1231
1232 CCValAssign &VA = ValLocs[0];
1233
1234 // Don't bother handling odd stuff for now.
1235 if (VA.getLocInfo() != CCValAssign::Full)
1236 return false;
1237 // Only handle register returns for now.
1238 if (!VA.isRegLoc())
1239 return false;
1240
1241 // The calling-convention tables for x87 returns don't tell
1242 // the whole story.
1243 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
1244 return false;
1245
1246 Register SrcReg = Reg + VA.getValNo();
1247 EVT SrcVT = TLI.getValueType(DL, Ty: RV->getType());
1248 EVT DstVT = VA.getValVT();
1249 // Special handling for extended integers.
1250 if (SrcVT != DstVT) {
1251 if (SrcVT != MVT::i1 && SrcVT != MVT::i8 && SrcVT != MVT::i16)
1252 return false;
1253
1254 if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
1255 return false;
1256
1257 if (SrcVT == MVT::i1) {
1258 if (Outs[0].Flags.isSExt())
1259 return false;
1260 SrcReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: SrcReg);
1261 SrcVT = MVT::i8;
1262 }
1263 if (SrcVT != DstVT) {
1264 unsigned Op =
1265 Outs[0].Flags.isZExt() ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
1266 SrcReg =
1267 fastEmit_r(VT: SrcVT.getSimpleVT(), RetVT: DstVT.getSimpleVT(), Opcode: Op, Op0: SrcReg);
1268 }
1269 }
1270
1271 // Make the copy.
1272 Register DstReg = VA.getLocReg();
1273 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
1274 // Avoid a cross-class copy. This is very unlikely.
1275 if (!SrcRC->contains(Reg: DstReg))
1276 return false;
1277 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1278 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg).addReg(RegNo: SrcReg);
1279
1280 // Add register to return instruction.
1281 RetRegs.push_back(Elt: VA.getLocReg());
1282 }
1283
1284 // Swift calling convention does not require we copy the sret argument
1285 // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
1286
1287 // All x86 ABIs require that for returning structs by value we copy
1288 // the sret argument into %rax/%eax (depending on ABI) for the return.
1289 // We saved the argument into a virtual register in the entry block,
1290 // so now we copy the value out and into %rax/%eax.
1291 if (F.hasStructRetAttr() && CC != CallingConv::Swift &&
1292 CC != CallingConv::SwiftTail) {
1293 Register Reg = X86MFInfo->getSRetReturnReg();
1294 assert(Reg &&
1295 "SRetReturnReg should have been set in LowerFormalArguments()!");
1296 Register RetReg = Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX;
1297 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1298 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: RetReg).addReg(RegNo: Reg);
1299 RetRegs.push_back(Elt: RetReg);
1300 }
1301
1302 // Now emit the RET.
1303 MachineInstrBuilder MIB;
1304 if (X86MFInfo->getBytesToPopOnReturn()) {
1305 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1306 MCID: TII.get(Opcode: Subtarget->is64Bit() ? X86::RETI64 : X86::RETI32))
1307 .addImm(Val: X86MFInfo->getBytesToPopOnReturn());
1308 } else {
1309 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1310 MCID: TII.get(Opcode: Subtarget->is64Bit() ? X86::RET64 : X86::RET32));
1311 }
1312 for (Register Reg : RetRegs)
1313 MIB.addReg(RegNo: Reg, Flags: RegState::Implicit);
1314 return true;
1315}
1316
1317/// X86SelectLoad - Select and emit code to implement load instructions.
1318///
1319bool X86FastISel::X86SelectLoad(const Instruction *I) {
1320 const LoadInst *LI = cast<LoadInst>(Val: I);
1321
1322 // Atomic loads need special handling.
1323 if (LI->isAtomic())
1324 return false;
1325
1326 const Value *SV = I->getOperand(i: 0);
1327 if (TLI.supportSwiftError()) {
1328 // Swifterror values can come from either a function parameter with
1329 // swifterror attribute or an alloca with swifterror attribute.
1330 if (const Argument *Arg = dyn_cast<Argument>(Val: SV)) {
1331 if (Arg->hasSwiftErrorAttr())
1332 return false;
1333 }
1334
1335 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(Val: SV)) {
1336 if (Alloca->isSwiftError())
1337 return false;
1338 }
1339 }
1340
1341 MVT VT;
1342 if (!isTypeLegal(Ty: LI->getType(), VT, /*AllowI1=*/true))
1343 return false;
1344
1345 const Value *Ptr = LI->getPointerOperand();
1346
1347 X86AddressMode AM;
1348 if (!X86SelectAddress(V: Ptr, AM))
1349 return false;
1350
1351 Register ResultReg;
1352 if (!X86FastEmitLoad(VT, AM, MMO: createMachineMemOperandFor(I: LI), ResultReg,
1353 Alignment: LI->getAlign().value()))
1354 return false;
1355
1356 updateValueMap(I, Reg: ResultReg);
1357 return true;
1358}
1359
1360static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget) {
1361 bool HasAVX512 = Subtarget->hasAVX512();
1362 bool HasAVX = Subtarget->hasAVX();
1363 bool HasSSE1 = Subtarget->hasSSE1();
1364 bool HasSSE2 = Subtarget->hasSSE2();
1365
1366 switch (VT.getSimpleVT().SimpleTy) {
1367 default: return 0;
1368 case MVT::i8: return X86::CMP8rr;
1369 case MVT::i16: return X86::CMP16rr;
1370 case MVT::i32: return X86::CMP32rr;
1371 case MVT::i64: return X86::CMP64rr;
1372 case MVT::f32:
1373 return HasAVX512 ? X86::VUCOMISSZrr
1374 : HasAVX ? X86::VUCOMISSrr
1375 : HasSSE1 ? X86::UCOMISSrr
1376 : 0;
1377 case MVT::f64:
1378 return HasAVX512 ? X86::VUCOMISDZrr
1379 : HasAVX ? X86::VUCOMISDrr
1380 : HasSSE2 ? X86::UCOMISDrr
1381 : 0;
1382 }
1383}
1384
1385/// If we have a comparison with RHS as the RHS of the comparison, return an
1386/// opcode that works for the compare (e.g. CMP32ri) otherwise return 0.
1387static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC) {
1388 switch (VT.getSimpleVT().SimpleTy) {
1389 // Otherwise, we can't fold the immediate into this comparison.
1390 default:
1391 return 0;
1392 case MVT::i8:
1393 return X86::CMP8ri;
1394 case MVT::i16:
1395 return X86::CMP16ri;
1396 case MVT::i32:
1397 return X86::CMP32ri;
1398 case MVT::i64:
1399 // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext
1400 // field.
1401 return isInt<32>(x: RHSC->getSExtValue()) ? X86::CMP64ri32 : 0;
1402 }
1403}
1404
1405bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, EVT VT,
1406 const DebugLoc &CurMIMD) {
1407 Register Op0Reg = getRegForValue(V: Op0);
1408 if (!Op0Reg)
1409 return false;
1410
1411 // Handle 'null' like i32/i64 0.
1412 if (isa<ConstantPointerNull>(Val: Op1))
1413 Op1 = Constant::getNullValue(Ty: DL.getIntPtrType(C&: Op0->getContext()));
1414
1415 // We have two options: compare with register or immediate. If the RHS of
1416 // the compare is an immediate that we can fold into this compare, use
1417 // CMPri, otherwise use CMPrr.
1418 if (const ConstantInt *Op1C = dyn_cast<ConstantInt>(Val: Op1)) {
1419 if (unsigned CompareImmOpc = X86ChooseCmpImmediateOpcode(VT, RHSC: Op1C)) {
1420 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: CurMIMD, MCID: TII.get(Opcode: CompareImmOpc))
1421 .addReg(RegNo: Op0Reg)
1422 .addImm(Val: Op1C->getSExtValue());
1423 return true;
1424 }
1425 }
1426
1427 unsigned CompareOpc = X86ChooseCmpOpcode(VT, Subtarget);
1428 if (CompareOpc == 0) return false;
1429
1430 Register Op1Reg = getRegForValue(V: Op1);
1431 if (!Op1Reg)
1432 return false;
1433 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD: CurMIMD, MCID: TII.get(Opcode: CompareOpc))
1434 .addReg(RegNo: Op0Reg)
1435 .addReg(RegNo: Op1Reg);
1436
1437 return true;
1438}
1439
1440#define GET_SETCC \
1441 ((!Subtarget->hasZU() || Subtarget->preferLegacySetCC()) ? X86::SETCCr \
1442 : X86::SETZUCCr)
1443
1444bool X86FastISel::X86SelectCmp(const Instruction *I) {
1445 const CmpInst *CI = cast<CmpInst>(Val: I);
1446
1447 MVT VT;
1448 if (!isTypeLegal(Ty: I->getOperand(i: 0)->getType(), VT))
1449 return false;
1450
1451 // Below code only works for scalars.
1452 if (VT.isVector())
1453 return false;
1454
1455 // Try to optimize or fold the cmp.
1456 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1457 Register ResultReg;
1458 switch (Predicate) {
1459 default: break;
1460 case CmpInst::FCMP_FALSE: {
1461 ResultReg = emitMOV32r0();
1462 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: ResultReg, Idx: X86::sub_8bit);
1463 if (!ResultReg)
1464 return false;
1465 break;
1466 }
1467 case CmpInst::FCMP_TRUE: {
1468 ResultReg = createResultReg(RC: &X86::GR8RegClass);
1469 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV8ri),
1470 DestReg: ResultReg).addImm(Val: 1);
1471 break;
1472 }
1473 }
1474
1475 if (ResultReg) {
1476 updateValueMap(I, Reg: ResultReg);
1477 return true;
1478 }
1479
1480 const Value *LHS = CI->getOperand(i_nocapture: 0);
1481 const Value *RHS = CI->getOperand(i_nocapture: 1);
1482
1483 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
1484 // We don't have to materialize a zero constant for this case and can just use
1485 // %x again on the RHS.
1486 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1487 const auto *RHSC = dyn_cast<ConstantFP>(Val: RHS);
1488 if (RHSC && RHSC->isNullValue())
1489 RHS = LHS;
1490 }
1491
1492 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1493 static const uint16_t SETFOpcTable[2][3] = {
1494 { X86::COND_E, X86::COND_NP, X86::AND8rr },
1495 { X86::COND_NE, X86::COND_P, X86::OR8rr }
1496 };
1497 const uint16_t *SETFOpc = nullptr;
1498 switch (Predicate) {
1499 default: break;
1500 case CmpInst::FCMP_OEQ: SETFOpc = &SETFOpcTable[0][0]; break;
1501 case CmpInst::FCMP_UNE: SETFOpc = &SETFOpcTable[1][0]; break;
1502 }
1503
1504 ResultReg = createResultReg(RC: &X86::GR8RegClass);
1505 if (SETFOpc) {
1506 if (!X86FastEmitCompare(Op0: LHS, Op1: RHS, VT, CurMIMD: I->getDebugLoc()))
1507 return false;
1508
1509 Register FlagReg1 = createResultReg(RC: &X86::GR8RegClass);
1510 Register FlagReg2 = createResultReg(RC: &X86::GR8RegClass);
1511 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
1512 DestReg: FlagReg1)
1513 .addImm(Val: SETFOpc[0]);
1514 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
1515 DestReg: FlagReg2)
1516 .addImm(Val: SETFOpc[1]);
1517 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: SETFOpc[2]),
1518 DestReg: ResultReg).addReg(RegNo: FlagReg1).addReg(RegNo: FlagReg2);
1519 updateValueMap(I, Reg: ResultReg);
1520 return true;
1521 }
1522
1523 X86::CondCode CC;
1524 bool SwapArgs;
1525 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1526 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1527
1528 if (SwapArgs)
1529 std::swap(a&: LHS, b&: RHS);
1530
1531 // Emit a compare of LHS/RHS.
1532 if (!X86FastEmitCompare(Op0: LHS, Op1: RHS, VT, CurMIMD: I->getDebugLoc()))
1533 return false;
1534
1535 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC), DestReg: ResultReg)
1536 .addImm(Val: CC);
1537 updateValueMap(I, Reg: ResultReg);
1538 return true;
1539}
1540
1541bool X86FastISel::X86SelectZExt(const Instruction *I) {
1542 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1543 if (!TLI.isTypeLegal(VT: DstVT))
1544 return false;
1545
1546 Register ResultReg = getRegForValue(V: I->getOperand(i: 0));
1547 if (!ResultReg)
1548 return false;
1549
1550 // Handle zero-extension from i1 to i8, which is common.
1551 MVT SrcVT = TLI.getSimpleValueType(DL, Ty: I->getOperand(i: 0)->getType());
1552 if (SrcVT == MVT::i1) {
1553 // Set the high bits to zero.
1554 ResultReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: ResultReg);
1555 SrcVT = MVT::i8;
1556
1557 if (!ResultReg)
1558 return false;
1559 }
1560
1561 if (DstVT == MVT::i64) {
1562 // Handle extension to 64-bits via sub-register shenanigans.
1563 unsigned MovInst;
1564
1565 switch (SrcVT.SimpleTy) {
1566 case MVT::i8: MovInst = X86::MOVZX32rr8; break;
1567 case MVT::i16: MovInst = X86::MOVZX32rr16; break;
1568 case MVT::i32: MovInst = X86::MOV32rr; break;
1569 default: llvm_unreachable("Unexpected zext to i64 source type");
1570 }
1571
1572 Register Result32 = createResultReg(RC: &X86::GR32RegClass);
1573 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: MovInst), DestReg: Result32)
1574 .addReg(RegNo: ResultReg);
1575
1576 ResultReg = createResultReg(RC: &X86::GR64RegClass);
1577 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1578 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: ResultReg)
1579 .addReg(RegNo: Result32)
1580 .addImm(Val: X86::sub_32bit);
1581 } else if (DstVT == MVT::i16) {
1582 // i8->i16 doesn't exist in the autogenerated isel table. Need to zero
1583 // extend to 32-bits and then extract down to 16-bits.
1584 Register Result32 = createResultReg(RC: &X86::GR32RegClass);
1585 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOVZX32rr8),
1586 DestReg: Result32).addReg(RegNo: ResultReg);
1587
1588 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0: Result32, Idx: X86::sub_16bit);
1589 } else if (DstVT != MVT::i8) {
1590 ResultReg = fastEmit_r(VT: MVT::i8, RetVT: DstVT.getSimpleVT(), Opcode: ISD::ZERO_EXTEND,
1591 Op0: ResultReg);
1592 if (!ResultReg)
1593 return false;
1594 }
1595
1596 updateValueMap(I, Reg: ResultReg);
1597 return true;
1598}
1599
1600bool X86FastISel::X86SelectSExt(const Instruction *I) {
1601 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
1602 if (!TLI.isTypeLegal(VT: DstVT))
1603 return false;
1604
1605 Register ResultReg = getRegForValue(V: I->getOperand(i: 0));
1606 if (!ResultReg)
1607 return false;
1608
1609 // Handle sign-extension from i1 to i8.
1610 MVT SrcVT = TLI.getSimpleValueType(DL, Ty: I->getOperand(i: 0)->getType());
1611 if (SrcVT == MVT::i1) {
1612 // Set the high bits to zero.
1613 Register ZExtReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: ResultReg);
1614 if (!ZExtReg)
1615 return false;
1616
1617 // Negate the result to make an 8-bit sign extended value.
1618 ResultReg = createResultReg(RC: &X86::GR8RegClass);
1619 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::NEG8r),
1620 DestReg: ResultReg).addReg(RegNo: ZExtReg);
1621
1622 SrcVT = MVT::i8;
1623 }
1624
1625 if (DstVT == MVT::i16) {
1626 // i8->i16 doesn't exist in the autogenerated isel table. Need to sign
1627 // extend to 32-bits and then extract down to 16-bits.
1628 Register Result32 = createResultReg(RC: &X86::GR32RegClass);
1629 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOVSX32rr8),
1630 DestReg: Result32).addReg(RegNo: ResultReg);
1631
1632 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0: Result32, Idx: X86::sub_16bit);
1633 } else if (DstVT != MVT::i8) {
1634 ResultReg = fastEmit_r(VT: MVT::i8, RetVT: DstVT.getSimpleVT(), Opcode: ISD::SIGN_EXTEND,
1635 Op0: ResultReg);
1636 if (!ResultReg)
1637 return false;
1638 }
1639
1640 updateValueMap(I, Reg: ResultReg);
1641 return true;
1642}
1643
1644bool X86FastISel::X86SelectBranch(const Instruction *I) {
1645 // Unconditional branches are selected by tablegen-generated code.
1646 // Handle a conditional branch.
1647 const CondBrInst *BI = cast<CondBrInst>(Val: I);
1648 MachineBasicBlock *TrueMBB = FuncInfo.getMBB(BB: BI->getSuccessor(i: 0));
1649 MachineBasicBlock *FalseMBB = FuncInfo.getMBB(BB: BI->getSuccessor(i: 1));
1650
1651 // Fold the common case of a conditional branch with a comparison
1652 // in the same block (values defined on other blocks may not have
1653 // initialized registers).
1654 X86::CondCode CC;
1655 if (const CmpInst *CI = dyn_cast<CmpInst>(Val: BI->getCondition())) {
1656 if (CI->hasOneUse() && CI->getParent() == I->getParent()) {
1657 EVT VT = TLI.getValueType(DL, Ty: CI->getOperand(i_nocapture: 0)->getType());
1658
1659 // Try to optimize or fold the cmp.
1660 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1661 switch (Predicate) {
1662 default: break;
1663 case CmpInst::FCMP_FALSE: fastEmitBranch(MSucc: FalseMBB, DbgLoc: MIMD.getDL()); return true;
1664 case CmpInst::FCMP_TRUE: fastEmitBranch(MSucc: TrueMBB, DbgLoc: MIMD.getDL()); return true;
1665 }
1666
1667 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
1668 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
1669
1670 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x,
1671 // 0.0.
1672 // We don't have to materialize a zero constant for this case and can just
1673 // use %x again on the RHS.
1674 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1675 const auto *CmpRHSC = dyn_cast<ConstantFP>(Val: CmpRHS);
1676 if (CmpRHSC && CmpRHSC->isNullValue())
1677 CmpRHS = CmpLHS;
1678 }
1679
1680 // Try to take advantage of fallthrough opportunities.
1681 if (FuncInfo.MBB->isLayoutSuccessor(MBB: TrueMBB)) {
1682 std::swap(a&: TrueMBB, b&: FalseMBB);
1683 Predicate = CmpInst::getInversePredicate(pred: Predicate);
1684 }
1685
1686 // FCMP_OEQ and FCMP_UNE cannot be expressed with a single flag/condition
1687 // code check. Instead two branch instructions are required to check all
1688 // the flags. First we change the predicate to a supported condition code,
1689 // which will be the first branch. Later one we will emit the second
1690 // branch.
1691 bool NeedExtraBranch = false;
1692 switch (Predicate) {
1693 default: break;
1694 case CmpInst::FCMP_OEQ:
1695 std::swap(a&: TrueMBB, b&: FalseMBB);
1696 [[fallthrough]];
1697 case CmpInst::FCMP_UNE:
1698 NeedExtraBranch = true;
1699 Predicate = CmpInst::FCMP_ONE;
1700 break;
1701 }
1702
1703 bool SwapArgs;
1704 std::tie(args&: CC, args&: SwapArgs) = X86::getX86ConditionCode(Predicate);
1705 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1706
1707 if (SwapArgs)
1708 std::swap(a&: CmpLHS, b&: CmpRHS);
1709
1710 // Emit a compare of the LHS and RHS, setting the flags.
1711 if (!X86FastEmitCompare(Op0: CmpLHS, Op1: CmpRHS, VT, CurMIMD: CI->getDebugLoc()))
1712 return false;
1713
1714 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1715 .addMBB(MBB: TrueMBB).addImm(Val: CC);
1716
1717 // X86 requires a second branch to handle UNE (and OEQ, which is mapped
1718 // to UNE above).
1719 if (NeedExtraBranch) {
1720 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1721 .addMBB(MBB: TrueMBB).addImm(Val: X86::COND_P);
1722 }
1723
1724 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1725 return true;
1726 }
1727 } else if (TruncInst *TI = dyn_cast<TruncInst>(Val: BI->getCondition())) {
1728 // Handle things like "%cond = trunc i32 %X to i1 / br i1 %cond", which
1729 // typically happen for _Bool and C++ bools.
1730 MVT SourceVT;
1731 if (TI->hasOneUse() && TI->getParent() == I->getParent() &&
1732 isTypeLegal(Ty: TI->getOperand(i_nocapture: 0)->getType(), VT&: SourceVT)) {
1733 unsigned TestOpc = 0;
1734 switch (SourceVT.SimpleTy) {
1735 default: break;
1736 case MVT::i8: TestOpc = X86::TEST8ri; break;
1737 case MVT::i16: TestOpc = X86::TEST16ri; break;
1738 case MVT::i32: TestOpc = X86::TEST32ri; break;
1739 case MVT::i64: TestOpc = X86::TEST64ri32; break;
1740 }
1741 if (TestOpc) {
1742 Register OpReg = getRegForValue(V: TI->getOperand(i_nocapture: 0));
1743 if (!OpReg)
1744 return false;
1745
1746 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TestOpc))
1747 .addReg(RegNo: OpReg).addImm(Val: 1);
1748
1749 unsigned JmpCond = X86::COND_NE;
1750 if (FuncInfo.MBB->isLayoutSuccessor(MBB: TrueMBB)) {
1751 std::swap(a&: TrueMBB, b&: FalseMBB);
1752 JmpCond = X86::COND_E;
1753 }
1754
1755 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1756 .addMBB(MBB: TrueMBB).addImm(Val: JmpCond);
1757
1758 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1759 return true;
1760 }
1761 }
1762 } else if (foldX86XALUIntrinsic(CC, I: BI, Cond: BI->getCondition())) {
1763 // Fake request the condition, otherwise the intrinsic might be completely
1764 // optimized away.
1765 Register TmpReg = getRegForValue(V: BI->getCondition());
1766 if (!TmpReg)
1767 return false;
1768
1769 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1770 .addMBB(MBB: TrueMBB).addImm(Val: CC);
1771 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1772 return true;
1773 }
1774
1775 // Otherwise do a clumsy setcc and re-test it.
1776 // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used
1777 // in an explicit cast, so make sure to handle that correctly.
1778 Register OpReg = getRegForValue(V: BI->getCondition());
1779 if (!OpReg)
1780 return false;
1781
1782 // In case OpReg is a K register, COPY to a GPR
1783 if (MRI.getRegClass(Reg: OpReg) == &X86::VK1RegClass) {
1784 Register KOpReg = OpReg;
1785 OpReg = createResultReg(RC: &X86::GR32RegClass);
1786 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1787 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: OpReg)
1788 .addReg(RegNo: KOpReg);
1789 OpReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: OpReg, Idx: X86::sub_8bit);
1790 }
1791 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TEST8ri))
1792 .addReg(RegNo: OpReg)
1793 .addImm(Val: 1);
1794 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::JCC_1))
1795 .addMBB(MBB: TrueMBB).addImm(Val: X86::COND_NE);
1796 finishCondBranch(BranchBB: BI->getParent(), TrueMBB, FalseMBB);
1797 return true;
1798}
1799
1800bool X86FastISel::X86SelectShift(const Instruction *I) {
1801 Register CReg;
1802 unsigned OpReg;
1803 const TargetRegisterClass *RC = nullptr;
1804 if (I->getType()->isIntegerTy(BitWidth: 8)) {
1805 CReg = X86::CL;
1806 RC = &X86::GR8RegClass;
1807 switch (I->getOpcode()) {
1808 case Instruction::LShr: OpReg = X86::SHR8rCL; break;
1809 case Instruction::AShr: OpReg = X86::SAR8rCL; break;
1810 case Instruction::Shl: OpReg = X86::SHL8rCL; break;
1811 default: return false;
1812 }
1813 } else if (I->getType()->isIntegerTy(BitWidth: 16)) {
1814 CReg = X86::CX;
1815 RC = &X86::GR16RegClass;
1816 switch (I->getOpcode()) {
1817 default: llvm_unreachable("Unexpected shift opcode");
1818 case Instruction::LShr: OpReg = X86::SHR16rCL; break;
1819 case Instruction::AShr: OpReg = X86::SAR16rCL; break;
1820 case Instruction::Shl: OpReg = X86::SHL16rCL; break;
1821 }
1822 } else if (I->getType()->isIntegerTy(BitWidth: 32)) {
1823 CReg = X86::ECX;
1824 RC = &X86::GR32RegClass;
1825 switch (I->getOpcode()) {
1826 default: llvm_unreachable("Unexpected shift opcode");
1827 case Instruction::LShr: OpReg = X86::SHR32rCL; break;
1828 case Instruction::AShr: OpReg = X86::SAR32rCL; break;
1829 case Instruction::Shl: OpReg = X86::SHL32rCL; break;
1830 }
1831 } else if (I->getType()->isIntegerTy(BitWidth: 64)) {
1832 CReg = X86::RCX;
1833 RC = &X86::GR64RegClass;
1834 switch (I->getOpcode()) {
1835 default: llvm_unreachable("Unexpected shift opcode");
1836 case Instruction::LShr: OpReg = X86::SHR64rCL; break;
1837 case Instruction::AShr: OpReg = X86::SAR64rCL; break;
1838 case Instruction::Shl: OpReg = X86::SHL64rCL; break;
1839 }
1840 } else {
1841 return false;
1842 }
1843
1844 MVT VT;
1845 if (!isTypeLegal(Ty: I->getType(), VT))
1846 return false;
1847
1848 Register Op0Reg = getRegForValue(V: I->getOperand(i: 0));
1849 if (!Op0Reg)
1850 return false;
1851
1852 Register Op1Reg = getRegForValue(V: I->getOperand(i: 1));
1853 if (!Op1Reg)
1854 return false;
1855 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
1856 DestReg: CReg).addReg(RegNo: Op1Reg);
1857
1858 // The shift instruction uses X86::CL. If we defined a super-register
1859 // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.
1860 if (CReg != X86::CL)
1861 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1862 MCID: TII.get(Opcode: TargetOpcode::KILL), DestReg: X86::CL)
1863 .addReg(RegNo: CReg, Flags: RegState::Kill);
1864
1865 Register ResultReg = createResultReg(RC);
1866 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: OpReg), DestReg: ResultReg)
1867 .addReg(RegNo: Op0Reg);
1868 updateValueMap(I, Reg: ResultReg);
1869 return true;
1870}
1871
1872bool X86FastISel::X86SelectDivRem(const Instruction *I) {
1873 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1874 const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem
1875 const static bool S = true; // IsSigned
1876 const static bool U = false; // !IsSigned
1877 const static unsigned Copy = TargetOpcode::COPY;
1878 // For the X86 DIV/IDIV instruction, in most cases the dividend
1879 // (numerator) must be in a specific register pair highreg:lowreg,
1880 // producing the quotient in lowreg and the remainder in highreg.
1881 // For most data types, to set up the instruction, the dividend is
1882 // copied into lowreg, and lowreg is sign-extended or zero-extended
1883 // into highreg. The exception is i8, where the dividend is defined
1884 // as a single register rather than a register pair, and we
1885 // therefore directly sign-extend or zero-extend the dividend into
1886 // lowreg, instead of copying, and ignore the highreg.
1887 const static struct DivRemEntry {
1888 // The following portion depends only on the data type.
1889 const TargetRegisterClass *RC;
1890 unsigned LowInReg; // low part of the register pair
1891 unsigned HighInReg; // high part of the register pair
1892 // The following portion depends on both the data type and the operation.
1893 struct DivRemResult {
1894 unsigned OpDivRem; // The specific DIV/IDIV opcode to use.
1895 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1896 // highreg, or copying a zero into highreg.
1897 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1898 // zero/sign-extending into lowreg for i8.
1899 unsigned DivRemResultReg; // Register containing the desired result.
1900 bool IsOpSigned; // Whether to use signed or unsigned form.
1901 } ResultTable[NumOps];
1902 } OpTable[NumTypes] = {
1903 { .RC: &X86::GR8RegClass, .LowInReg: X86::AX, .HighInReg: 0, .ResultTable: {
1904 { .OpDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .DivRemResultReg: X86::AL, .IsOpSigned: S }, // SDiv
1905 { .OpDivRem: X86::IDIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVSX16rr8, .DivRemResultReg: X86::AH, .IsOpSigned: S }, // SRem
1906 { .OpDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .DivRemResultReg: X86::AL, .IsOpSigned: U }, // UDiv
1907 { .OpDivRem: X86::DIV8r, .OpSignExtend: 0, .OpCopy: X86::MOVZX16rr8, .DivRemResultReg: X86::AH, .IsOpSigned: U }, // URem
1908 }
1909 }, // i8
1910 { .RC: &X86::GR16RegClass, .LowInReg: X86::AX, .HighInReg: X86::DX, .ResultTable: {
1911 { .OpDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .DivRemResultReg: X86::AX, .IsOpSigned: S }, // SDiv
1912 { .OpDivRem: X86::IDIV16r, .OpSignExtend: X86::CWD, .OpCopy: Copy, .DivRemResultReg: X86::DX, .IsOpSigned: S }, // SRem
1913 { .OpDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::AX, .IsOpSigned: U }, // UDiv
1914 { .OpDivRem: X86::DIV16r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::DX, .IsOpSigned: U }, // URem
1915 }
1916 }, // i16
1917 { .RC: &X86::GR32RegClass, .LowInReg: X86::EAX, .HighInReg: X86::EDX, .ResultTable: {
1918 { .OpDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .DivRemResultReg: X86::EAX, .IsOpSigned: S }, // SDiv
1919 { .OpDivRem: X86::IDIV32r, .OpSignExtend: X86::CDQ, .OpCopy: Copy, .DivRemResultReg: X86::EDX, .IsOpSigned: S }, // SRem
1920 { .OpDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::EAX, .IsOpSigned: U }, // UDiv
1921 { .OpDivRem: X86::DIV32r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::EDX, .IsOpSigned: U }, // URem
1922 }
1923 }, // i32
1924 { .RC: &X86::GR64RegClass, .LowInReg: X86::RAX, .HighInReg: X86::RDX, .ResultTable: {
1925 { .OpDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .DivRemResultReg: X86::RAX, .IsOpSigned: S }, // SDiv
1926 { .OpDivRem: X86::IDIV64r, .OpSignExtend: X86::CQO, .OpCopy: Copy, .DivRemResultReg: X86::RDX, .IsOpSigned: S }, // SRem
1927 { .OpDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::RAX, .IsOpSigned: U }, // UDiv
1928 { .OpDivRem: X86::DIV64r, .OpSignExtend: X86::MOV32r0, .OpCopy: Copy, .DivRemResultReg: X86::RDX, .IsOpSigned: U }, // URem
1929 }
1930 }, // i64
1931 };
1932
1933 MVT VT;
1934 if (!isTypeLegal(Ty: I->getType(), VT))
1935 return false;
1936
1937 unsigned TypeIndex, OpIndex;
1938 switch (VT.SimpleTy) {
1939 default: return false;
1940 case MVT::i8: TypeIndex = 0; break;
1941 case MVT::i16: TypeIndex = 1; break;
1942 case MVT::i32: TypeIndex = 2; break;
1943 case MVT::i64: TypeIndex = 3;
1944 if (!Subtarget->is64Bit())
1945 return false;
1946 break;
1947 }
1948
1949 switch (I->getOpcode()) {
1950 default: llvm_unreachable("Unexpected div/rem opcode");
1951 case Instruction::SDiv: OpIndex = 0; break;
1952 case Instruction::SRem: OpIndex = 1; break;
1953 case Instruction::UDiv: OpIndex = 2; break;
1954 case Instruction::URem: OpIndex = 3; break;
1955 }
1956
1957 const DivRemEntry &TypeEntry = OpTable[TypeIndex];
1958 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1959 Register Op0Reg = getRegForValue(V: I->getOperand(i: 0));
1960 if (!Op0Reg)
1961 return false;
1962 Register Op1Reg = getRegForValue(V: I->getOperand(i: 1));
1963 if (!Op1Reg)
1964 return false;
1965
1966 // Move op0 into low-order input register.
1967 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1968 MCID: TII.get(Opcode: OpEntry.OpCopy), DestReg: TypeEntry.LowInReg).addReg(RegNo: Op0Reg);
1969 // Zero-extend or sign-extend into high-order input register.
1970 if (OpEntry.OpSignExtend) {
1971 if (OpEntry.IsOpSigned)
1972 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1973 MCID: TII.get(Opcode: OpEntry.OpSignExtend));
1974 else {
1975 Register Zero32 = emitMOV32r0();
1976
1977 // Copy the zero into the appropriate sub/super/identical physical
1978 // register. Unfortunately the operations needed are not uniform enough
1979 // to fit neatly into the table above.
1980 if (VT == MVT::i16) {
1981 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Copy),
1982 DestReg: TypeEntry.HighInReg)
1983 .addReg(RegNo: Zero32, Flags: {}, SubReg: X86::sub_16bit);
1984 } else if (VT == MVT::i32) {
1985 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1986 MCID: TII.get(Opcode: Copy), DestReg: TypeEntry.HighInReg)
1987 .addReg(RegNo: Zero32);
1988 } else if (VT == MVT::i64) {
1989 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1990 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: TypeEntry.HighInReg)
1991 .addReg(RegNo: Zero32)
1992 .addImm(Val: X86::sub_32bit);
1993 }
1994 }
1995 }
1996 // Generate the DIV/IDIV instruction.
1997 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
1998 MCID: TII.get(Opcode: OpEntry.OpDivRem)).addReg(RegNo: Op1Reg);
1999 // For i8 remainder, we can't reference ah directly, as we'll end
2000 // up with bogus copies like %r9b = COPY %ah. Reference ax
2001 // instead to prevent ah references in a rex instruction.
2002 //
2003 // The current assumption of the fast register allocator is that isel
2004 // won't generate explicit references to the GR8_NOREX registers. If
2005 // the allocator and/or the backend get enhanced to be more robust in
2006 // that regard, this can be, and should be, removed.
2007 Register ResultReg;
2008 if ((I->getOpcode() == Instruction::SRem ||
2009 I->getOpcode() == Instruction::URem) &&
2010 OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
2011 Register SourceSuperReg = createResultReg(RC: &X86::GR16RegClass);
2012 Register ResultSuperReg = createResultReg(RC: &X86::GR16RegClass);
2013 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2014 MCID: TII.get(Opcode: Copy), DestReg: SourceSuperReg).addReg(RegNo: X86::AX);
2015
2016 // Shift AX right by 8 bits instead of using AH.
2017 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::SHR16ri),
2018 DestReg: ResultSuperReg).addReg(RegNo: SourceSuperReg).addImm(Val: 8);
2019
2020 // Now reference the 8-bit subreg of the result.
2021 ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: ResultSuperReg,
2022 Idx: X86::sub_8bit);
2023 }
2024 // Copy the result out of the physreg if we haven't already.
2025 if (!ResultReg) {
2026 ResultReg = createResultReg(RC: TypeEntry.RC);
2027 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Copy), DestReg: ResultReg)
2028 .addReg(RegNo: OpEntry.DivRemResultReg);
2029 }
2030 updateValueMap(I, Reg: ResultReg);
2031
2032 return true;
2033}
2034
2035/// Emit a conditional move instruction (if the are supported) to lower
2036/// the select.
2037bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
2038 // Check if the subtarget supports these instructions.
2039 if (!Subtarget->canUseCMOV())
2040 return false;
2041
2042 // FIXME: Add support for i8.
2043 if (RetVT < MVT::i16 || RetVT > MVT::i64)
2044 return false;
2045
2046 const Value *Cond = I->getOperand(i: 0);
2047 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2048 bool NeedTest = true;
2049 X86::CondCode CC = X86::COND_NE;
2050
2051 // Optimize conditions coming from a compare if both instructions are in the
2052 // same basic block (values defined in other basic blocks may not have
2053 // initialized registers).
2054 const auto *CI = dyn_cast<CmpInst>(Val: Cond);
2055 if (CI && (CI->getParent() == I->getParent())) {
2056 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2057
2058 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
2059 static const uint16_t SETFOpcTable[2][3] = {
2060 { X86::COND_NP, X86::COND_E, X86::TEST8rr },
2061 { X86::COND_P, X86::COND_NE, X86::OR8rr }
2062 };
2063 const uint16_t *SETFOpc = nullptr;
2064 switch (Predicate) {
2065 default: break;
2066 case CmpInst::FCMP_OEQ:
2067 SETFOpc = &SETFOpcTable[0][0];
2068 Predicate = CmpInst::ICMP_NE;
2069 break;
2070 case CmpInst::FCMP_UNE:
2071 SETFOpc = &SETFOpcTable[1][0];
2072 Predicate = CmpInst::ICMP_NE;
2073 break;
2074 }
2075
2076 bool NeedSwap;
2077 std::tie(args&: CC, args&: NeedSwap) = X86::getX86ConditionCode(Predicate);
2078 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
2079
2080 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
2081 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
2082 if (NeedSwap)
2083 std::swap(a&: CmpLHS, b&: CmpRHS);
2084
2085 EVT CmpVT = TLI.getValueType(DL, Ty: CmpLHS->getType());
2086 // Emit a compare of the LHS and RHS, setting the flags.
2087 if (!X86FastEmitCompare(Op0: CmpLHS, Op1: CmpRHS, VT: CmpVT, CurMIMD: CI->getDebugLoc()))
2088 return false;
2089
2090 if (SETFOpc) {
2091 Register FlagReg1 = createResultReg(RC: &X86::GR8RegClass);
2092 Register FlagReg2 = createResultReg(RC: &X86::GR8RegClass);
2093 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
2094 DestReg: FlagReg1)
2095 .addImm(Val: SETFOpc[0]);
2096 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
2097 DestReg: FlagReg2)
2098 .addImm(Val: SETFOpc[1]);
2099 auto const &II = TII.get(Opcode: SETFOpc[2]);
2100 if (II.getNumDefs()) {
2101 Register TmpReg = createResultReg(RC: &X86::GR8RegClass);
2102 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: TmpReg)
2103 .addReg(RegNo: FlagReg2).addReg(RegNo: FlagReg1);
2104 } else {
2105 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
2106 .addReg(RegNo: FlagReg2).addReg(RegNo: FlagReg1);
2107 }
2108 }
2109 NeedTest = false;
2110 } else if (foldX86XALUIntrinsic(CC, I, Cond)) {
2111 // Fake request the condition, otherwise the intrinsic might be completely
2112 // optimized away.
2113 Register TmpReg = getRegForValue(V: Cond);
2114 if (!TmpReg)
2115 return false;
2116
2117 NeedTest = false;
2118 }
2119
2120 if (NeedTest) {
2121 // Selects operate on i1, however, CondReg is 8 bits width and may contain
2122 // garbage. Indeed, only the less significant bit is supposed to be
2123 // accurate. If we read more than the lsb, we may see non-zero values
2124 // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for
2125 // the select. This is achieved by performing TEST against 1.
2126 Register CondReg = getRegForValue(V: Cond);
2127 if (!CondReg)
2128 return false;
2129
2130 // In case OpReg is a K register, COPY to a GPR
2131 if (MRI.getRegClass(Reg: CondReg) == &X86::VK1RegClass) {
2132 Register KCondReg = CondReg;
2133 CondReg = createResultReg(RC: &X86::GR32RegClass);
2134 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2135 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CondReg)
2136 .addReg(RegNo: KCondReg);
2137 CondReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: CondReg, Idx: X86::sub_8bit);
2138 }
2139 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TEST8ri))
2140 .addReg(RegNo: CondReg)
2141 .addImm(Val: 1);
2142 }
2143
2144 const Value *LHS = I->getOperand(i: 1);
2145 const Value *RHS = I->getOperand(i: 2);
2146
2147 Register RHSReg = getRegForValue(V: RHS);
2148 Register LHSReg = getRegForValue(V: LHS);
2149 if (!LHSReg || !RHSReg)
2150 return false;
2151
2152 const TargetRegisterInfo &TRI = *Subtarget->getRegisterInfo();
2153 unsigned Opc = X86::getCMovOpcode(RegBytes: TRI.getRegSizeInBits(RC: *RC) / 8, HasMemoryOperand: false,
2154 HasNDD: Subtarget->hasNDD());
2155 Register ResultReg = fastEmitInst_rri(MachineInstOpcode: Opc, RC, Op0: RHSReg, Op1: LHSReg, Imm: CC);
2156 updateValueMap(I, Reg: ResultReg);
2157 return true;
2158}
2159
2160/// Emit SSE or AVX instructions to lower the select.
2161///
2162/// Try to use SSE1/SSE2 instructions to simulate a select without branches.
2163/// This lowers fp selects into a CMP/AND/ANDN/OR sequence when the necessary
2164/// SSE instructions are available. If AVX is available, try to use a VBLENDV.
2165bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
2166 // Optimize conditions coming from a compare if both instructions are in the
2167 // same basic block (values defined in other basic blocks may not have
2168 // initialized registers).
2169 const auto *CI = dyn_cast<FCmpInst>(Val: I->getOperand(i: 0));
2170 if (!CI || (CI->getParent() != I->getParent()))
2171 return false;
2172
2173 if (I->getType() != CI->getOperand(i_nocapture: 0)->getType() ||
2174 !((Subtarget->hasSSE1() && RetVT == MVT::f32) ||
2175 (Subtarget->hasSSE2() && RetVT == MVT::f64)))
2176 return false;
2177
2178 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
2179 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
2180 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2181
2182 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
2183 // We don't have to materialize a zero constant for this case and can just use
2184 // %x again on the RHS.
2185 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
2186 const auto *CmpRHSC = dyn_cast<ConstantFP>(Val: CmpRHS);
2187 if (CmpRHSC && CmpRHSC->isNullValue())
2188 CmpRHS = CmpLHS;
2189 }
2190
2191 unsigned CC;
2192 bool NeedSwap;
2193 std::tie(args&: CC, args&: NeedSwap) = getX86SSEConditionCode(Predicate);
2194 if (CC > 7 && !Subtarget->hasAVX())
2195 return false;
2196
2197 if (NeedSwap)
2198 std::swap(a&: CmpLHS, b&: CmpRHS);
2199
2200 const Value *LHS = I->getOperand(i: 1);
2201 const Value *RHS = I->getOperand(i: 2);
2202
2203 Register LHSReg = getRegForValue(V: LHS);
2204 Register RHSReg = getRegForValue(V: RHS);
2205 Register CmpLHSReg = getRegForValue(V: CmpLHS);
2206 Register CmpRHSReg = getRegForValue(V: CmpRHS);
2207 if (!LHSReg || !RHSReg || !CmpLHSReg || !CmpRHSReg)
2208 return false;
2209
2210 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2211 Register ResultReg;
2212
2213 if (Subtarget->hasAVX512()) {
2214 // If we have AVX512 we can use a mask compare and masked movss/sd.
2215 const TargetRegisterClass *VR128X = &X86::VR128XRegClass;
2216 const TargetRegisterClass *VK1 = &X86::VK1RegClass;
2217
2218 unsigned CmpOpcode =
2219 (RetVT == MVT::f32) ? X86::VCMPSSZrri : X86::VCMPSDZrri;
2220 Register CmpReg = fastEmitInst_rri(MachineInstOpcode: CmpOpcode, RC: VK1, Op0: CmpLHSReg, Op1: CmpRHSReg,
2221 Imm: CC);
2222
2223 // Need an IMPLICIT_DEF for the input that is used to generate the upper
2224 // bits of the result register since its not based on any of the inputs.
2225 Register ImplicitDefReg = createResultReg(RC: VR128X);
2226 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2227 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2228
2229 // Place RHSReg is the passthru of the masked movss/sd operation and put
2230 // LHS in the input. The mask input comes from the compare.
2231 unsigned MovOpcode =
2232 (RetVT == MVT::f32) ? X86::VMOVSSZrrk : X86::VMOVSDZrrk;
2233 Register MovReg = fastEmitInst_rrrr(MachineInstOpcode: MovOpcode, RC: VR128X, Op0: RHSReg, Op1: CmpReg,
2234 Op2: ImplicitDefReg, Op3: LHSReg);
2235
2236 ResultReg = createResultReg(RC);
2237 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2238 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: MovReg);
2239
2240 } else if (Subtarget->hasAVX()) {
2241 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2242
2243 // If we have AVX, create 1 blendv instead of 3 logic instructions.
2244 // Blendv was introduced with SSE 4.1, but the 2 register form implicitly
2245 // uses XMM0 as the selection register. That may need just as many
2246 // instructions as the AND/ANDN/OR sequence due to register moves, so
2247 // don't bother.
2248 unsigned CmpOpcode =
2249 (RetVT == MVT::f32) ? X86::VCMPSSrri : X86::VCMPSDrri;
2250 unsigned BlendOpcode =
2251 (RetVT == MVT::f32) ? X86::VBLENDVPSrrr : X86::VBLENDVPDrrr;
2252
2253 Register CmpReg = fastEmitInst_rri(MachineInstOpcode: CmpOpcode, RC, Op0: CmpLHSReg, Op1: CmpRHSReg,
2254 Imm: CC);
2255 Register VBlendReg = fastEmitInst_rrr(MachineInstOpcode: BlendOpcode, RC: VR128, Op0: RHSReg, Op1: LHSReg,
2256 Op2: CmpReg);
2257 ResultReg = createResultReg(RC);
2258 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2259 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: VBlendReg);
2260 } else {
2261 // Choose the SSE instruction sequence based on data type (float or double).
2262 static const uint16_t OpcTable[2][4] = {
2263 { X86::CMPSSrri, X86::ANDPSrr, X86::ANDNPSrr, X86::ORPSrr },
2264 { X86::CMPSDrri, X86::ANDPDrr, X86::ANDNPDrr, X86::ORPDrr }
2265 };
2266
2267 const uint16_t *Opc = nullptr;
2268 switch (RetVT.SimpleTy) {
2269 default: return false;
2270 case MVT::f32: Opc = &OpcTable[0][0]; break;
2271 case MVT::f64: Opc = &OpcTable[1][0]; break;
2272 }
2273
2274 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2275 Register CmpReg = fastEmitInst_rri(MachineInstOpcode: Opc[0], RC, Op0: CmpLHSReg, Op1: CmpRHSReg, Imm: CC);
2276 Register AndReg = fastEmitInst_rr(MachineInstOpcode: Opc[1], RC: VR128, Op0: CmpReg, Op1: LHSReg);
2277 Register AndNReg = fastEmitInst_rr(MachineInstOpcode: Opc[2], RC: VR128, Op0: CmpReg, Op1: RHSReg);
2278 Register OrReg = fastEmitInst_rr(MachineInstOpcode: Opc[3], RC: VR128, Op0: AndNReg, Op1: AndReg);
2279 ResultReg = createResultReg(RC);
2280 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2281 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg).addReg(RegNo: OrReg);
2282 }
2283 updateValueMap(I, Reg: ResultReg);
2284 return true;
2285}
2286
2287bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
2288 // These are pseudo CMOV instructions and will be later expanded into control-
2289 // flow.
2290 unsigned Opc;
2291 switch (RetVT.SimpleTy) {
2292 default: return false;
2293 case MVT::i8: Opc = X86::CMOV_GR8; break;
2294 case MVT::i16: Opc = X86::CMOV_GR16; break;
2295 case MVT::i32: Opc = X86::CMOV_GR32; break;
2296 case MVT::f16:
2297 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR16X : X86::CMOV_FR16; break;
2298 case MVT::f32:
2299 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR32X : X86::CMOV_FR32; break;
2300 case MVT::f64:
2301 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR64X : X86::CMOV_FR64; break;
2302 }
2303
2304 const Value *Cond = I->getOperand(i: 0);
2305 X86::CondCode CC = X86::COND_NE;
2306
2307 // Optimize conditions coming from a compare if both instructions are in the
2308 // same basic block (values defined in other basic blocks may not have
2309 // initialized registers).
2310 const auto *CI = dyn_cast<CmpInst>(Val: Cond);
2311 if (CI && (CI->getParent() == I->getParent())) {
2312 bool NeedSwap;
2313 std::tie(args&: CC, args&: NeedSwap) = X86::getX86ConditionCode(Predicate: CI->getPredicate());
2314 if (CC > X86::LAST_VALID_COND)
2315 return false;
2316
2317 const Value *CmpLHS = CI->getOperand(i_nocapture: 0);
2318 const Value *CmpRHS = CI->getOperand(i_nocapture: 1);
2319
2320 if (NeedSwap)
2321 std::swap(a&: CmpLHS, b&: CmpRHS);
2322
2323 EVT CmpVT = TLI.getValueType(DL, Ty: CmpLHS->getType());
2324 if (!X86FastEmitCompare(Op0: CmpLHS, Op1: CmpRHS, VT: CmpVT, CurMIMD: CI->getDebugLoc()))
2325 return false;
2326 } else {
2327 Register CondReg = getRegForValue(V: Cond);
2328 if (!CondReg)
2329 return false;
2330
2331 // In case OpReg is a K register, COPY to a GPR
2332 if (MRI.getRegClass(Reg: CondReg) == &X86::VK1RegClass) {
2333 Register KCondReg = CondReg;
2334 CondReg = createResultReg(RC: &X86::GR32RegClass);
2335 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2336 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CondReg)
2337 .addReg(RegNo: KCondReg);
2338 CondReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: CondReg, Idx: X86::sub_8bit);
2339 }
2340 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TEST8ri))
2341 .addReg(RegNo: CondReg)
2342 .addImm(Val: 1);
2343 }
2344
2345 const Value *LHS = I->getOperand(i: 1);
2346 const Value *RHS = I->getOperand(i: 2);
2347
2348 Register LHSReg = getRegForValue(V: LHS);
2349 Register RHSReg = getRegForValue(V: RHS);
2350 if (!LHSReg || !RHSReg)
2351 return false;
2352
2353 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2354
2355 Register ResultReg =
2356 fastEmitInst_rri(MachineInstOpcode: Opc, RC, Op0: RHSReg, Op1: LHSReg, Imm: CC);
2357 updateValueMap(I, Reg: ResultReg);
2358 return true;
2359}
2360
2361bool X86FastISel::X86SelectSelect(const Instruction *I) {
2362 MVT RetVT;
2363 if (!isTypeLegal(Ty: I->getType(), VT&: RetVT))
2364 return false;
2365
2366 // Check if we can fold the select.
2367 if (const auto *CI = dyn_cast<CmpInst>(Val: I->getOperand(i: 0))) {
2368 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2369 const Value *Opnd = nullptr;
2370 switch (Predicate) {
2371 default: break;
2372 case CmpInst::FCMP_FALSE: Opnd = I->getOperand(i: 2); break;
2373 case CmpInst::FCMP_TRUE: Opnd = I->getOperand(i: 1); break;
2374 }
2375 // No need for a select anymore - this is an unconditional move.
2376 if (Opnd) {
2377 Register OpReg = getRegForValue(V: Opnd);
2378 if (!OpReg)
2379 return false;
2380 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: RetVT);
2381 Register ResultReg = createResultReg(RC);
2382 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2383 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg)
2384 .addReg(RegNo: OpReg);
2385 updateValueMap(I, Reg: ResultReg);
2386 return true;
2387 }
2388 }
2389
2390 // First try to use real conditional move instructions.
2391 if (X86FastEmitCMoveSelect(RetVT, I))
2392 return true;
2393
2394 // Try to use a sequence of SSE instructions to simulate a conditional move.
2395 if (X86FastEmitSSESelect(RetVT, I))
2396 return true;
2397
2398 // Fall-back to pseudo conditional move instructions, which will be later
2399 // converted to control-flow.
2400 if (X86FastEmitPseudoSelect(RetVT, I))
2401 return true;
2402
2403 return false;
2404}
2405
2406// Common code for X86SelectSIToFP and X86SelectUIToFP.
2407bool X86FastISel::X86SelectIntToFP(const Instruction *I, bool IsSigned) {
2408 // The target-independent selection algorithm in FastISel already knows how
2409 // to select a SINT_TO_FP if the target is SSE but not AVX.
2410 // Early exit if the subtarget doesn't have AVX.
2411 // Unsigned conversion requires avx512.
2412 bool HasAVX512 = Subtarget->hasAVX512();
2413 if (!Subtarget->hasAVX() || (!IsSigned && !HasAVX512))
2414 return false;
2415
2416 // TODO: We could sign extend narrower types.
2417 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
2418 if (SrcVT != MVT::i32 && SrcVT != MVT::i64)
2419 return false;
2420
2421 // Select integer to float/double conversion.
2422 Register OpReg = getRegForValue(V: I->getOperand(i: 0));
2423 if (!OpReg)
2424 return false;
2425
2426 unsigned Opcode;
2427
2428 static const uint16_t SCvtOpc[2][2][2] = {
2429 { { X86::VCVTSI2SSrr, X86::VCVTSI642SSrr },
2430 { X86::VCVTSI2SDrr, X86::VCVTSI642SDrr } },
2431 { { X86::VCVTSI2SSZrr, X86::VCVTSI642SSZrr },
2432 { X86::VCVTSI2SDZrr, X86::VCVTSI642SDZrr } },
2433 };
2434 static const uint16_t UCvtOpc[2][2] = {
2435 { X86::VCVTUSI2SSZrr, X86::VCVTUSI642SSZrr },
2436 { X86::VCVTUSI2SDZrr, X86::VCVTUSI642SDZrr },
2437 };
2438 bool Is64Bit = SrcVT == MVT::i64;
2439
2440 if (I->getType()->isDoubleTy()) {
2441 // s/uitofp int -> double
2442 Opcode = IsSigned ? SCvtOpc[HasAVX512][1][Is64Bit] : UCvtOpc[1][Is64Bit];
2443 } else if (I->getType()->isFloatTy()) {
2444 // s/uitofp int -> float
2445 Opcode = IsSigned ? SCvtOpc[HasAVX512][0][Is64Bit] : UCvtOpc[0][Is64Bit];
2446 } else
2447 return false;
2448
2449 MVT DstVT = TLI.getValueType(DL, Ty: I->getType()).getSimpleVT();
2450 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: DstVT);
2451 Register ImplicitDefReg = createResultReg(RC);
2452 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2453 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2454 Register ResultReg = fastEmitInst_rr(MachineInstOpcode: Opcode, RC, Op0: ImplicitDefReg, Op1: OpReg);
2455 updateValueMap(I, Reg: ResultReg);
2456 return true;
2457}
2458
2459bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
2460 return X86SelectIntToFP(I, /*IsSigned*/true);
2461}
2462
2463bool X86FastISel::X86SelectUIToFP(const Instruction *I) {
2464 return X86SelectIntToFP(I, /*IsSigned*/false);
2465}
2466
2467// Helper method used by X86SelectFPExt and X86SelectFPTrunc.
2468bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction *I,
2469 unsigned TargetOpc,
2470 const TargetRegisterClass *RC) {
2471 assert((I->getOpcode() == Instruction::FPExt ||
2472 I->getOpcode() == Instruction::FPTrunc) &&
2473 "Instruction must be an FPExt or FPTrunc!");
2474 bool HasAVX = Subtarget->hasAVX();
2475
2476 Register OpReg = getRegForValue(V: I->getOperand(i: 0));
2477 if (!OpReg)
2478 return false;
2479
2480 Register ImplicitDefReg;
2481 if (HasAVX) {
2482 ImplicitDefReg = createResultReg(RC);
2483 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2484 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2485
2486 }
2487
2488 Register ResultReg = createResultReg(RC);
2489 MachineInstrBuilder MIB;
2490 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpc),
2491 DestReg: ResultReg);
2492
2493 if (HasAVX)
2494 MIB.addReg(RegNo: ImplicitDefReg);
2495
2496 MIB.addReg(RegNo: OpReg);
2497 updateValueMap(I, Reg: ResultReg);
2498 return true;
2499}
2500
2501bool X86FastISel::X86SelectFPExt(const Instruction *I) {
2502 if (Subtarget->hasSSE2() && I->getType()->isDoubleTy() &&
2503 I->getOperand(i: 0)->getType()->isFloatTy()) {
2504 bool HasAVX512 = Subtarget->hasAVX512();
2505 // fpext from float to double.
2506 unsigned Opc =
2507 HasAVX512 ? X86::VCVTSS2SDZrr
2508 : Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr;
2509 return X86SelectFPExtOrFPTrunc(I, TargetOpc: Opc, RC: TLI.getRegClassFor(VT: MVT::f64));
2510 }
2511
2512 return false;
2513}
2514
2515bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
2516 if (Subtarget->hasSSE2() && I->getType()->isFloatTy() &&
2517 I->getOperand(i: 0)->getType()->isDoubleTy()) {
2518 bool HasAVX512 = Subtarget->hasAVX512();
2519 // fptrunc from double to float.
2520 unsigned Opc =
2521 HasAVX512 ? X86::VCVTSD2SSZrr
2522 : Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr;
2523 return X86SelectFPExtOrFPTrunc(I, TargetOpc: Opc, RC: TLI.getRegClassFor(VT: MVT::f32));
2524 }
2525
2526 return false;
2527}
2528
2529bool X86FastISel::X86SelectTrunc(const Instruction *I) {
2530 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
2531 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
2532
2533 // This code only handles truncation to byte.
2534 if (DstVT != MVT::i8 && DstVT != MVT::i1)
2535 return false;
2536 if (!TLI.isTypeLegal(VT: SrcVT))
2537 return false;
2538
2539 Register InputReg = getRegForValue(V: I->getOperand(i: 0));
2540 if (!InputReg)
2541 // Unhandled operand. Halt "fast" selection and bail.
2542 return false;
2543
2544 if (SrcVT == MVT::i8) {
2545 // Truncate from i8 to i1; no code needed.
2546 updateValueMap(I, Reg: InputReg);
2547 return true;
2548 }
2549
2550 // Issue an extract_subreg.
2551 Register ResultReg = fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: InputReg,
2552 Idx: X86::sub_8bit);
2553 if (!ResultReg)
2554 return false;
2555
2556 updateValueMap(I, Reg: ResultReg);
2557 return true;
2558}
2559
2560bool X86FastISel::X86SelectBitCast(const Instruction *I) {
2561 // Select SSE2/AVX bitcasts between 128/256/512 bit vector types.
2562 MVT SrcVT, DstVT;
2563 if (!Subtarget->hasSSE2() ||
2564 !isTypeLegal(Ty: I->getOperand(i: 0)->getType(), VT&: SrcVT) ||
2565 !isTypeLegal(Ty: I->getType(), VT&: DstVT))
2566 return false;
2567
2568 // Only allow vectors that use xmm/ymm/zmm.
2569 if (!SrcVT.isVector() || !DstVT.isVector() ||
2570 SrcVT.getVectorElementType() == MVT::i1 ||
2571 DstVT.getVectorElementType() == MVT::i1)
2572 return false;
2573
2574 Register Reg = getRegForValue(V: I->getOperand(i: 0));
2575 if (!Reg)
2576 return false;
2577
2578 // Emit a reg-reg copy so we don't propagate cached known bits information
2579 // with the wrong VT if we fall out of fast isel after selecting this.
2580 const TargetRegisterClass *DstClass = TLI.getRegClassFor(VT: DstVT);
2581 Register ResultReg = createResultReg(RC: DstClass);
2582 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
2583 DestReg: ResultReg)
2584 .addReg(RegNo: Reg);
2585
2586 updateValueMap(I, Reg: ResultReg);
2587 return true;
2588}
2589
2590bool X86FastISel::IsMemcpySmall(uint64_t Len) {
2591 return Len <= (Subtarget->is64Bit() ? 32 : 16);
2592}
2593
2594bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,
2595 X86AddressMode SrcAM, uint64_t Len) {
2596
2597 // Make sure we don't bloat code by inlining very large memcpy's.
2598 if (!IsMemcpySmall(Len))
2599 return false;
2600
2601 bool i64Legal = Subtarget->is64Bit();
2602
2603 // We don't care about alignment here since we just emit integer accesses.
2604 while (Len) {
2605 MVT VT;
2606 if (Len >= 8 && i64Legal)
2607 VT = MVT::i64;
2608 else if (Len >= 4)
2609 VT = MVT::i32;
2610 else if (Len >= 2)
2611 VT = MVT::i16;
2612 else
2613 VT = MVT::i8;
2614
2615 Register Reg;
2616 bool RV = X86FastEmitLoad(VT, AM&: SrcAM, MMO: nullptr, ResultReg&: Reg);
2617 RV &= X86FastEmitStore(VT, ValReg: Reg, AM&: DestAM);
2618 assert(RV && "Failed to emit load or store??");
2619 (void)RV;
2620
2621 unsigned Size = VT.getSizeInBits()/8;
2622 Len -= Size;
2623 DestAM.Disp += Size;
2624 SrcAM.Disp += Size;
2625 }
2626
2627 return true;
2628}
2629
2630bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
2631 // FIXME: Handle more intrinsics.
2632 switch (II->getIntrinsicID()) {
2633 default:
2634 return false;
2635 case Intrinsic::frameaddress: {
2636 MachineFunction *MF = FuncInfo.MF;
2637 if (MF->getTarget().getMCAsmInfo().usesWindowsCFI())
2638 return false;
2639
2640 Type *RetTy = II->getCalledFunction()->getReturnType();
2641
2642 MVT VT;
2643 if (!isTypeLegal(Ty: RetTy, VT))
2644 return false;
2645
2646 unsigned Opc;
2647 const TargetRegisterClass *RC = nullptr;
2648
2649 switch (VT.SimpleTy) {
2650 default: llvm_unreachable("Invalid result type for frameaddress.");
2651 case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass; break;
2652 case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass; break;
2653 }
2654
2655 // This needs to be set before we call getPtrSizedFrameRegister, otherwise
2656 // we get the wrong frame register.
2657 MachineFrameInfo &MFI = MF->getFrameInfo();
2658 MFI.setFrameAddressIsTaken(true);
2659
2660 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2661 Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF: *MF);
2662 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
2663 (FrameReg == X86::EBP && VT == MVT::i32)) &&
2664 "Invalid Frame Register!");
2665
2666 // Always make a copy of the frame register to a vreg first, so that we
2667 // never directly reference the frame register (the TwoAddressInstruction-
2668 // Pass doesn't like that).
2669 Register SrcReg = createResultReg(RC);
2670 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2671 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: SrcReg).addReg(RegNo: FrameReg);
2672
2673 // Now recursively load from the frame address.
2674 // movq (%rbp), %rax
2675 // movq (%rax), %rax
2676 // movq (%rax), %rax
2677 // ...
2678 unsigned Depth = cast<ConstantInt>(Val: II->getOperand(i_nocapture: 0))->getZExtValue();
2679 while (Depth--) {
2680 Register DestReg = createResultReg(RC);
2681 addDirectMem(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2682 MCID: TII.get(Opcode: Opc), DestReg), Reg: SrcReg);
2683 SrcReg = DestReg;
2684 }
2685
2686 updateValueMap(I: II, Reg: SrcReg);
2687 return true;
2688 }
2689 case Intrinsic::memcpy: {
2690 const MemCpyInst *MCI = cast<MemCpyInst>(Val: II);
2691 // Don't handle volatile or variable length memcpys.
2692 if (MCI->isVolatile())
2693 return false;
2694
2695 if (isa<ConstantInt>(Val: MCI->getLength())) {
2696 // Small memcpy's are common enough that we want to do them
2697 // without a call if possible.
2698 uint64_t Len = cast<ConstantInt>(Val: MCI->getLength())->getZExtValue();
2699 if (IsMemcpySmall(Len)) {
2700 X86AddressMode DestAM, SrcAM;
2701 if (!X86SelectAddress(V: MCI->getRawDest(), AM&: DestAM) ||
2702 !X86SelectAddress(V: MCI->getRawSource(), AM&: SrcAM))
2703 return false;
2704 TryEmitSmallMemcpy(DestAM, SrcAM, Len);
2705 return true;
2706 }
2707 }
2708
2709 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2710 if (!MCI->getLength()->getType()->isIntegerTy(BitWidth: SizeWidth))
2711 return false;
2712
2713 if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255)
2714 return false;
2715
2716 return lowerCallTo(CI: II, SymName: "memcpy", NumArgs: II->arg_size() - 1);
2717 }
2718 case Intrinsic::memset: {
2719 const MemSetInst *MSI = cast<MemSetInst>(Val: II);
2720
2721 if (MSI->isVolatile())
2722 return false;
2723
2724 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2725 if (!MSI->getLength()->getType()->isIntegerTy(BitWidth: SizeWidth))
2726 return false;
2727
2728 if (MSI->getDestAddressSpace() > 255)
2729 return false;
2730
2731 return lowerCallTo(CI: II, SymName: "memset", NumArgs: II->arg_size() - 1);
2732 }
2733 case Intrinsic::stackprotector: {
2734 // Emit code to store the stack guard onto the stack.
2735 EVT PtrTy = TLI.getPointerTy(DL);
2736
2737 const Value *Op1 = II->getArgOperand(i: 0); // The guard's value.
2738 const AllocaInst *Slot = cast<AllocaInst>(Val: II->getArgOperand(i: 1));
2739
2740 MFI.setStackProtectorIndex(FuncInfo.StaticAllocaMap[Slot]);
2741
2742 // Grab the frame index.
2743 X86AddressMode AM;
2744 if (!X86SelectAddress(V: Slot, AM)) return false;
2745 if (!X86FastEmitStore(VT: PtrTy, Val: Op1, AM)) return false;
2746 return true;
2747 }
2748 case Intrinsic::dbg_declare: {
2749 const DbgDeclareInst *DI = cast<DbgDeclareInst>(Val: II);
2750 X86AddressMode AM;
2751 assert(DI->getAddress() && "Null address should be checked earlier!");
2752 if (!X86SelectAddress(V: DI->getAddress(), AM))
2753 return false;
2754 const MCInstrDesc &II = TII.get(Opcode: TargetOpcode::DBG_VALUE);
2755 assert(DI->getVariable()->isValidLocationForIntrinsic(MIMD.getDL()) &&
2756 "Expected inlined-at fields to agree");
2757 addFullAddress(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II), AM)
2758 .addImm(Val: 0)
2759 .addMetadata(MD: DI->getVariable())
2760 .addMetadata(MD: DI->getExpression());
2761 return true;
2762 }
2763 case Intrinsic::trap: {
2764 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::TRAP));
2765 return true;
2766 }
2767 case Intrinsic::sqrt: {
2768 if (!Subtarget->hasSSE1())
2769 return false;
2770
2771 Type *RetTy = II->getCalledFunction()->getReturnType();
2772
2773 MVT VT;
2774 if (!isTypeLegal(Ty: RetTy, VT))
2775 return false;
2776
2777 // Unfortunately we can't use fastEmit_r, because the AVX version of FSQRT
2778 // is not generated by FastISel yet.
2779 // FIXME: Update this code once tablegen can handle it.
2780 static const uint16_t SqrtOpc[3][2] = {
2781 { X86::SQRTSSr, X86::SQRTSDr },
2782 { X86::VSQRTSSr, X86::VSQRTSDr },
2783 { X86::VSQRTSSZr, X86::VSQRTSDZr },
2784 };
2785 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
2786 Subtarget->hasAVX() ? 1 :
2787 0;
2788 unsigned Opc;
2789 switch (VT.SimpleTy) {
2790 default: return false;
2791 case MVT::f32: Opc = SqrtOpc[AVXLevel][0]; break;
2792 case MVT::f64: Opc = SqrtOpc[AVXLevel][1]; break;
2793 }
2794
2795 const Value *SrcVal = II->getArgOperand(i: 0);
2796 Register SrcReg = getRegForValue(V: SrcVal);
2797
2798 if (!SrcReg)
2799 return false;
2800
2801 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
2802 Register ImplicitDefReg;
2803 if (AVXLevel > 0) {
2804 ImplicitDefReg = createResultReg(RC);
2805 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2806 MCID: TII.get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImplicitDefReg);
2807 }
2808
2809 Register ResultReg = createResultReg(RC);
2810 MachineInstrBuilder MIB;
2811 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc),
2812 DestReg: ResultReg);
2813
2814 if (ImplicitDefReg)
2815 MIB.addReg(RegNo: ImplicitDefReg);
2816
2817 MIB.addReg(RegNo: SrcReg);
2818
2819 updateValueMap(I: II, Reg: ResultReg);
2820 return true;
2821 }
2822 case Intrinsic::sadd_with_overflow:
2823 case Intrinsic::uadd_with_overflow:
2824 case Intrinsic::ssub_with_overflow:
2825 case Intrinsic::usub_with_overflow:
2826 case Intrinsic::smul_with_overflow:
2827 case Intrinsic::umul_with_overflow: {
2828 // This implements the basic lowering of the xalu with overflow intrinsics
2829 // into add/sub/mul followed by either seto or setb.
2830 const Function *Callee = II->getCalledFunction();
2831 auto *Ty = cast<StructType>(Val: Callee->getReturnType());
2832 Type *RetTy = Ty->getTypeAtIndex(N: 0U);
2833 assert(Ty->getTypeAtIndex(1)->isIntegerTy() &&
2834 Ty->getTypeAtIndex(1)->getScalarSizeInBits() == 1 &&
2835 "Overflow value expected to be an i1");
2836
2837 MVT VT;
2838 if (!isTypeLegal(Ty: RetTy, VT))
2839 return false;
2840
2841 if (VT < MVT::i8 || VT > MVT::i64)
2842 return false;
2843
2844 const Value *LHS = II->getArgOperand(i: 0);
2845 const Value *RHS = II->getArgOperand(i: 1);
2846
2847 // Canonicalize immediate to the RHS.
2848 if (isa<ConstantInt>(Val: LHS) && !isa<ConstantInt>(Val: RHS) && II->isCommutative())
2849 std::swap(a&: LHS, b&: RHS);
2850
2851 unsigned BaseOpc, CondCode;
2852 switch (II->getIntrinsicID()) {
2853 default: llvm_unreachable("Unexpected intrinsic!");
2854 case Intrinsic::sadd_with_overflow:
2855 BaseOpc = ISD::ADD; CondCode = X86::COND_O; break;
2856 case Intrinsic::uadd_with_overflow:
2857 BaseOpc = ISD::ADD; CondCode = X86::COND_B; break;
2858 case Intrinsic::ssub_with_overflow:
2859 BaseOpc = ISD::SUB; CondCode = X86::COND_O; break;
2860 case Intrinsic::usub_with_overflow:
2861 BaseOpc = ISD::SUB; CondCode = X86::COND_B; break;
2862 case Intrinsic::smul_with_overflow:
2863 BaseOpc = X86ISD::SMUL; CondCode = X86::COND_B; break;
2864 case Intrinsic::umul_with_overflow:
2865 BaseOpc = X86ISD::UMUL; CondCode = X86::COND_B; break;
2866 }
2867
2868 Register LHSReg = getRegForValue(V: LHS);
2869 if (!LHSReg)
2870 return false;
2871
2872 Register ResultReg;
2873 // Check if we have an immediate version.
2874 if (const auto *CI = dyn_cast<ConstantInt>(Val: RHS)) {
2875 static const uint16_t Opc[2][4] = {
2876 { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },
2877 { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
2878 };
2879
2880 if (CI->isOne() && (BaseOpc == ISD::ADD || BaseOpc == ISD::SUB) &&
2881 CondCode == X86::COND_O) {
2882 // We can use INC/DEC.
2883 ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
2884 bool IsDec = BaseOpc == ISD::SUB;
2885 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2886 MCID: TII.get(Opcode: Opc[IsDec][VT.SimpleTy-MVT::i8]), DestReg: ResultReg)
2887 .addReg(RegNo: LHSReg);
2888 } else
2889 ResultReg = fastEmit_ri(VT, RetVT: VT, Opcode: BaseOpc, Op0: LHSReg, imm1: CI->getZExtValue());
2890 }
2891
2892 Register RHSReg;
2893 if (!ResultReg) {
2894 RHSReg = getRegForValue(V: RHS);
2895 if (!RHSReg)
2896 return false;
2897 ResultReg = fastEmit_rr(VT, RetVT: VT, Opcode: BaseOpc, Op0: LHSReg, Op1: RHSReg);
2898 }
2899
2900 // FastISel doesn't have a pattern for all X86::MUL*r and X86::IMUL*r. Emit
2901 // it manually.
2902 if (BaseOpc == X86ISD::UMUL && !ResultReg) {
2903 static const uint16_t MULOpc[] =
2904 { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };
2905 static const MCPhysReg Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };
2906 // First copy the first operand into RAX, which is an implicit input to
2907 // the X86::MUL*r instruction.
2908 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2909 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Reg[VT.SimpleTy-MVT::i8])
2910 .addReg(RegNo: LHSReg);
2911 ResultReg = fastEmitInst_r(MachineInstOpcode: MULOpc[VT.SimpleTy-MVT::i8],
2912 RC: TLI.getRegClassFor(VT), Op0: RHSReg);
2913 } else if (BaseOpc == X86ISD::SMUL && !ResultReg) {
2914 static const uint16_t MULOpc[] =
2915 { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };
2916 if (VT == MVT::i8) {
2917 // Copy the first operand into AL, which is an implicit input to the
2918 // X86::IMUL8r instruction.
2919 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
2920 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::AL)
2921 .addReg(RegNo: LHSReg);
2922 ResultReg = fastEmitInst_r(MachineInstOpcode: MULOpc[0], RC: TLI.getRegClassFor(VT), Op0: RHSReg);
2923 } else
2924 ResultReg = fastEmitInst_rr(MachineInstOpcode: MULOpc[VT.SimpleTy-MVT::i8],
2925 RC: TLI.getRegClassFor(VT), Op0: LHSReg, Op1: RHSReg);
2926 }
2927
2928 if (!ResultReg)
2929 return false;
2930
2931 // Assign to a GPR since the overflow return value is lowered to a SETcc.
2932 Register ResultReg2 = createResultReg(RC: &X86::GR8RegClass);
2933 assert((ResultReg+1) == ResultReg2 && "Nonconsecutive result registers.");
2934 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(GET_SETCC),
2935 DestReg: ResultReg2)
2936 .addImm(Val: CondCode);
2937
2938 updateValueMap(I: II, Reg: ResultReg, NumRegs: 2);
2939 return true;
2940 }
2941 case Intrinsic::x86_sse_cvttss2si:
2942 case Intrinsic::x86_sse_cvttss2si64:
2943 case Intrinsic::x86_sse2_cvttsd2si:
2944 case Intrinsic::x86_sse2_cvttsd2si64: {
2945 bool IsInputDouble;
2946 switch (II->getIntrinsicID()) {
2947 default: llvm_unreachable("Unexpected intrinsic.");
2948 case Intrinsic::x86_sse_cvttss2si:
2949 case Intrinsic::x86_sse_cvttss2si64:
2950 if (!Subtarget->hasSSE1())
2951 return false;
2952 IsInputDouble = false;
2953 break;
2954 case Intrinsic::x86_sse2_cvttsd2si:
2955 case Intrinsic::x86_sse2_cvttsd2si64:
2956 if (!Subtarget->hasSSE2())
2957 return false;
2958 IsInputDouble = true;
2959 break;
2960 }
2961
2962 Type *RetTy = II->getCalledFunction()->getReturnType();
2963 MVT VT;
2964 if (!isTypeLegal(Ty: RetTy, VT))
2965 return false;
2966
2967 static const uint16_t CvtOpc[3][2][2] = {
2968 { { X86::CVTTSS2SIrr, X86::CVTTSS2SI64rr },
2969 { X86::CVTTSD2SIrr, X86::CVTTSD2SI64rr } },
2970 { { X86::VCVTTSS2SIrr, X86::VCVTTSS2SI64rr },
2971 { X86::VCVTTSD2SIrr, X86::VCVTTSD2SI64rr } },
2972 { { X86::VCVTTSS2SIZrr, X86::VCVTTSS2SI64Zrr },
2973 { X86::VCVTTSD2SIZrr, X86::VCVTTSD2SI64Zrr } },
2974 };
2975 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
2976 Subtarget->hasAVX() ? 1 :
2977 0;
2978 unsigned Opc;
2979 switch (VT.SimpleTy) {
2980 default: llvm_unreachable("Unexpected result type.");
2981 case MVT::i32: Opc = CvtOpc[AVXLevel][IsInputDouble][0]; break;
2982 case MVT::i64: Opc = CvtOpc[AVXLevel][IsInputDouble][1]; break;
2983 }
2984
2985 // Check if we can fold insertelement instructions into the convert.
2986 const Value *Op = II->getArgOperand(i: 0);
2987 while (auto *IE = dyn_cast<InsertElementInst>(Val: Op)) {
2988 const Value *Index = IE->getOperand(i_nocapture: 2);
2989 if (!isa<ConstantInt>(Val: Index))
2990 break;
2991 unsigned Idx = cast<ConstantInt>(Val: Index)->getZExtValue();
2992
2993 if (!Idx) {
2994 Op = IE->getOperand(i_nocapture: 1);
2995 break;
2996 }
2997 Op = IE->getOperand(i_nocapture: 0);
2998 }
2999
3000 Register Reg = getRegForValue(V: Op);
3001 if (!Reg)
3002 return false;
3003
3004 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3005 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: ResultReg)
3006 .addReg(RegNo: Reg);
3007
3008 updateValueMap(I: II, Reg: ResultReg);
3009 return true;
3010 }
3011 case Intrinsic::x86_sse42_crc32_32_8:
3012 case Intrinsic::x86_sse42_crc32_32_16:
3013 case Intrinsic::x86_sse42_crc32_32_32:
3014 case Intrinsic::x86_sse42_crc32_64_64: {
3015 if (!Subtarget->hasCRC32())
3016 return false;
3017
3018 Type *RetTy = II->getCalledFunction()->getReturnType();
3019
3020 MVT VT;
3021 if (!isTypeLegal(Ty: RetTy, VT))
3022 return false;
3023
3024 unsigned Opc;
3025 const TargetRegisterClass *RC = nullptr;
3026
3027 switch (II->getIntrinsicID()) {
3028 default:
3029 llvm_unreachable("Unexpected intrinsic.");
3030#define GET_EGPR_IF_ENABLED(OPC) Subtarget->hasEGPR() ? OPC##_EVEX : OPC
3031 case Intrinsic::x86_sse42_crc32_32_8:
3032 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r8);
3033 RC = &X86::GR32RegClass;
3034 break;
3035 case Intrinsic::x86_sse42_crc32_32_16:
3036 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r16);
3037 RC = &X86::GR32RegClass;
3038 break;
3039 case Intrinsic::x86_sse42_crc32_32_32:
3040 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r32r32);
3041 RC = &X86::GR32RegClass;
3042 break;
3043 case Intrinsic::x86_sse42_crc32_64_64:
3044 Opc = GET_EGPR_IF_ENABLED(X86::CRC32r64r64);
3045 RC = &X86::GR64RegClass;
3046 break;
3047#undef GET_EGPR_IF_ENABLED
3048 }
3049
3050 const Value *LHS = II->getArgOperand(i: 0);
3051 const Value *RHS = II->getArgOperand(i: 1);
3052
3053 Register LHSReg = getRegForValue(V: LHS);
3054 Register RHSReg = getRegForValue(V: RHS);
3055 if (!LHSReg || !RHSReg)
3056 return false;
3057
3058 Register ResultReg = fastEmitInst_rr(MachineInstOpcode: Opc, RC, Op0: LHSReg, Op1: RHSReg);
3059 if (!ResultReg)
3060 return false;
3061
3062 updateValueMap(I: II, Reg: ResultReg);
3063 return true;
3064 }
3065 }
3066}
3067
3068bool X86FastISel::fastLowerArguments() {
3069 if (!FuncInfo.CanLowerReturn)
3070 return false;
3071
3072 const Function *F = FuncInfo.Fn;
3073 if (F->isVarArg())
3074 return false;
3075
3076 CallingConv::ID CC = F->getCallingConv();
3077 if (CC != CallingConv::C)
3078 return false;
3079
3080 if (Subtarget->isCallingConvWin64(CC))
3081 return false;
3082
3083 if (!Subtarget->is64Bit())
3084 return false;
3085
3086 if (Subtarget->useSoftFloat())
3087 return false;
3088
3089 // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.
3090 unsigned GPRCnt = 0;
3091 unsigned FPRCnt = 0;
3092 for (auto const &Arg : F->args()) {
3093 if (Arg.hasAttribute(Kind: Attribute::ByVal) ||
3094 Arg.hasAttribute(Kind: Attribute::InReg) ||
3095 Arg.hasAttribute(Kind: Attribute::StructRet) ||
3096 Arg.hasAttribute(Kind: Attribute::SwiftSelf) ||
3097 Arg.hasAttribute(Kind: Attribute::SwiftAsync) ||
3098 Arg.hasAttribute(Kind: Attribute::SwiftError) ||
3099 Arg.hasAttribute(Kind: Attribute::Nest))
3100 return false;
3101
3102 Type *ArgTy = Arg.getType();
3103 if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())
3104 return false;
3105
3106 EVT ArgVT = TLI.getValueType(DL, Ty: ArgTy);
3107 if (!ArgVT.isSimple()) return false;
3108 switch (ArgVT.getSimpleVT().SimpleTy) {
3109 default: return false;
3110 case MVT::i32:
3111 case MVT::i64:
3112 ++GPRCnt;
3113 break;
3114 case MVT::f32:
3115 case MVT::f64:
3116 if (!Subtarget->hasSSE1())
3117 return false;
3118 ++FPRCnt;
3119 break;
3120 }
3121
3122 if (GPRCnt > 6)
3123 return false;
3124
3125 if (FPRCnt > 8)
3126 return false;
3127 }
3128
3129 static const MCPhysReg GPR32ArgRegs[] = {
3130 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
3131 };
3132 static const MCPhysReg GPR64ArgRegs[] = {
3133 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9
3134 };
3135 static const MCPhysReg XMMArgRegs[] = {
3136 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3137 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3138 };
3139
3140 unsigned GPRIdx = 0;
3141 unsigned FPRIdx = 0;
3142 for (auto const &Arg : F->args()) {
3143 MVT VT = TLI.getSimpleValueType(DL, Ty: Arg.getType());
3144 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
3145 MCRegister SrcReg;
3146 switch (VT.SimpleTy) {
3147 default: llvm_unreachable("Unexpected value type.");
3148 case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;
3149 case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++]; break;
3150 case MVT::f32: [[fallthrough]];
3151 case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++]; break;
3152 }
3153 Register DstReg = FuncInfo.MF->addLiveIn(PReg: SrcReg, RC);
3154 // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.
3155 // Without this, EmitLiveInCopies may eliminate the livein if its only
3156 // use is a bitcast (which isn't turned into an instruction).
3157 Register ResultReg = createResultReg(RC);
3158 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3159 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: ResultReg)
3160 .addReg(RegNo: DstReg, Flags: getKillRegState(B: true));
3161 updateValueMap(I: &Arg, Reg: ResultReg);
3162 }
3163 return true;
3164}
3165
3166static unsigned computeBytesPoppedByCalleeForSRet(const X86Subtarget *Subtarget,
3167 CallingConv::ID CC,
3168 const CallBase *CB) {
3169 if (Subtarget->is64Bit())
3170 return 0;
3171 if (Subtarget->getTargetTriple().isOSMSVCRT())
3172 return 0;
3173 if (CC == CallingConv::Fast || CC == CallingConv::GHC ||
3174 CC == CallingConv::HiPE || CC == CallingConv::Tail ||
3175 CC == CallingConv::SwiftTail)
3176 return 0;
3177
3178 if (CB)
3179 if (CB->arg_empty() || !CB->paramHasAttr(ArgNo: 0, Kind: Attribute::StructRet) ||
3180 CB->paramHasAttr(ArgNo: 0, Kind: Attribute::InReg) || Subtarget->isTargetMCU())
3181 return 0;
3182
3183 return 4;
3184}
3185
3186bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3187 auto &OutVals = CLI.OutVals;
3188 auto &OutFlags = CLI.OutFlags;
3189 auto &OutRegs = CLI.OutRegs;
3190 auto &Ins = CLI.Ins;
3191 auto &InRegs = CLI.InRegs;
3192 CallingConv::ID CC = CLI.CallConv;
3193 bool &IsTailCall = CLI.IsTailCall;
3194 bool IsVarArg = CLI.IsVarArg;
3195 const Value *Callee = CLI.Callee;
3196 MCSymbol *Symbol = CLI.Symbol;
3197 const auto *CB = CLI.CB;
3198
3199 bool Is64Bit = Subtarget->is64Bit();
3200 bool IsWin64 = Subtarget->isCallingConvWin64(CC);
3201
3202 // If the return type is illegal, check if the ABI requires a type conversion
3203 // that FastISel cannot handle. Fall back to DAG ISel in such cases.
3204 // For example, bfloat is returned as f16 in XMM0, however FastISel would
3205 // assign f32 register type and store it in FuncInfo.ValueMap. This would
3206 // cause DAG incorrectly perform type conversion from f32 to bfloat after get
3207 // the value from FuncInfo.ValueMap.
3208 // However, i1 is promoted to i8 and return i8 defined by ABI, so FastISel can
3209 // lower it without switching to DAGISel.
3210 SmallVector<Type *> RetTys;
3211 ComputeValueTypes(DL, Ty: CLI.RetTy, Types&: RetTys);
3212 for (Type *RetTy : RetTys) {
3213 MVT RetVT = MVT::Other;
3214 if (!isTypeLegal(Ty: RetTy, VT&: RetVT)) {
3215 if (RetVT == MVT::Other)
3216 return false; // Unknown type, let DAG ISel handle it.
3217
3218 // RetVT is not MVT::Other, it must be simple now. It is something rely on
3219 // the logic of isTypeLegal().
3220 MVT ABIVT = TLI.getRegisterTypeForCallingConv(Context&: CLI.RetTy->getContext(),
3221 CC: CLI.CallConv, VT: RetVT);
3222 MVT RegVT = TLI.getRegisterType(Context&: CLI.RetTy->getContext(), VT: RetVT);
3223 if (ABIVT != RegVT)
3224 return false;
3225 }
3226 }
3227
3228 // Call / invoke instructions with NoCfCheck attribute require special
3229 // handling.
3230 if (CB && CB->doesNoCfCheck())
3231 return false;
3232
3233 // Functions with no_caller_saved_registers that need special handling.
3234 if ((CB && isa<CallInst>(Val: CB) && CB->hasFnAttr(Kind: "no_caller_saved_registers")))
3235 return false;
3236
3237 // Functions with no_callee_saved_registers that need special handling.
3238 if ((CB && CB->hasFnAttr(Kind: "no_callee_saved_registers")))
3239 return false;
3240
3241 // Indirect calls with CFI checks need special handling.
3242 if (CB && CB->isIndirectCall() && CB->getOperandBundle(ID: LLVMContext::OB_kcfi))
3243 return false;
3244
3245 // Functions using thunks for indirect calls need to use SDISel.
3246 if (Subtarget->useIndirectThunkCalls())
3247 return false;
3248
3249 // Handle only C and fastcc calling conventions for now.
3250 switch (CC) {
3251 default: return false;
3252 case CallingConv::C:
3253 case CallingConv::Fast:
3254 case CallingConv::Tail:
3255 case CallingConv::Swift:
3256 case CallingConv::SwiftTail:
3257 case CallingConv::X86_FastCall:
3258 case CallingConv::X86_StdCall:
3259 case CallingConv::X86_ThisCall:
3260 case CallingConv::Win64:
3261 case CallingConv::X86_64_SysV:
3262 case CallingConv::CFGuard_Check:
3263 break;
3264 }
3265
3266 // Allow SelectionDAG isel to handle tail calls.
3267 if (IsTailCall)
3268 return false;
3269
3270 // fastcc with -tailcallopt is intended to provide a guaranteed
3271 // tail call optimization. Fastisel doesn't know how to do that.
3272 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
3273 CC == CallingConv::Tail || CC == CallingConv::SwiftTail)
3274 return false;
3275
3276 // Don't know how to handle Win64 varargs yet. Nothing special needed for
3277 // x86-32. Special handling for x86-64 is implemented.
3278 if (IsVarArg && IsWin64)
3279 return false;
3280
3281 // Don't know about inalloca yet.
3282 if (CLI.CB && CLI.CB->hasInAllocaArgument())
3283 return false;
3284
3285 for (auto Flag : CLI.OutFlags)
3286 if (Flag.isSwiftError() || Flag.isPreallocated())
3287 return false;
3288
3289 // Can't handle import call optimization.
3290 if (Is64Bit &&
3291 MF->getFunction().getParent()->getModuleFlag(Key: "import-call-optimization"))
3292 return false;
3293
3294 SmallVector<MVT, 16> OutVTs;
3295 SmallVector<Type *, 16> ArgTys;
3296 SmallVector<Register, 16> ArgRegs;
3297
3298 // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
3299 // instruction. This is safe because it is common to all FastISel supported
3300 // calling conventions on x86.
3301 for (int i = 0, e = OutVals.size(); i != e; ++i) {
3302 Value *&Val = OutVals[i];
3303 ISD::ArgFlagsTy Flags = OutFlags[i];
3304 if (auto *CI = dyn_cast<ConstantInt>(Val)) {
3305 if (CI->getBitWidth() < 32) {
3306 if (Flags.isSExt())
3307 Val = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue().sext(width: 32));
3308 else
3309 Val = ConstantInt::get(Context&: CI->getContext(), V: CI->getValue().zext(width: 32));
3310 }
3311 }
3312
3313 // Passing bools around ends up doing a trunc to i1 and passing it.
3314 // Codegen this as an argument + "and 1".
3315 MVT VT;
3316 auto *TI = dyn_cast<TruncInst>(Val);
3317 Register ResultReg;
3318 if (TI && TI->getType()->isIntegerTy(BitWidth: 1) && CLI.CB &&
3319 (TI->getParent() == CLI.CB->getParent()) && TI->hasOneUse()) {
3320 Value *PrevVal = TI->getOperand(i_nocapture: 0);
3321 ResultReg = getRegForValue(V: PrevVal);
3322
3323 if (!ResultReg)
3324 return false;
3325
3326 if (!isTypeLegal(Ty: PrevVal->getType(), VT))
3327 return false;
3328
3329 ResultReg = fastEmit_ri(VT, RetVT: VT, Opcode: ISD::AND, Op0: ResultReg, imm1: 1);
3330 } else {
3331 if (!isTypeLegal(Ty: Val->getType(), VT) || VT.isVectorOf(EltVT: MVT::i1))
3332 return false;
3333 ResultReg = getRegForValue(V: Val);
3334 }
3335
3336 if (!ResultReg)
3337 return false;
3338
3339 ArgRegs.push_back(Elt: ResultReg);
3340 OutVTs.push_back(Elt: VT);
3341 ArgTys.push_back(Elt: Val->getType());
3342 }
3343
3344 // Analyze operands of the call, assigning locations to each operand.
3345 SmallVector<CCValAssign, 16> ArgLocs;
3346 CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());
3347
3348 // Allocate shadow area for Win64
3349 if (IsWin64)
3350 CCInfo.AllocateStack(Size: 32, Alignment: Align(8));
3351
3352 CCInfo.AnalyzeCallOperands(ArgVTs&: OutVTs, Flags&: OutFlags, OrigTys&: ArgTys, Fn: CC_X86);
3353
3354 // Get a count of how many bytes are to be pushed on the stack.
3355 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
3356
3357 // Issue CALLSEQ_START
3358 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
3359 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackDown))
3360 .addImm(Val: NumBytes).addImm(Val: 0).addImm(Val: 0);
3361
3362 // Walk the register/memloc assignments, inserting copies/loads.
3363 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3364 for (const CCValAssign &VA : ArgLocs) {
3365 const Value *ArgVal = OutVals[VA.getValNo()];
3366 MVT ArgVT = OutVTs[VA.getValNo()];
3367
3368 if (ArgVT == MVT::x86mmx)
3369 return false;
3370
3371 Register ArgReg = ArgRegs[VA.getValNo()];
3372
3373 // Promote the value if needed.
3374 switch (VA.getLocInfo()) {
3375 case CCValAssign::Full: break;
3376 case CCValAssign::SExt: {
3377 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3378 "Unexpected extend");
3379
3380 if (ArgVT == MVT::i1)
3381 return false;
3382
3383 bool Emitted = X86FastEmitExtend(Opc: ISD::SIGN_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3384 SrcVT: ArgVT, ResultReg&: ArgReg);
3385 assert(Emitted && "Failed to emit a sext!"); (void)Emitted;
3386 ArgVT = VA.getLocVT();
3387 break;
3388 }
3389 case CCValAssign::ZExt: {
3390 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3391 "Unexpected extend");
3392
3393 // Handle zero-extension from i1 to i8, which is common.
3394 if (ArgVT == MVT::i1) {
3395 // Set the high bits to zero.
3396 ArgReg = fastEmitZExtFromI1(VT: MVT::i8, Op0: ArgReg);
3397 ArgVT = MVT::i8;
3398
3399 if (!ArgReg)
3400 return false;
3401 }
3402
3403 bool Emitted = X86FastEmitExtend(Opc: ISD::ZERO_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3404 SrcVT: ArgVT, ResultReg&: ArgReg);
3405 assert(Emitted && "Failed to emit a zext!"); (void)Emitted;
3406 ArgVT = VA.getLocVT();
3407 break;
3408 }
3409 case CCValAssign::AExt: {
3410 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
3411 "Unexpected extend");
3412 bool Emitted = X86FastEmitExtend(Opc: ISD::ANY_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3413 SrcVT: ArgVT, ResultReg&: ArgReg);
3414 if (!Emitted)
3415 Emitted = X86FastEmitExtend(Opc: ISD::ZERO_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3416 SrcVT: ArgVT, ResultReg&: ArgReg);
3417 if (!Emitted)
3418 Emitted = X86FastEmitExtend(Opc: ISD::SIGN_EXTEND, DstVT: VA.getLocVT(), Src: ArgReg,
3419 SrcVT: ArgVT, ResultReg&: ArgReg);
3420
3421 assert(Emitted && "Failed to emit a aext!"); (void)Emitted;
3422 ArgVT = VA.getLocVT();
3423 break;
3424 }
3425 case CCValAssign::BCvt: {
3426 ArgReg = fastEmit_r(VT: ArgVT, RetVT: VA.getLocVT(), Opcode: ISD::BITCAST, Op0: ArgReg);
3427 assert(ArgReg && "Failed to emit a bitcast!");
3428 ArgVT = VA.getLocVT();
3429 break;
3430 }
3431 case CCValAssign::VExt:
3432 // VExt has not been implemented, so this should be impossible to reach
3433 // for now. However, fallback to Selection DAG isel once implemented.
3434 return false;
3435 case CCValAssign::AExtUpper:
3436 case CCValAssign::SExtUpper:
3437 case CCValAssign::ZExtUpper:
3438 case CCValAssign::FPExt:
3439 case CCValAssign::Trunc:
3440 llvm_unreachable("Unexpected loc info!");
3441 case CCValAssign::Indirect:
3442 // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully
3443 // support this.
3444 return false;
3445 }
3446
3447 if (VA.isRegLoc()) {
3448 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3449 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: VA.getLocReg()).addReg(RegNo: ArgReg);
3450 OutRegs.push_back(Elt: VA.getLocReg());
3451 } else {
3452 assert(VA.isMemLoc() && "Unknown value location!");
3453
3454 // Don't emit stores for undef values.
3455 if (isa<UndefValue>(Val: ArgVal))
3456 continue;
3457
3458 unsigned LocMemOffset = VA.getLocMemOffset();
3459 X86AddressMode AM;
3460 AM.Base.Reg = RegInfo->getStackRegister();
3461 AM.Disp = LocMemOffset;
3462 ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];
3463 Align Alignment = DL.getABITypeAlign(Ty: ArgVal->getType());
3464 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3465 PtrInfo: MachinePointerInfo::getStack(MF&: *FuncInfo.MF, Offset: LocMemOffset),
3466 F: MachineMemOperand::MOStore, Size: ArgVT.getStoreSize(), BaseAlignment: Alignment);
3467 if (Flags.isByVal()) {
3468 X86AddressMode SrcAM;
3469 SrcAM.Base.Reg = ArgReg;
3470 if (!TryEmitSmallMemcpy(DestAM: AM, SrcAM, Len: Flags.getByValSize()))
3471 return false;
3472 } else if (isa<ConstantInt>(Val: ArgVal) || isa<ConstantPointerNull>(Val: ArgVal)) {
3473 // If this is a really simple value, emit this with the Value* version
3474 // of X86FastEmitStore. If it isn't simple, we don't want to do this,
3475 // as it can cause us to reevaluate the argument.
3476 if (!X86FastEmitStore(VT: ArgVT, Val: ArgVal, AM, MMO))
3477 return false;
3478 } else {
3479 if (!X86FastEmitStore(VT: ArgVT, ValReg: ArgReg, AM, MMO))
3480 return false;
3481 }
3482 }
3483 }
3484
3485 // ELF / PIC requires GOT in the EBX register before function calls via PLT
3486 // GOT pointer.
3487 if (Subtarget->isPICStyleGOT()) {
3488 Register Base = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
3489 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3490 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: X86::EBX).addReg(RegNo: Base);
3491 }
3492
3493 if (Is64Bit && IsVarArg && !IsWin64) {
3494 // From AMD64 ABI document:
3495 // For calls that may call functions that use varargs or stdargs
3496 // (prototype-less calls or calls to functions containing ellipsis (...) in
3497 // the declaration) %al is used as hidden argument to specify the number
3498 // of SSE registers used. The contents of %al do not need to match exactly
3499 // the number of registers, but must be an ubound on the number of SSE
3500 // registers used and is in the range 0 - 8 inclusive.
3501
3502 // Count the number of XMM registers allocated.
3503 static const MCPhysReg XMMArgRegs[] = {
3504 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3505 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3506 };
3507 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(Regs: XMMArgRegs);
3508 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3509 && "SSE registers cannot be used when SSE is disabled");
3510 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV8ri),
3511 DestReg: X86::AL).addImm(Val: NumXMMRegs);
3512 }
3513
3514 // Materialize callee address in a register. FIXME: GV address can be
3515 // handled with a CALLpcrel32 instead.
3516 X86AddressMode CalleeAM;
3517 if (!X86SelectCallAddress(V: Callee, AM&: CalleeAM))
3518 return false;
3519
3520 Register CalleeOp;
3521 const GlobalValue *GV = nullptr;
3522 if (CalleeAM.GV != nullptr) {
3523 GV = CalleeAM.GV;
3524 } else if (CalleeAM.Base.Reg) {
3525 CalleeOp = CalleeAM.Base.Reg;
3526 } else
3527 return false;
3528
3529 // Issue the call.
3530 MachineInstrBuilder MIB;
3531 if (CalleeOp) {
3532 // Register-indirect call.
3533 unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
3534 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: CallOpc))
3535 .addReg(RegNo: CalleeOp);
3536 } else {
3537 // Direct call.
3538 assert(GV && "Not a direct call");
3539 // See if we need any target-specific flags on the GV operand.
3540 unsigned char OpFlags = Subtarget->classifyGlobalFunctionReference(GV);
3541 if (OpFlags == X86II::MO_PLT && !Is64Bit &&
3542 TM.getRelocationModel() == Reloc::Static && isa<Function>(Val: GV) &&
3543 cast<Function>(Val: GV)->isIntrinsic())
3544 OpFlags = X86II::MO_NO_FLAG;
3545
3546 // This will be a direct call, or an indirect call through memory for
3547 // NonLazyBind calls or dllimport calls.
3548 bool NeedLoad = OpFlags == X86II::MO_DLLIMPORT ||
3549 OpFlags == X86II::MO_GOTPCREL ||
3550 OpFlags == X86II::MO_GOTPCREL_NORELAX ||
3551 OpFlags == X86II::MO_COFFSTUB;
3552 unsigned CallOpc = NeedLoad
3553 ? (Is64Bit ? X86::CALL64m : X86::CALL32m)
3554 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32);
3555
3556 MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: CallOpc));
3557 if (NeedLoad)
3558 MIB.addReg(RegNo: Is64Bit ? X86::RIP : X86::NoRegister).addImm(Val: 1).addReg(RegNo: 0);
3559 if (Symbol)
3560 MIB.addSym(Sym: Symbol, TargetFlags: OpFlags);
3561 else
3562 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
3563 if (NeedLoad)
3564 MIB.addReg(RegNo: 0);
3565 }
3566
3567 // Add a register mask operand representing the call-preserved registers.
3568 // Proper defs for return values will be added by setPhysRegsDeadExcept().
3569 MIB.addRegMask(Mask: TRI.getCallPreservedMask(MF: *FuncInfo.MF, CC));
3570
3571 // Add an implicit use GOT pointer in EBX.
3572 if (Subtarget->isPICStyleGOT())
3573 MIB.addReg(RegNo: X86::EBX, Flags: RegState::Implicit);
3574
3575 if (Is64Bit && IsVarArg && !IsWin64)
3576 MIB.addReg(RegNo: X86::AL, Flags: RegState::Implicit);
3577
3578 // Add implicit physical register uses to the call.
3579 for (auto Reg : OutRegs)
3580 MIB.addReg(RegNo: Reg, Flags: RegState::Implicit);
3581
3582 // Issue CALLSEQ_END
3583 unsigned NumBytesForCalleeToPop =
3584 X86::isCalleePop(CallingConv: CC, is64Bit: Subtarget->is64Bit(), IsVarArg,
3585 GuaranteeTCO: TM.Options.GuaranteedTailCallOpt)
3586 ? NumBytes // Callee pops everything.
3587 : computeBytesPoppedByCalleeForSRet(Subtarget, CC, CB: CLI.CB);
3588 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
3589 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: AdjStackUp))
3590 .addImm(Val: NumBytes).addImm(Val: NumBytesForCalleeToPop);
3591
3592 // Now handle call return values.
3593 SmallVector<CCValAssign, 16> RVLocs;
3594 CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,
3595 CLI.RetTy->getContext());
3596 CCRetInfo.AnalyzeCallResult(Ins, Fn: RetCC_X86);
3597
3598 // Copy all of the result registers out of their specified physreg.
3599 Register ResultReg = FuncInfo.CreateRegs(Ty: CLI.RetTy);
3600 for (unsigned i = 0; i != RVLocs.size(); ++i) {
3601 CCValAssign &VA = RVLocs[i];
3602 EVT CopyVT = VA.getValVT();
3603 Register CopyReg = ResultReg + i;
3604 Register SrcReg = VA.getLocReg();
3605
3606 // If this is x86-64, and we disabled SSE, we can't return FP values
3607 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
3608 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
3609 report_fatal_error(reason: "SSE register return with SSE disabled");
3610 }
3611
3612 // If we prefer to use the value in xmm registers, copy it out as f80 and
3613 // use a truncate to move it from fp stack reg to xmm reg.
3614 if ((SrcReg == X86::FP0 || SrcReg == X86::FP1) &&
3615 isScalarFPTypeInSSEReg(VT: VA.getValVT())) {
3616 CopyVT = MVT::f80;
3617 CopyReg = createResultReg(RC: &X86::RFP80RegClass);
3618 }
3619
3620 // Copy out the result.
3621 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3622 MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: CopyReg).addReg(RegNo: SrcReg);
3623 InRegs.push_back(Elt: VA.getLocReg());
3624
3625 // Round the f80 to the right size, which also moves it to the appropriate
3626 // xmm register. This is accomplished by storing the f80 value in memory
3627 // and then loading it back.
3628 if (CopyVT != VA.getValVT()) {
3629 EVT ResVT = VA.getValVT();
3630 unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
3631 unsigned MemSize = ResVT.getSizeInBits()/8;
3632 int FI = MFI.CreateStackObject(Size: MemSize, Alignment: Align(MemSize), isSpillSlot: false);
3633 addFrameReference(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3634 MCID: TII.get(Opcode: Opc)), FI)
3635 .addReg(RegNo: CopyReg);
3636 Opc = ResVT == MVT::f32 ? X86::MOVSSrm_alt : X86::MOVSDrm_alt;
3637 addFrameReference(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3638 MCID: TII.get(Opcode: Opc), DestReg: ResultReg + i), FI);
3639 }
3640 }
3641
3642 CLI.ResultReg = ResultReg;
3643 CLI.NumResultRegs = RVLocs.size();
3644 CLI.Call = MIB;
3645
3646 // Add call site info for call graph section.
3647 if (TM.Options.EmitCallGraphSection && CB && CB->isIndirectCall()) {
3648 MachineFunction::CallSiteInfo CSInfo(*CB);
3649 MF->addCallSiteInfo(CallI: CLI.Call, CallInfo: std::move(CSInfo));
3650 }
3651
3652 return true;
3653}
3654
3655bool
3656X86FastISel::fastSelectInstruction(const Instruction *I) {
3657 switch (I->getOpcode()) {
3658 default: break;
3659 case Instruction::Load:
3660 return X86SelectLoad(I);
3661 case Instruction::Store:
3662 return X86SelectStore(I);
3663 case Instruction::Ret:
3664 return X86SelectRet(I);
3665 case Instruction::ICmp:
3666 case Instruction::FCmp:
3667 return X86SelectCmp(I);
3668 case Instruction::ZExt:
3669 return X86SelectZExt(I);
3670 case Instruction::SExt:
3671 return X86SelectSExt(I);
3672 case Instruction::CondBr:
3673 return X86SelectBranch(I);
3674 case Instruction::LShr:
3675 case Instruction::AShr:
3676 case Instruction::Shl:
3677 return X86SelectShift(I);
3678 case Instruction::SDiv:
3679 case Instruction::UDiv:
3680 case Instruction::SRem:
3681 case Instruction::URem:
3682 return X86SelectDivRem(I);
3683 case Instruction::Select:
3684 return X86SelectSelect(I);
3685 case Instruction::Trunc:
3686 return X86SelectTrunc(I);
3687 case Instruction::FPExt:
3688 return X86SelectFPExt(I);
3689 case Instruction::FPTrunc:
3690 return X86SelectFPTrunc(I);
3691 case Instruction::SIToFP:
3692 return X86SelectSIToFP(I);
3693 case Instruction::UIToFP:
3694 return X86SelectUIToFP(I);
3695 case Instruction::IntToPtr: // Deliberate fall-through.
3696 case Instruction::PtrToInt: {
3697 EVT SrcVT = TLI.getValueType(DL, Ty: I->getOperand(i: 0)->getType());
3698 EVT DstVT = TLI.getValueType(DL, Ty: I->getType());
3699 if (DstVT.bitsGT(VT: SrcVT))
3700 return X86SelectZExt(I);
3701 if (DstVT.bitsLT(VT: SrcVT))
3702 return X86SelectTrunc(I);
3703 Register Reg = getRegForValue(V: I->getOperand(i: 0));
3704 if (!Reg)
3705 return false;
3706 updateValueMap(I, Reg);
3707 return true;
3708 }
3709 case Instruction::BitCast:
3710 return X86SelectBitCast(I);
3711 }
3712
3713 return false;
3714}
3715
3716Register X86FastISel::emitMOV32r0() {
3717 Register ResultReg = createResultReg(RC: &X86::GR32RegClass);
3718 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV32r0),
3719 DestReg: ResultReg)
3720 .setOperandDead(1);
3721 return ResultReg;
3722}
3723
3724Register X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
3725 if (VT > MVT::i64)
3726 return Register();
3727
3728 uint64_t Imm = CI->getZExtValue();
3729 if (Imm == 0) {
3730 Register SrcReg = emitMOV32r0();
3731 switch (VT.SimpleTy) {
3732 default: llvm_unreachable("Unexpected value type");
3733 case MVT::i1:
3734 case MVT::i8:
3735 return fastEmitInst_extractsubreg(RetVT: MVT::i8, Op0: SrcReg, Idx: X86::sub_8bit);
3736 case MVT::i16:
3737 return fastEmitInst_extractsubreg(RetVT: MVT::i16, Op0: SrcReg, Idx: X86::sub_16bit);
3738 case MVT::i32:
3739 return SrcReg;
3740 case MVT::i64: {
3741 Register ResultReg = createResultReg(RC: &X86::GR64RegClass);
3742 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3743 MCID: TII.get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: ResultReg)
3744 .addReg(RegNo: SrcReg)
3745 .addImm(Val: X86::sub_32bit);
3746 return ResultReg;
3747 }
3748 }
3749 }
3750
3751 unsigned Opc = 0;
3752 switch (VT.SimpleTy) {
3753 default: llvm_unreachable("Unexpected value type");
3754 case MVT::i1:
3755 VT = MVT::i8;
3756 [[fallthrough]];
3757 case MVT::i8: Opc = X86::MOV8ri; break;
3758 case MVT::i16: Opc = X86::MOV16ri; break;
3759 case MVT::i32: Opc = X86::MOV32ri; break;
3760 case MVT::i64:
3761 Opc = X86::getMOVriOpcode(/*Use64BitReg=*/true, Imm);
3762 break;
3763 }
3764 return fastEmitInst_i(MachineInstOpcode: Opc, RC: TLI.getRegClassFor(VT), Imm);
3765}
3766
3767Register X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
3768 if (CFP->isNullValue())
3769 return fastMaterializeFloatZero(CF: CFP);
3770
3771 // Can't handle alternate code models yet.
3772 CodeModel::Model CM = TM.getCodeModel();
3773 if (CM != CodeModel::Small && CM != CodeModel::Medium &&
3774 CM != CodeModel::Large)
3775 return Register();
3776
3777 // Get opcode and regclass of the output for the given load instruction.
3778 unsigned Opc = 0;
3779 bool HasSSE1 = Subtarget->hasSSE1();
3780 bool HasSSE2 = Subtarget->hasSSE2();
3781 bool HasAVX = Subtarget->hasAVX();
3782 bool HasAVX512 = Subtarget->hasAVX512();
3783 switch (VT.SimpleTy) {
3784 default:
3785 return Register();
3786 case MVT::f32:
3787 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
3788 : HasAVX ? X86::VMOVSSrm_alt
3789 : HasSSE1 ? X86::MOVSSrm_alt
3790 : X86::LD_Fp32m;
3791 break;
3792 case MVT::f64:
3793 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
3794 : HasAVX ? X86::VMOVSDrm_alt
3795 : HasSSE2 ? X86::MOVSDrm_alt
3796 : X86::LD_Fp64m;
3797 break;
3798 case MVT::f80:
3799 // No f80 support yet.
3800 return Register();
3801 }
3802
3803 // MachineConstantPool wants an explicit alignment.
3804 Align Alignment = DL.getPrefTypeAlign(Ty: CFP->getType());
3805
3806 // x86-32 PIC requires a PIC base register for constant pools.
3807 Register PICBase;
3808 unsigned char OpFlag = Subtarget->classifyLocalReference(GV: nullptr);
3809 if (OpFlag == X86II::MO_PIC_BASE_OFFSET)
3810 PICBase = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
3811 else if (OpFlag == X86II::MO_GOTOFF)
3812 PICBase = getInstrInfo()->getGlobalBaseReg(MF: FuncInfo.MF);
3813 else if (Subtarget->is64Bit() && TM.getCodeModel() != CodeModel::Large)
3814 PICBase = X86::RIP;
3815
3816 // Create the load from the constant pool.
3817 unsigned CPI = MCP.getConstantPoolIndex(C: CFP, Alignment);
3818 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT: VT.SimpleTy));
3819
3820 // Large code model only applies to 64-bit mode.
3821 if (Subtarget->is64Bit() && CM == CodeModel::Large) {
3822 Register AddrReg = createResultReg(RC: &X86::GR64RegClass);
3823 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV64ri),
3824 DestReg: AddrReg)
3825 .addConstantPoolIndex(Idx: CPI, Offset: 0, TargetFlags: OpFlag);
3826 MachineInstrBuilder MIB = BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3827 MCID: TII.get(Opcode: Opc), DestReg: ResultReg);
3828 addRegReg(MIB, Reg1: AddrReg, isKill1: false, SubReg1: X86::NoSubRegister, Reg2: PICBase, isKill2: false,
3829 SubReg2: X86::NoSubRegister);
3830 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3831 PtrInfo: MachinePointerInfo::getConstantPool(MF&: *FuncInfo.MF),
3832 F: MachineMemOperand::MOLoad, Size: DL.getPointerSize(), BaseAlignment: Alignment);
3833 MIB->addMemOperand(MF&: *FuncInfo.MF, MO: MMO);
3834 return ResultReg;
3835 }
3836
3837 addConstantPoolReference(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3838 MCID: TII.get(Opcode: Opc), DestReg: ResultReg),
3839 CPI, GlobalBaseReg: PICBase, OpFlags: OpFlag);
3840 return ResultReg;
3841}
3842
3843Register X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
3844 // Can't handle large GlobalValues yet.
3845 if (TM.getCodeModel() != CodeModel::Small &&
3846 TM.getCodeModel() != CodeModel::Medium)
3847 return Register();
3848 if (TM.isLargeGlobalValue(GV))
3849 return Register();
3850
3851 // Materialize addresses with LEA/MOV instructions.
3852 X86AddressMode AM;
3853 if (X86SelectAddress(V: GV, AM)) {
3854 // If the expression is just a basereg, then we're done, otherwise we need
3855 // to emit an LEA.
3856 if (AM.BaseType == X86AddressMode::RegBase &&
3857 AM.IndexReg == 0 && AM.Disp == 0 && AM.GV == nullptr)
3858 return AM.Base.Reg;
3859
3860 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3861 if (TM.getRelocationModel() == Reloc::Static &&
3862 TLI.getPointerTy(DL) == MVT::i64) {
3863 // The displacement code could be more than 32 bits away so we need to use
3864 // an instruction with a 64 bit immediate
3865 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: X86::MOV64ri),
3866 DestReg: ResultReg)
3867 .addGlobalAddress(GV);
3868 } else {
3869 unsigned Opc =
3870 TLI.getPointerTy(DL) == MVT::i32
3871 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3872 : X86::LEA64r;
3873 addFullAddress(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3874 MCID: TII.get(Opcode: Opc), DestReg: ResultReg), AM);
3875 }
3876 return ResultReg;
3877 }
3878 return Register();
3879}
3880
3881Register X86FastISel::fastMaterializeConstant(const Constant *C) {
3882 EVT CEVT = TLI.getValueType(DL, Ty: C->getType(), AllowUnknown: true);
3883
3884 // Only handle simple types.
3885 if (!CEVT.isSimple())
3886 return Register();
3887 MVT VT = CEVT.getSimpleVT();
3888
3889 if (const auto *CI = dyn_cast<ConstantInt>(Val: C))
3890 return X86MaterializeInt(CI, VT);
3891 if (const auto *CFP = dyn_cast<ConstantFP>(Val: C))
3892 return X86MaterializeFP(CFP, VT);
3893 if (const auto *GV = dyn_cast<GlobalValue>(Val: C))
3894 return X86MaterializeGV(GV, VT);
3895 if (isa<UndefValue>(Val: C)) {
3896 unsigned Opc = 0;
3897 switch (VT.SimpleTy) {
3898 default:
3899 break;
3900 case MVT::f32:
3901 if (!Subtarget->hasSSE1())
3902 Opc = X86::LD_Fp032;
3903 break;
3904 case MVT::f64:
3905 if (!Subtarget->hasSSE2())
3906 Opc = X86::LD_Fp064;
3907 break;
3908 case MVT::f80:
3909 Opc = X86::LD_Fp080;
3910 break;
3911 }
3912
3913 if (Opc) {
3914 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3915 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc),
3916 DestReg: ResultReg);
3917 return ResultReg;
3918 }
3919 }
3920
3921 return Register();
3922}
3923
3924Register X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {
3925 // Fail on dynamic allocas. At this point, getRegForValue has already
3926 // checked its CSE maps, so if we're here trying to handle a dynamic
3927 // alloca, we're not going to succeed. X86SelectAddress has a
3928 // check for dynamic allocas, because it's called directly from
3929 // various places, but targetMaterializeAlloca also needs a check
3930 // in order to avoid recursion between getRegForValue,
3931 // X86SelectAddrss, and targetMaterializeAlloca.
3932 if (!FuncInfo.StaticAllocaMap.count(Val: C))
3933 return Register();
3934 assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?");
3935
3936 X86AddressMode AM;
3937 if (!X86SelectAddress(V: C, AM))
3938 return Register();
3939 unsigned Opc =
3940 TLI.getPointerTy(DL) == MVT::i32
3941 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3942 : X86::LEA64r;
3943 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: TLI.getPointerTy(DL));
3944 Register ResultReg = createResultReg(RC);
3945 addFullAddress(MIB: BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD,
3946 MCID: TII.get(Opcode: Opc), DestReg: ResultReg), AM);
3947 return ResultReg;
3948}
3949
3950Register X86FastISel::fastMaterializeFloatZero(const ConstantFP *CF) {
3951 MVT VT;
3952 if (!isTypeLegal(Ty: CF->getType(), VT))
3953 return Register();
3954
3955 // Get opcode and regclass for the given zero.
3956 bool HasSSE1 = Subtarget->hasSSE1();
3957 bool HasSSE2 = Subtarget->hasSSE2();
3958 bool HasAVX512 = Subtarget->hasAVX512();
3959 unsigned Opc = 0;
3960 switch (VT.SimpleTy) {
3961 default: return 0;
3962 case MVT::f16:
3963 Opc = HasAVX512 ? X86::AVX512_FsFLD0SH : X86::FsFLD0SH;
3964 break;
3965 case MVT::f32:
3966 Opc = HasAVX512 ? X86::AVX512_FsFLD0SS
3967 : HasSSE1 ? X86::FsFLD0SS
3968 : X86::LD_Fp032;
3969 break;
3970 case MVT::f64:
3971 Opc = HasAVX512 ? X86::AVX512_FsFLD0SD
3972 : HasSSE2 ? X86::FsFLD0SD
3973 : X86::LD_Fp064;
3974 break;
3975 case MVT::f80:
3976 // No f80 support yet.
3977 return Register();
3978 }
3979
3980 Register ResultReg = createResultReg(RC: TLI.getRegClassFor(VT));
3981 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: Opc), DestReg: ResultReg);
3982 return ResultReg;
3983}
3984
3985bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
3986 const LoadInst *LI) {
3987 const Value *Ptr = LI->getPointerOperand();
3988 X86AddressMode AM;
3989 if (!X86SelectAddress(V: Ptr, AM))
3990 return false;
3991
3992 const X86InstrInfo &XII = (const X86InstrInfo &)TII;
3993
3994 unsigned Size = DL.getTypeAllocSize(Ty: LI->getType());
3995
3996 SmallVector<MachineOperand, 8> AddrOps;
3997 AM.getFullAddress(MO&: AddrOps);
3998
3999 MachineInstr *CopyMI = nullptr;
4000 MachineInstr *Result = XII.foldMemoryOperandImpl(
4001 MF&: *FuncInfo.MF, MI&: *MI, OpNum: OpNo, MOs: AddrOps, InsertPt: FuncInfo.InsertPt, Size, Alignment: LI->getAlign(),
4002 /*AllowCommute=*/true, CopyMI);
4003 if (!Result)
4004 return false;
4005
4006 // The index register could be in the wrong register class. Unfortunately,
4007 // foldMemoryOperandImpl could have commuted the instruction so its not enough
4008 // to just look at OpNo + the offset to the index reg. We actually need to
4009 // scan the instruction to find the index reg and see if its the correct reg
4010 // class.
4011 unsigned OperandNo = 0;
4012 for (MachineInstr::mop_iterator I = Result->operands_begin(),
4013 E = Result->operands_end(); I != E; ++I, ++OperandNo) {
4014 MachineOperand &MO = *I;
4015 if (!MO.isReg() || MO.isDef() || MO.getReg() != AM.IndexReg)
4016 continue;
4017 // Found the index reg, now try to rewrite it.
4018 Register IndexReg = constrainOperandRegClass(II: Result->getDesc(),
4019 Op: MO.getReg(), OpNum: OperandNo);
4020 if (IndexReg == MO.getReg())
4021 continue;
4022 MO.setReg(IndexReg);
4023 }
4024
4025 if (MI->isCall())
4026 FuncInfo.MF->moveAdditionalCallInfo(Old: MI, New: Result);
4027 Result->addMemOperand(MF&: *FuncInfo.MF, MO: createMachineMemOperandFor(I: LI));
4028 Result->cloneInstrSymbols(MF&: *FuncInfo.MF, MI: *MI);
4029 MachineBasicBlock::iterator I(MI);
4030 removeDeadCode(I, E: std::next(x: I));
4031 return true;
4032}
4033
4034Register X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode,
4035 const TargetRegisterClass *RC,
4036 Register Op0, Register Op1,
4037 Register Op2, Register Op3) {
4038 const MCInstrDesc &II = TII.get(Opcode: MachineInstOpcode);
4039
4040 Register ResultReg = createResultReg(RC);
4041 Op0 = constrainOperandRegClass(II, Op: Op0, OpNum: II.getNumDefs());
4042 Op1 = constrainOperandRegClass(II, Op: Op1, OpNum: II.getNumDefs() + 1);
4043 Op2 = constrainOperandRegClass(II, Op: Op2, OpNum: II.getNumDefs() + 2);
4044 Op3 = constrainOperandRegClass(II, Op: Op3, OpNum: II.getNumDefs() + 3);
4045
4046 if (II.getNumDefs() >= 1)
4047 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II, DestReg: ResultReg)
4048 .addReg(RegNo: Op0)
4049 .addReg(RegNo: Op1)
4050 .addReg(RegNo: Op2)
4051 .addReg(RegNo: Op3);
4052 else {
4053 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: II)
4054 .addReg(RegNo: Op0)
4055 .addReg(RegNo: Op1)
4056 .addReg(RegNo: Op2)
4057 .addReg(RegNo: Op3);
4058 BuildMI(BB&: *FuncInfo.MBB, I: FuncInfo.InsertPt, MIMD, MCID: TII.get(Opcode: TargetOpcode::COPY),
4059 DestReg: ResultReg)
4060 .addReg(RegNo: II.implicit_defs()[0]);
4061 }
4062 return ResultReg;
4063}
4064
4065namespace llvm {
4066FastISel *X86::createFastISel(FunctionLoweringInfo &funcInfo,
4067 const TargetLibraryInfo *libInfo,
4068 const LibcallLoweringInfo *libcallLowering) {
4069 return new X86FastISel(funcInfo, libInfo, libcallLowering);
4070}
4071}
4072