1//===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling, if-conversion, other late
11// optimizations, or simply the encoding of the instructions.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86FrameLowering.h"
17#include "X86InstrInfo.h"
18#include "X86MachineFunctionInfo.h"
19#include "X86Subtarget.h"
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineDominators.h"
22#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachinePassManager.h"
26#include "llvm/CodeGen/Passes.h" // For IDs of passes that are preserved.
27#include "llvm/IR/Analysis.h"
28#include "llvm/IR/EHPersonalities.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/Target/TargetMachine.h"
31using namespace llvm;
32
33#define DEBUG_TYPE "x86-expand-pseudo"
34#define X86_EXPAND_PSEUDO_NAME "X86 pseudo instruction expansion pass"
35
36namespace {
37class X86ExpandPseudoImpl {
38public:
39 const X86Subtarget *STI = nullptr;
40 const X86InstrInfo *TII = nullptr;
41 const X86RegisterInfo *TRI = nullptr;
42 const X86MachineFunctionInfo *X86FI = nullptr;
43 const X86FrameLowering *X86FL = nullptr;
44
45 bool runOnMachineFunction(MachineFunction &MF);
46
47private:
48 void expandICallBranchFunnel(MachineBasicBlock *MBB,
49 MachineBasicBlock::iterator MBBI);
50 void expandCALL_RVMARKER(MachineBasicBlock &MBB,
51 MachineBasicBlock::iterator MBBI);
52 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
53 bool expandMBB(MachineBasicBlock &MBB);
54
55 /// This function expands pseudos which affects control flow.
56 /// It is done in separate pass to simplify blocks navigation in main
57 /// pass(calling expandMBB).
58 bool expandPseudosWhichAffectControlFlow(MachineFunction &MF);
59
60 /// Expand X86::VASTART_SAVE_XMM_REGS into set of xmm copying instructions,
61 /// placed into separate block guarded by check for al register(for SystemV
62 /// abi).
63 void expandVastartSaveXmmRegs(
64 MachineBasicBlock *EntryBlk,
65 MachineBasicBlock::iterator VAStartPseudoInstr) const;
66};
67
68class X86ExpandPseudoLegacy : public MachineFunctionPass {
69public:
70 static char ID;
71 X86ExpandPseudoLegacy() : MachineFunctionPass(ID) {}
72
73 void getAnalysisUsage(AnalysisUsage &AU) const override {
74 AU.setPreservesCFG();
75 MachineFunctionPass::getAnalysisUsage(AU);
76 }
77
78 const X86Subtarget *STI = nullptr;
79 const X86InstrInfo *TII = nullptr;
80 const X86RegisterInfo *TRI = nullptr;
81 const X86MachineFunctionInfo *X86FI = nullptr;
82 const X86FrameLowering *X86FL = nullptr;
83
84 bool runOnMachineFunction(MachineFunction &MF) override;
85
86 MachineFunctionProperties getRequiredProperties() const override {
87 return MachineFunctionProperties().setNoVRegs();
88 }
89
90 StringRef getPassName() const override {
91 return "X86 pseudo instruction expansion pass";
92 }
93};
94char X86ExpandPseudoLegacy::ID = 0;
95} // End anonymous namespace.
96
97INITIALIZE_PASS(X86ExpandPseudoLegacy, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME,
98 false, false)
99
100void X86ExpandPseudoImpl::expandICallBranchFunnel(
101 MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) {
102 MachineBasicBlock *JTMBB = MBB;
103 MachineInstr *JTInst = &*MBBI;
104 MachineFunction *MF = MBB->getParent();
105 const BasicBlock *BB = MBB->getBasicBlock();
106 auto InsPt = MachineFunction::iterator(MBB);
107 ++InsPt;
108
109 std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs;
110 const DebugLoc &DL = JTInst->getDebugLoc();
111 MachineOperand Selector = JTInst->getOperand(i: 0);
112 const GlobalValue *CombinedGlobal = JTInst->getOperand(i: 1).getGlobal();
113
114 auto CmpTarget = [&](unsigned Target) {
115 if (Selector.isReg())
116 MBB->addLiveIn(PhysReg: Selector.getReg());
117 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::LEA64r), DestReg: X86::R11)
118 .addReg(RegNo: X86::RIP)
119 .addImm(Val: 1)
120 .addReg(RegNo: 0)
121 .addGlobalAddress(GV: CombinedGlobal,
122 Offset: JTInst->getOperand(i: 2 + 2 * Target).getImm())
123 .addReg(RegNo: 0);
124 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::CMP64rr))
125 .add(MO: Selector)
126 .addReg(RegNo: X86::R11);
127 };
128
129 auto CreateMBB = [&]() {
130 auto *NewMBB = MF->CreateMachineBasicBlock(BB);
131 MBB->addSuccessor(Succ: NewMBB);
132 if (!MBB->isLiveIn(Reg: X86::EFLAGS))
133 MBB->addLiveIn(PhysReg: X86::EFLAGS);
134 return NewMBB;
135 };
136
137 auto EmitCondJump = [&](unsigned CC, MachineBasicBlock *ThenMBB) {
138 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::JCC_1)).addMBB(MBB: ThenMBB).addImm(Val: CC);
139
140 auto *ElseMBB = CreateMBB();
141 MF->insert(MBBI: InsPt, MBB: ElseMBB);
142 MBB = ElseMBB;
143 MBBI = MBB->end();
144 };
145
146 auto EmitCondJumpTarget = [&](unsigned CC, unsigned Target) {
147 auto *ThenMBB = CreateMBB();
148 TargetMBBs.push_back(x: {ThenMBB, Target});
149 EmitCondJump(CC, ThenMBB);
150 };
151
152 auto EmitTailCall = [&](unsigned Target) {
153 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::TAILJMPd64))
154 .add(MO: JTInst->getOperand(i: 3 + 2 * Target));
155 };
156
157 std::function<void(unsigned, unsigned)> EmitBranchFunnel =
158 [&](unsigned FirstTarget, unsigned NumTargets) {
159 if (NumTargets == 1) {
160 EmitTailCall(FirstTarget);
161 return;
162 }
163
164 if (NumTargets == 2) {
165 CmpTarget(FirstTarget + 1);
166 EmitCondJumpTarget(X86::COND_B, FirstTarget);
167 EmitTailCall(FirstTarget + 1);
168 return;
169 }
170
171 if (NumTargets < 6) {
172 CmpTarget(FirstTarget + 1);
173 EmitCondJumpTarget(X86::COND_B, FirstTarget);
174 EmitCondJumpTarget(X86::COND_E, FirstTarget + 1);
175 EmitBranchFunnel(FirstTarget + 2, NumTargets - 2);
176 return;
177 }
178
179 auto *ThenMBB = CreateMBB();
180 CmpTarget(FirstTarget + (NumTargets / 2));
181 EmitCondJump(X86::COND_B, ThenMBB);
182 EmitCondJumpTarget(X86::COND_E, FirstTarget + (NumTargets / 2));
183 EmitBranchFunnel(FirstTarget + (NumTargets / 2) + 1,
184 NumTargets - (NumTargets / 2) - 1);
185
186 MF->insert(MBBI: InsPt, MBB: ThenMBB);
187 MBB = ThenMBB;
188 MBBI = MBB->end();
189 EmitBranchFunnel(FirstTarget, NumTargets / 2);
190 };
191
192 EmitBranchFunnel(0, (JTInst->getNumOperands() - 2) / 2);
193 for (auto P : TargetMBBs) {
194 MF->insert(MBBI: InsPt, MBB: P.first);
195 BuildMI(BB: P.first, MIMD: DL, MCID: TII->get(Opcode: X86::TAILJMPd64))
196 .add(MO: JTInst->getOperand(i: 3 + 2 * P.second));
197 }
198 JTMBB->erase(I: JTInst);
199}
200
201void X86ExpandPseudoImpl::expandCALL_RVMARKER(
202 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
203 // Expand CALL_RVMARKER pseudo to call instruction, followed by the special
204 //"movq %rax, %rdi" marker.
205 MachineInstr &MI = *MBBI;
206
207 MachineInstr *OriginalCall;
208 assert((MI.getOperand(1).isGlobal() || MI.getOperand(1).isReg()) &&
209 "invalid operand for regular call");
210 unsigned Opc = -1;
211 if (MI.getOpcode() == X86::CALL64m_RVMARKER)
212 Opc = X86::CALL64m;
213 else if (MI.getOpcode() == X86::CALL64r_RVMARKER)
214 Opc = X86::CALL64r;
215 else if (MI.getOpcode() == X86::CALL64pcrel32_RVMARKER)
216 Opc = X86::CALL64pcrel32;
217 else
218 llvm_unreachable("unexpected opcode");
219
220 OriginalCall = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc)).getInstr();
221 bool RAXImplicitDead = false;
222 for (MachineOperand &Op : llvm::drop_begin(RangeOrContainer: MI.operands())) {
223 // RAX may be 'implicit dead', if there are no other users of the return
224 // value. We introduce a new use, so change it to 'implicit def'.
225 if (Op.isReg() && Op.isImplicit() && Op.isDead() &&
226 TRI->regsOverlap(RegA: Op.getReg(), RegB: X86::RAX)) {
227 Op.setIsDead(false);
228 Op.setIsDef(true);
229 RAXImplicitDead = true;
230 }
231 OriginalCall->addOperand(Op);
232 }
233
234 // Emit marker "movq %rax, %rdi". %rdi is not callee-saved, so it cannot be
235 // live across the earlier call. The call to the ObjC runtime function returns
236 // the first argument, so the value of %rax is unchanged after the ObjC
237 // runtime call. On Windows targets, the runtime call follows the regular
238 // x64 calling convention and expects the first argument in %rcx.
239 auto TargetReg = STI->getTargetTriple().isOSWindows() ? X86::RCX : X86::RDI;
240 auto *Marker = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: X86::MOV64rr))
241 .addReg(RegNo: TargetReg, Flags: RegState::Define)
242 .addReg(RegNo: X86::RAX)
243 .getInstr();
244 if (MI.shouldUpdateAdditionalCallInfo())
245 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: Marker);
246
247 // Emit call to ObjC runtime.
248 const uint32_t *RegMask =
249 TRI->getCallPreservedMask(MF: *MBB.getParent(), CallingConv::C);
250 MachineInstr *RtCall =
251 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: X86::CALL64pcrel32))
252 .addGlobalAddress(GV: MI.getOperand(i: 0).getGlobal(), Offset: 0, TargetFlags: 0)
253 .addRegMask(Mask: RegMask)
254 .addReg(RegNo: X86::RAX,
255 Flags: RegState::Implicit |
256 (RAXImplicitDead ? (RegState::Dead | RegState::Define)
257 : RegState::Define))
258 .getInstr();
259 MI.eraseFromParent();
260
261 auto &TM = MBB.getParent()->getTarget();
262 // On Darwin platforms, wrap the expanded sequence in a bundle to prevent
263 // later optimizations from breaking up the sequence.
264 if (TM.getTargetTriple().isOSDarwin())
265 finalizeBundle(MBB, FirstMI: OriginalCall->getIterator(),
266 LastMI: std::next(x: RtCall->getIterator()));
267}
268
269/// If \p MBBI is a pseudo instruction, this method expands
270/// it to the corresponding (sequence of) actual instruction(s).
271/// \returns true if \p MBBI has been expanded.
272bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
273 MachineBasicBlock::iterator MBBI) {
274 MachineInstr &MI = *MBBI;
275 unsigned Opcode = MI.getOpcode();
276 const DebugLoc &DL = MBBI->getDebugLoc();
277#define GET_EGPR_IF_ENABLED(OPC) (STI->hasEGPR() ? OPC##_EVEX : OPC)
278 switch (Opcode) {
279 default:
280 return false;
281 case X86::TCRETURNdi:
282 case X86::TCRETURNdicc:
283 case X86::TCRETURNri:
284 case X86::TCRETURN_WIN64ri:
285 case X86::TCRETURN_HIPE32ri:
286 case X86::TCRETURNmi:
287 case X86::TCRETURNdi64:
288 case X86::TCRETURNdi64cc:
289 case X86::TCRETURNri64:
290 case X86::TCRETURNri64_ImpCall:
291 case X86::TCRETURNmi64:
292 case X86::TCRETURN_WINmi64: {
293 bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||
294 Opcode == X86::TCRETURN_WINmi64;
295 MachineOperand &JumpTarget = MBBI->getOperand(i: 0);
296 MachineOperand &StackAdjust = MBBI->getOperand(i: isMem ? X86::AddrNumOperands
297 : 1);
298 assert(StackAdjust.isImm() && "Expecting immediate value.");
299
300 // Adjust stack pointer.
301 int StackAdj = StackAdjust.getImm();
302 int MaxTCDelta = X86FI->getTCReturnAddrDelta();
303 int64_t Offset = 0;
304 assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
305
306 // Incoporate the retaddr area.
307 Offset = StackAdj - MaxTCDelta;
308 assert(Offset >= 0 && "Offset should never be negative");
309
310 if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) {
311 assert(Offset == 0 && "Conditional tail call cannot adjust the stack.");
312 }
313
314 if (Offset) {
315 // Check for possible merge with preceding ADD instruction.
316 Offset = X86FL->mergeSPAdd(MBB, MBBI, AddOffset: Offset, doMergeWithPrevious: true);
317 X86FL->emitSPUpdate(MBB, MBBI, DL, NumBytes: Offset, /*InEpilogue=*/true);
318 }
319
320 // Use this predicate to set REX prefix for X86_64 targets.
321 bool IsX64 = STI->isTargetWin64() || STI->isTargetUEFI64();
322 // Jump to label or value in register.
323 if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||
324 Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {
325 unsigned Op;
326 switch (Opcode) {
327 case X86::TCRETURNdi:
328 Op = X86::TAILJMPd;
329 break;
330 case X86::TCRETURNdicc:
331 Op = X86::TAILJMPd_CC;
332 break;
333 case X86::TCRETURNdi64cc:
334 assert(!MBB.getParent()->hasWinCFI() &&
335 "Conditional tail calls confuse "
336 "the Win64 unwinder.");
337 Op = X86::TAILJMPd64_CC;
338 break;
339 default:
340 // Note: Win64 uses REX prefixes indirect jumps out of functions, but
341 // not direct ones.
342 Op = X86::TAILJMPd64;
343 break;
344 }
345 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Op));
346 if (JumpTarget.isGlobal()) {
347 MIB.addGlobalAddress(GV: JumpTarget.getGlobal(), Offset: JumpTarget.getOffset(),
348 TargetFlags: JumpTarget.getTargetFlags());
349 } else {
350 assert(JumpTarget.isSymbol());
351 MIB.addExternalSymbol(FnName: JumpTarget.getSymbolName(),
352 TargetFlags: JumpTarget.getTargetFlags());
353 }
354 if (Op == X86::TAILJMPd_CC || Op == X86::TAILJMPd64_CC) {
355 MIB.addImm(Val: MBBI->getOperand(i: 2).getImm());
356 }
357
358 } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||
359 Opcode == X86::TCRETURN_WINmi64) {
360 unsigned Op = (Opcode == X86::TCRETURNmi)
361 ? X86::TAILJMPm
362 : (IsX64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
363 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Op));
364 for (unsigned i = 0; i != X86::AddrNumOperands; ++i)
365 MIB.add(MO: MBBI->getOperand(i));
366 } else if (Opcode == X86::TCRETURNri64 ||
367 Opcode == X86::TCRETURNri64_ImpCall ||
368 Opcode == X86::TCRETURN_WIN64ri) {
369 JumpTarget.setIsKill();
370 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
371 MCID: TII->get(Opcode: IsX64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
372 .add(MO: JumpTarget);
373 } else {
374 assert(!IsX64 && "Win64 and UEFI64 require REX for indirect jumps.");
375 JumpTarget.setIsKill();
376 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::TAILJMPr))
377 .add(MO: JumpTarget);
378 }
379
380 MachineInstr &NewMI = *std::prev(x: MBBI);
381 NewMI.copyImplicitOps(MF&: *MBBI->getParent()->getParent(), MI: *MBBI);
382 NewMI.setCFIType(MF&: *MBB.getParent(), Type: MI.getCFIType());
383
384 // Update the call info.
385 if (MBBI->isCandidateForAdditionalCallInfo())
386 MBB.getParent()->moveAdditionalCallInfo(Old: &*MBBI, New: &NewMI);
387
388 // Delete the pseudo instruction TCRETURN.
389 MBB.erase(I: MBBI);
390
391 return true;
392 }
393 case X86::EH_RETURN:
394 case X86::EH_RETURN64: {
395 MachineOperand &DestAddr = MBBI->getOperand(i: 0);
396 assert(DestAddr.isReg() && "Offset should be in register!");
397 const bool Uses64BitFramePtr = STI->isTarget64BitLP64();
398 Register StackPtr = TRI->getStackRegister();
399 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
400 MCID: TII->get(Opcode: Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), DestReg: StackPtr)
401 .addReg(RegNo: DestAddr.getReg());
402 if (STI->hasSHSTK()) {
403 unsigned PopOpcode = STI->is64Bit() ? X86::POP64r : X86::POP32r;
404 unsigned JumpOpcode = X86::JMP32r;
405 if (Uses64BitFramePtr)
406 JumpOpcode = STI->isTargetWin64() || STI->isTargetUEFI64()
407 ? X86::JMP64r_REX
408 : X86::JMP64r;
409 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: PopOpcode))
410 .addReg(RegNo: DestAddr.getReg(), Flags: RegState::Define);
411 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: JumpOpcode)).addReg(RegNo: DestAddr.getReg());
412 MBB.erase(I: MBBI);
413 }
414 // The EH_RETURN pseudo is really removed during the MC Lowering.
415 return true;
416 }
417 case X86::IRET: {
418 // Adjust stack to erase error code
419 int64_t StackAdj = MBBI->getOperand(i: 0).getImm();
420 X86FL->emitSPUpdate(MBB, MBBI, DL, NumBytes: StackAdj, InEpilogue: true);
421 // Replace pseudo with machine iret
422 unsigned RetOp = STI->is64Bit() ? X86::IRET64 : X86::IRET32;
423 // Use UIRET if UINTR is present (except for building kernel)
424 if (STI->is64Bit() && STI->hasUINTR() &&
425 MBB.getParent()->getTarget().getCodeModel() != CodeModel::Kernel)
426 RetOp = X86::UIRET;
427 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RetOp));
428 MBB.erase(I: MBBI);
429 return true;
430 }
431 case X86::RET: {
432 // Adjust stack to erase error code
433 int64_t StackAdj = MBBI->getOperand(i: 0).getImm();
434 MachineInstrBuilder MIB;
435 if (StackAdj == 0) {
436 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
437 MCID: TII->get(Opcode: STI->is64Bit() ? X86::RET64 : X86::RET32));
438 } else if (isUInt<16>(x: StackAdj)) {
439 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
440 MCID: TII->get(Opcode: STI->is64Bit() ? X86::RETI64 : X86::RETI32))
441 .addImm(Val: StackAdj);
442 } else {
443 assert(!STI->is64Bit() &&
444 "shouldn't need to do this for x86_64 targets!");
445 // A ret can only handle immediates as big as 2**16-1. If we need to pop
446 // off bytes before the return address, we must do it manually.
447 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::POP32r)).addReg(RegNo: X86::ECX, Flags: RegState::Define);
448 X86FL->emitSPUpdate(MBB, MBBI, DL, NumBytes: StackAdj, /*InEpilogue=*/true);
449 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::PUSH32r)).addReg(RegNo: X86::ECX);
450 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::RET32));
451 }
452 for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; ++I)
453 MIB.add(MO: MBBI->getOperand(i: I));
454 MBB.erase(I: MBBI);
455 return true;
456 }
457 case X86::LCMPXCHG16B_SAVE_RBX: {
458 // Perform the following transformation.
459 // SaveRbx = pseudocmpxchg Addr, <4 opds for the address>, InArg, SaveRbx
460 // =>
461 // RBX = InArg
462 // actualcmpxchg Addr
463 // RBX = SaveRbx
464 const MachineOperand &InArg = MBBI->getOperand(i: 6);
465 Register SaveRbx = MBBI->getOperand(i: 7).getReg();
466
467 // Copy the input argument of the pseudo into the argument of the
468 // actual instruction.
469 // NOTE: We don't copy the kill flag since the input might be the same reg
470 // as one of the other operands of LCMPXCHG16B.
471 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::RBX, SrcReg: InArg.getReg(), KillSrc: false);
472 // Create the actual instruction.
473 MachineInstr *NewInstr = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::LCMPXCHG16B));
474 // Copy the operands related to the address. If we access a frame variable,
475 // we need to replace the RBX base with SaveRbx, as RBX has another value.
476 const MachineOperand &Base = MBBI->getOperand(i: 1);
477 if (Base.getReg() == X86::RBX || Base.getReg() == X86::EBX)
478 NewInstr->addOperand(Op: MachineOperand::CreateReg(
479 Reg: Base.getReg() == X86::RBX
480 ? SaveRbx
481 : Register(TRI->getSubReg(Reg: SaveRbx, Idx: X86::sub_32bit)),
482 /*IsDef=*/isDef: false));
483 else
484 NewInstr->addOperand(Op: Base);
485 for (unsigned Idx = 1 + 1; Idx < 1 + X86::AddrNumOperands; ++Idx)
486 NewInstr->addOperand(Op: MBBI->getOperand(i: Idx));
487 // Finally, restore the value of RBX.
488 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::RBX, SrcReg: SaveRbx,
489 /*SrcIsKill*/ KillSrc: true);
490
491 // Delete the pseudo.
492 MBBI->eraseFromParent();
493 return true;
494 }
495 // Loading/storing mask pairs requires two kmov operations. The second one of
496 // these needs a 2 byte displacement relative to the specified address (with
497 // 32 bit spill size). The pairs of 1bit masks up to 16 bit masks all use the
498 // same spill size, they all are stored using MASKPAIR16STORE, loaded using
499 // MASKPAIR16LOAD.
500 //
501 // The displacement value might wrap around in theory, thus the asserts in
502 // both cases.
503 case X86::MASKPAIR16LOAD: {
504 int64_t Disp = MBBI->getOperand(i: 1 + X86::AddrDisp).getImm();
505 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");
506 Register Reg = MBBI->getOperand(i: 0).getReg();
507 bool DstIsDead = MBBI->getOperand(i: 0).isDead();
508 Register Reg0 = TRI->getSubReg(Reg, Idx: X86::sub_mask_0);
509 Register Reg1 = TRI->getSubReg(Reg, Idx: X86::sub_mask_1);
510
511 auto MIBLo =
512 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm)))
513 .addReg(RegNo: Reg0, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
514 auto MIBHi =
515 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm)))
516 .addReg(RegNo: Reg1, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
517
518 for (int i = 0; i < X86::AddrNumOperands; ++i) {
519 MIBLo.add(MO: MBBI->getOperand(i: 1 + i));
520 if (i == X86::AddrDisp)
521 MIBHi.addImm(Val: Disp + 2);
522 else
523 MIBHi.add(MO: MBBI->getOperand(i: 1 + i));
524 }
525
526 // Split the memory operand, adjusting the offset and size for the halves.
527 MachineMemOperand *OldMMO = MBBI->memoperands().front();
528 MachineFunction *MF = MBB.getParent();
529 MachineMemOperand *MMOLo = MF->getMachineMemOperand(MMO: OldMMO, Offset: 0, Size: 2);
530 MachineMemOperand *MMOHi = MF->getMachineMemOperand(MMO: OldMMO, Offset: 2, Size: 2);
531
532 MIBLo.setMemRefs(MMOLo);
533 MIBHi.setMemRefs(MMOHi);
534
535 // Delete the pseudo.
536 MBB.erase(I: MBBI);
537 return true;
538 }
539 case X86::MASKPAIR16STORE: {
540 int64_t Disp = MBBI->getOperand(i: X86::AddrDisp).getImm();
541 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");
542 Register Reg = MBBI->getOperand(i: X86::AddrNumOperands).getReg();
543 bool SrcIsKill = MBBI->getOperand(i: X86::AddrNumOperands).isKill();
544 Register Reg0 = TRI->getSubReg(Reg, Idx: X86::sub_mask_0);
545 Register Reg1 = TRI->getSubReg(Reg, Idx: X86::sub_mask_1);
546
547 auto MIBLo =
548 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk)));
549 auto MIBHi =
550 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk)));
551
552 for (int i = 0; i < X86::AddrNumOperands; ++i) {
553 MIBLo.add(MO: MBBI->getOperand(i));
554 if (i == X86::AddrDisp)
555 MIBHi.addImm(Val: Disp + 2);
556 else
557 MIBHi.add(MO: MBBI->getOperand(i));
558 }
559 MIBLo.addReg(RegNo: Reg0, Flags: getKillRegState(B: SrcIsKill));
560 MIBHi.addReg(RegNo: Reg1, Flags: getKillRegState(B: SrcIsKill));
561
562 // Split the memory operand, adjusting the offset and size for the halves.
563 MachineMemOperand *OldMMO = MBBI->memoperands().front();
564 MachineFunction *MF = MBB.getParent();
565 MachineMemOperand *MMOLo = MF->getMachineMemOperand(MMO: OldMMO, Offset: 0, Size: 2);
566 MachineMemOperand *MMOHi = MF->getMachineMemOperand(MMO: OldMMO, Offset: 2, Size: 2);
567
568 MIBLo.setMemRefs(MMOLo);
569 MIBHi.setMemRefs(MMOHi);
570
571 // Delete the pseudo.
572 MBB.erase(I: MBBI);
573 return true;
574 }
575 case X86::MWAITX_SAVE_RBX: {
576 // Perform the following transformation.
577 // SaveRbx = pseudomwaitx InArg, SaveRbx
578 // =>
579 // [E|R]BX = InArg
580 // actualmwaitx
581 // [E|R]BX = SaveRbx
582 const MachineOperand &InArg = MBBI->getOperand(i: 1);
583 // Copy the input argument of the pseudo into the argument of the
584 // actual instruction.
585 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::EBX, SrcReg: InArg.getReg(), KillSrc: InArg.isKill());
586 // Create the actual instruction.
587 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::MWAITXrrr));
588 // Finally, restore the value of RBX.
589 Register SaveRbx = MBBI->getOperand(i: 2).getReg();
590 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::RBX, SrcReg: SaveRbx, /*SrcIsKill*/ KillSrc: true);
591 // Delete the pseudo.
592 MBBI->eraseFromParent();
593 return true;
594 }
595 case TargetOpcode::ICALL_BRANCH_FUNNEL:
596 expandICallBranchFunnel(MBB: &MBB, MBBI);
597 return true;
598 case X86::PLDTILECFGV: {
599 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::LDTILECFG)));
600 return true;
601 }
602 case X86::PTILELOADDV:
603 case X86::PTILELOADDT1V:
604 case X86::PTILELOADDRSV:
605 case X86::PTILELOADDRST1V:
606 case X86::PTCVTROWD2PSrteV:
607 case X86::PTCVTROWD2PSrtiV:
608 case X86::PTCVTROWPS2BF16HrteV:
609 case X86::PTCVTROWPS2BF16HrtiV:
610 case X86::PTCVTROWPS2BF16LrteV:
611 case X86::PTCVTROWPS2BF16LrtiV:
612 case X86::PTCVTROWPS2PHHrteV:
613 case X86::PTCVTROWPS2PHHrtiV:
614 case X86::PTCVTROWPS2PHLrteV:
615 case X86::PTCVTROWPS2PHLrtiV:
616 case X86::PTILEMOVROWrteV:
617 case X86::PTILEMOVROWrtiV: {
618 for (unsigned i = 2; i > 0; --i)
619 MI.removeOperand(OpNo: i);
620 unsigned Opc;
621 switch (Opcode) {
622 case X86::PTILELOADDRSV:
623 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRS);
624 break;
625 case X86::PTILELOADDRST1V:
626 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRST1);
627 break;
628 case X86::PTILELOADDV:
629 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
630 break;
631 case X86::PTILELOADDT1V:
632 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1);
633 break;
634 case X86::PTCVTROWD2PSrteV:
635 Opc = X86::TCVTROWD2PSrte;
636 break;
637 case X86::PTCVTROWD2PSrtiV:
638 Opc = X86::TCVTROWD2PSrti;
639 break;
640 case X86::PTCVTROWPS2BF16HrteV:
641 Opc = X86::TCVTROWPS2BF16Hrte;
642 break;
643 case X86::PTCVTROWPS2BF16HrtiV:
644 Opc = X86::TCVTROWPS2BF16Hrti;
645 break;
646 case X86::PTCVTROWPS2BF16LrteV:
647 Opc = X86::TCVTROWPS2BF16Lrte;
648 break;
649 case X86::PTCVTROWPS2BF16LrtiV:
650 Opc = X86::TCVTROWPS2BF16Lrti;
651 break;
652 case X86::PTCVTROWPS2PHHrteV:
653 Opc = X86::TCVTROWPS2PHHrte;
654 break;
655 case X86::PTCVTROWPS2PHHrtiV:
656 Opc = X86::TCVTROWPS2PHHrti;
657 break;
658 case X86::PTCVTROWPS2PHLrteV:
659 Opc = X86::TCVTROWPS2PHLrte;
660 break;
661 case X86::PTCVTROWPS2PHLrtiV:
662 Opc = X86::TCVTROWPS2PHLrti;
663 break;
664 case X86::PTILEMOVROWrteV:
665 Opc = X86::TILEMOVROWrte;
666 break;
667 case X86::PTILEMOVROWrtiV:
668 Opc = X86::TILEMOVROWrti;
669 break;
670 default:
671 llvm_unreachable("Unexpected Opcode");
672 }
673 MI.setDesc(TII->get(Opcode: Opc));
674 return true;
675 }
676 case X86::PTCMMIMFP16PSV:
677 case X86::PTCMMRLFP16PSV:
678 case X86::PTDPBSSDV:
679 case X86::PTDPBSUDV:
680 case X86::PTDPBUSDV:
681 case X86::PTDPBUUDV:
682 case X86::PTDPBF16PSV:
683 case X86::PTDPFP16PSV:
684 case X86::PTDPBF8PSV:
685 case X86::PTDPBHF8PSV:
686 case X86::PTDPHBF8PSV:
687 case X86::PTDPHF8PSV: {
688 MI.untieRegOperand(OpIdx: 4);
689 for (unsigned i = 3; i > 0; --i)
690 MI.removeOperand(OpNo: i);
691 unsigned Opc;
692 switch (Opcode) {
693 // clang-format off
694 case X86::PTCMMIMFP16PSV: Opc = X86::TCMMIMFP16PS; break;
695 case X86::PTCMMRLFP16PSV: Opc = X86::TCMMRLFP16PS; break;
696 case X86::PTDPBSSDV: Opc = X86::TDPBSSD; break;
697 case X86::PTDPBSUDV: Opc = X86::TDPBSUD; break;
698 case X86::PTDPBUSDV: Opc = X86::TDPBUSD; break;
699 case X86::PTDPBUUDV: Opc = X86::TDPBUUD; break;
700 case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS; break;
701 case X86::PTDPFP16PSV: Opc = X86::TDPFP16PS; break;
702 case X86::PTDPBF8PSV: Opc = X86::TDPBF8PS; break;
703 case X86::PTDPBHF8PSV: Opc = X86::TDPBHF8PS; break;
704 case X86::PTDPHBF8PSV: Opc = X86::TDPHBF8PS; break;
705 case X86::PTDPHF8PSV: Opc = X86::TDPHF8PS; break;
706 // clang-format on
707 default:
708 llvm_unreachable("Unexpected Opcode");
709 }
710 MI.setDesc(TII->get(Opcode: Opc));
711 MI.tieOperands(DefIdx: 0, UseIdx: 1);
712 return true;
713 }
714 case X86::PTILESTOREDV: {
715 for (int i = 1; i >= 0; --i)
716 MI.removeOperand(OpNo: i);
717 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::TILESTORED)));
718 return true;
719 }
720#undef GET_EGPR_IF_ENABLED
721 case X86::PTILEZEROV: {
722 for (int i = 2; i > 0; --i) // Remove row, col
723 MI.removeOperand(OpNo: i);
724 MI.setDesc(TII->get(Opcode: X86::TILEZERO));
725 return true;
726 }
727 case X86::CALL64pcrel32_RVMARKER:
728 case X86::CALL64r_RVMARKER:
729 case X86::CALL64m_RVMARKER:
730 expandCALL_RVMARKER(MBB, MBBI);
731 return true;
732 case X86::CALL64r_ImpCall:
733 MI.setDesc(TII->get(Opcode: X86::CALL64r));
734 return true;
735 case X86::ADD32mi_ND:
736 case X86::ADD64mi32_ND:
737 case X86::SUB32mi_ND:
738 case X86::SUB64mi32_ND:
739 case X86::AND32mi_ND:
740 case X86::AND64mi32_ND:
741 case X86::OR32mi_ND:
742 case X86::OR64mi32_ND:
743 case X86::XOR32mi_ND:
744 case X86::XOR64mi32_ND:
745 case X86::ADC32mi_ND:
746 case X86::ADC64mi32_ND:
747 case X86::SBB32mi_ND:
748 case X86::SBB64mi32_ND: {
749 // It's possible for an EVEX-encoded legacy instruction to reach the 15-byte
750 // instruction length limit: 4 bytes of EVEX prefix + 1 byte of opcode + 1
751 // byte of ModRM + 1 byte of SIB + 4 bytes of displacement + 4 bytes of
752 // immediate = 15 bytes in total, e.g.
753 //
754 // subq $184, %fs:257(%rbx, %rcx), %rax
755 //
756 // In such a case, no additional (ADSIZE or segment override) prefix can be
757 // used. To resolve the issue, we split the “long” instruction into 2
758 // instructions:
759 //
760 // movq %fs:257(%rbx, %rcx)，%rax
761 // subq $184, %rax
762 //
763 // Therefore we consider the OPmi_ND to be a pseudo instruction to some
764 // extent.
765 const MachineOperand &ImmOp =
766 MI.getOperand(i: MI.getNumExplicitOperands() - 1);
767 // If the immediate is a expr, conservatively estimate 4 bytes.
768 if (ImmOp.isImm() && isInt<8>(x: ImmOp.getImm()))
769 return false;
770 int MemOpNo = X86::getFirstAddrOperandIdx(MI);
771 const MachineOperand &DispOp = MI.getOperand(i: MemOpNo + X86::AddrDisp);
772 Register Base = MI.getOperand(i: MemOpNo + X86::AddrBaseReg).getReg();
773 // If the displacement is a expr, conservatively estimate 4 bytes.
774 if (Base && DispOp.isImm() && isInt<8>(x: DispOp.getImm()))
775 return false;
776 // There can only be one of three: SIB, segment override register, ADSIZE
777 Register Index = MI.getOperand(i: MemOpNo + X86::AddrIndexReg).getReg();
778 unsigned Count = !!MI.getOperand(i: MemOpNo + X86::AddrSegmentReg).getReg();
779 if (X86II::needSIB(BaseReg: Base, IndexReg: Index, /*In64BitMode=*/true))
780 ++Count;
781 if (getX86MCRegisterClass(RC: X86::GR32RegClassID).contains(Reg: Base) ||
782 getX86MCRegisterClass(RC: X86::GR32RegClassID).contains(Reg: Index))
783 ++Count;
784 if (Count < 2)
785 return false;
786 unsigned Opc, LoadOpc;
787 switch (Opcode) {
788#define MI_TO_RI(OP) \
789 case X86::OP##32mi_ND: \
790 Opc = X86::OP##32ri; \
791 LoadOpc = X86::MOV32rm; \
792 break; \
793 case X86::OP##64mi32_ND: \
794 Opc = X86::OP##64ri32; \
795 LoadOpc = X86::MOV64rm; \
796 break;
797
798 default:
799 llvm_unreachable("Unexpected Opcode");
800 MI_TO_RI(ADD);
801 MI_TO_RI(SUB);
802 MI_TO_RI(AND);
803 MI_TO_RI(OR);
804 MI_TO_RI(XOR);
805 MI_TO_RI(ADC);
806 MI_TO_RI(SBB);
807#undef MI_TO_RI
808 }
809 // Insert OPri.
810 Register DestReg = MI.getOperand(i: 0).getReg();
811 BuildMI(BB&: MBB, I: std::next(x: MBBI), MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg)
812 .addReg(RegNo: DestReg)
813 .add(MO: ImmOp);
814 // Change OPmi_ND to MOVrm.
815 for (unsigned I = MI.getNumImplicitOperands() + 1; I != 0; --I)
816 MI.removeOperand(OpNo: MI.getNumOperands() - 1);
817 MI.setDesc(TII->get(Opcode: LoadOpc));
818 return true;
819 }
820 }
821 llvm_unreachable("Previous switch has a fallthrough?");
822}
823
824// This function creates additional block for storing varargs guarded
825// registers. It adds check for %al into entry block, to skip
826// GuardedRegsBlk if xmm registers should not be stored.
827//
828// EntryBlk[VAStartPseudoInstr] EntryBlk
829// | | .
830// | | .
831// | | GuardedRegsBlk
832// | => | .
833// | | .
834// | TailBlk
835// | |
836// | |
837//
838void X86ExpandPseudoImpl::expandVastartSaveXmmRegs(
839 MachineBasicBlock *EntryBlk,
840 MachineBasicBlock::iterator VAStartPseudoInstr) const {
841 assert(VAStartPseudoInstr->getOpcode() == X86::VASTART_SAVE_XMM_REGS);
842
843 MachineFunction *Func = EntryBlk->getParent();
844 const TargetInstrInfo *TII = STI->getInstrInfo();
845 const DebugLoc &DL = VAStartPseudoInstr->getDebugLoc();
846 Register CountReg = VAStartPseudoInstr->getOperand(i: 0).getReg();
847
848 // Calculate liveins for newly created blocks.
849 LivePhysRegs LiveRegs(*STI->getRegisterInfo());
850 SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers;
851
852 LiveRegs.addLiveIns(MBB: *EntryBlk);
853 for (MachineInstr &MI : EntryBlk->instrs()) {
854 if (MI.getOpcode() == VAStartPseudoInstr->getOpcode())
855 break;
856
857 LiveRegs.stepForward(MI, Clobbers);
858 }
859
860 // Create the new basic blocks. One block contains all the XMM stores,
861 // and another block is the final destination regardless of whether any
862 // stores were performed.
863 const BasicBlock *LLVMBlk = EntryBlk->getBasicBlock();
864 MachineFunction::iterator EntryBlkIter = ++EntryBlk->getIterator();
865 MachineBasicBlock *GuardedRegsBlk = Func->CreateMachineBasicBlock(BB: LLVMBlk);
866 MachineBasicBlock *TailBlk = Func->CreateMachineBasicBlock(BB: LLVMBlk);
867 Func->insert(MBBI: EntryBlkIter, MBB: GuardedRegsBlk);
868 Func->insert(MBBI: EntryBlkIter, MBB: TailBlk);
869
870 // Transfer the remainder of EntryBlk and its successor edges to TailBlk.
871 TailBlk->splice(Where: TailBlk->begin(), Other: EntryBlk,
872 From: std::next(x: MachineBasicBlock::iterator(VAStartPseudoInstr)),
873 To: EntryBlk->end());
874 TailBlk->transferSuccessorsAndUpdatePHIs(FromMBB: EntryBlk);
875
876 uint64_t FrameOffset = VAStartPseudoInstr->getOperand(i: 4).getImm();
877 uint64_t VarArgsRegsOffset = VAStartPseudoInstr->getOperand(i: 6).getImm();
878
879 // TODO: add support for YMM and ZMM here.
880 unsigned MOVOpc = STI->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
881
882 // In the XMM save block, save all the XMM argument registers.
883 for (int64_t OpndIdx = 7, RegIdx = 0;
884 OpndIdx < VAStartPseudoInstr->getNumOperands() - 1;
885 OpndIdx++, RegIdx++) {
886 auto NewMI = BuildMI(BB: GuardedRegsBlk, MIMD: DL, MCID: TII->get(Opcode: MOVOpc));
887 for (int i = 0; i < X86::AddrNumOperands; ++i) {
888 if (i == X86::AddrDisp)
889 NewMI.addImm(Val: FrameOffset + VarArgsRegsOffset + RegIdx * 16);
890 else
891 NewMI.add(MO: VAStartPseudoInstr->getOperand(i: i + 1));
892 }
893 NewMI.addReg(RegNo: VAStartPseudoInstr->getOperand(i: OpndIdx).getReg());
894 assert(VAStartPseudoInstr->getOperand(OpndIdx).getReg().isPhysical());
895 }
896
897 // The original block will now fall through to the GuardedRegsBlk.
898 EntryBlk->addSuccessor(Succ: GuardedRegsBlk);
899 // The GuardedRegsBlk will fall through to the TailBlk.
900 GuardedRegsBlk->addSuccessor(Succ: TailBlk);
901
902 if (!STI->isCallingConvWin64(CC: Func->getFunction().getCallingConv())) {
903 // If %al is 0, branch around the XMM save block.
904 BuildMI(BB: EntryBlk, MIMD: DL, MCID: TII->get(Opcode: X86::TEST8rr))
905 .addReg(RegNo: CountReg)
906 .addReg(RegNo: CountReg);
907 BuildMI(BB: EntryBlk, MIMD: DL, MCID: TII->get(Opcode: X86::JCC_1))
908 .addMBB(MBB: TailBlk)
909 .addImm(Val: X86::COND_E);
910 EntryBlk->addSuccessor(Succ: TailBlk);
911 }
912
913 // Add liveins to the created block.
914 addLiveIns(MBB&: *GuardedRegsBlk, LiveRegs);
915 addLiveIns(MBB&: *TailBlk, LiveRegs);
916
917 // Delete the pseudo.
918 VAStartPseudoInstr->eraseFromParent();
919}
920
921/// Expand all pseudo instructions contained in \p MBB.
922/// \returns true if any expansion occurred for \p MBB.
923bool X86ExpandPseudoImpl::expandMBB(MachineBasicBlock &MBB) {
924 bool Modified = false;
925
926 // MBBI may be invalidated by the expansion.
927 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
928 while (MBBI != E) {
929 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
930 Modified |= expandMI(MBB, MBBI);
931 MBBI = NMBBI;
932 }
933
934 return Modified;
935}
936
937bool X86ExpandPseudoImpl::expandPseudosWhichAffectControlFlow(
938 MachineFunction &MF) {
939 // Currently pseudo which affects control flow is only
940 // X86::VASTART_SAVE_XMM_REGS which is located in Entry block.
941 // So we do not need to evaluate other blocks.
942 for (MachineInstr &Instr : MF.front().instrs()) {
943 if (Instr.getOpcode() == X86::VASTART_SAVE_XMM_REGS) {
944 expandVastartSaveXmmRegs(EntryBlk: &(MF.front()), VAStartPseudoInstr: Instr);
945 return true;
946 }
947 }
948
949 return false;
950}
951
952bool X86ExpandPseudoImpl::runOnMachineFunction(MachineFunction &MF) {
953 STI = &MF.getSubtarget<X86Subtarget>();
954 TII = STI->getInstrInfo();
955 TRI = STI->getRegisterInfo();
956 X86FI = MF.getInfo<X86MachineFunctionInfo>();
957 X86FL = STI->getFrameLowering();
958
959 bool Modified = expandPseudosWhichAffectControlFlow(MF);
960
961 for (MachineBasicBlock &MBB : MF)
962 Modified |= expandMBB(MBB);
963 return Modified;
964}
965
966/// Returns an instance of the pseudo instruction expansion pass.
967FunctionPass *llvm::createX86ExpandPseudoLegacyPass() {
968 return new X86ExpandPseudoLegacy();
969}
970
971bool X86ExpandPseudoLegacy::runOnMachineFunction(MachineFunction &MF) {
972 X86ExpandPseudoImpl Impl;
973 return Impl.runOnMachineFunction(MF);
974}
975
976PreservedAnalyses
977X86ExpandPseudoPass::run(MachineFunction &MF,
978 MachineFunctionAnalysisManager &MFAM) {
979 X86ExpandPseudoImpl Impl;
980 bool Changed = Impl.runOnMachineFunction(MF);
981 if (!Changed)
982 return PreservedAnalyses::all();
983
984 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
985 PA.preserveSet<CFGAnalyses>();
986 return PA;
987}
988