1//===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling, if-conversion, other late
11// optimizations, or simply the encoding of the instructions.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86FrameLowering.h"
17#include "X86InstrInfo.h"
18#include "X86MachineFunctionInfo.h"
19#include "X86Subtarget.h"
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineDominators.h"
22#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachinePassManager.h"
26#include "llvm/CodeGen/Passes.h" // For IDs of passes that are preserved.
27#include "llvm/IR/Analysis.h"
28#include "llvm/IR/EHPersonalities.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/Target/TargetMachine.h"
31using namespace llvm;
32
33#define DEBUG_TYPE "x86-expand-pseudo"
34#define X86_EXPAND_PSEUDO_NAME "X86 pseudo instruction expansion pass"
35
36namespace {
37class X86ExpandPseudoImpl {
38public:
39 const X86Subtarget *STI = nullptr;
40 const X86InstrInfo *TII = nullptr;
41 const X86RegisterInfo *TRI = nullptr;
42 const X86MachineFunctionInfo *X86FI = nullptr;
43 const X86FrameLowering *X86FL = nullptr;
44
45 bool runOnMachineFunction(MachineFunction &MF);
46
47private:
48 void expandICallBranchFunnel(MachineBasicBlock *MBB,
49 MachineBasicBlock::iterator MBBI);
50 void expandCALL_RVMARKER(MachineBasicBlock &MBB,
51 MachineBasicBlock::iterator MBBI);
52 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
53 bool expandMBB(MachineBasicBlock &MBB);
54
55 /// This function expands pseudos which affects control flow.
56 /// It is done in separate pass to simplify blocks navigation in main
57 /// pass(calling expandMBB).
58 bool expandPseudosWhichAffectControlFlow(MachineFunction &MF);
59
60 /// Expand X86::VASTART_SAVE_XMM_REGS into set of xmm copying instructions,
61 /// placed into separate block guarded by check for al register(for SystemV
62 /// abi).
63 void expandVastartSaveXmmRegs(
64 MachineBasicBlock *EntryBlk,
65 MachineBasicBlock::iterator VAStartPseudoInstr) const;
66};
67
68class X86ExpandPseudoLegacy : public MachineFunctionPass {
69public:
70 static char ID;
71 X86ExpandPseudoLegacy() : MachineFunctionPass(ID) {}
72
73 void getAnalysisUsage(AnalysisUsage &AU) const override {
74 AU.setPreservesCFG();
75 MachineFunctionPass::getAnalysisUsage(AU);
76 }
77
78 const X86Subtarget *STI = nullptr;
79 const X86InstrInfo *TII = nullptr;
80 const X86RegisterInfo *TRI = nullptr;
81 const X86MachineFunctionInfo *X86FI = nullptr;
82 const X86FrameLowering *X86FL = nullptr;
83
84 bool runOnMachineFunction(MachineFunction &MF) override;
85
86 MachineFunctionProperties getRequiredProperties() const override {
87 return MachineFunctionProperties().setNoVRegs();
88 }
89
90 StringRef getPassName() const override {
91 return "X86 pseudo instruction expansion pass";
92 }
93};
94char X86ExpandPseudoLegacy::ID = 0;
95} // End anonymous namespace.
96
97INITIALIZE_PASS(X86ExpandPseudoLegacy, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME,
98 false, false)
99
100void X86ExpandPseudoImpl::expandICallBranchFunnel(
101 MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) {
102 MachineBasicBlock *JTMBB = MBB;
103 MachineInstr *JTInst = &*MBBI;
104 MachineFunction *MF = MBB->getParent();
105 const BasicBlock *BB = MBB->getBasicBlock();
106 auto InsPt = MachineFunction::iterator(MBB);
107 ++InsPt;
108
109 std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs;
110 const DebugLoc &DL = JTInst->getDebugLoc();
111 MachineOperand Selector = JTInst->getOperand(i: 0);
112 const GlobalValue *CombinedGlobal = JTInst->getOperand(i: 1).getGlobal();
113
114 auto CmpTarget = [&](unsigned Target) {
115 if (Selector.isReg())
116 MBB->addLiveIn(PhysReg: Selector.getReg());
117 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::LEA64r), DestReg: X86::R11)
118 .addReg(RegNo: X86::RIP)
119 .addImm(Val: 1)
120 .addReg(RegNo: 0)
121 .addGlobalAddress(GV: CombinedGlobal,
122 Offset: JTInst->getOperand(i: 2 + 2 * Target).getImm())
123 .addReg(RegNo: 0);
124 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::CMP64rr))
125 .add(MO: Selector)
126 .addReg(RegNo: X86::R11);
127 };
128
129 auto CreateMBB = [&]() {
130 auto *NewMBB = MF->CreateMachineBasicBlock(BB);
131 MBB->addSuccessor(Succ: NewMBB);
132 if (!MBB->isLiveIn(Reg: X86::EFLAGS))
133 MBB->addLiveIn(PhysReg: X86::EFLAGS);
134 return NewMBB;
135 };
136
137 auto EmitCondJump = [&](unsigned CC, MachineBasicBlock *ThenMBB) {
138 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::JCC_1)).addMBB(MBB: ThenMBB).addImm(Val: CC);
139
140 auto *ElseMBB = CreateMBB();
141 MF->insert(MBBI: InsPt, MBB: ElseMBB);
142 MBB = ElseMBB;
143 MBBI = MBB->end();
144 };
145
146 auto EmitCondJumpTarget = [&](unsigned CC, unsigned Target) {
147 auto *ThenMBB = CreateMBB();
148 TargetMBBs.push_back(x: {ThenMBB, Target});
149 EmitCondJump(CC, ThenMBB);
150 };
151
152 auto EmitTailCall = [&](unsigned Target) {
153 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::TAILJMPd64))
154 .add(MO: JTInst->getOperand(i: 3 + 2 * Target));
155 };
156
157 std::function<void(unsigned, unsigned)> EmitBranchFunnel =
158 [&](unsigned FirstTarget, unsigned NumTargets) {
159 if (NumTargets == 1) {
160 EmitTailCall(FirstTarget);
161 return;
162 }
163
164 if (NumTargets == 2) {
165 CmpTarget(FirstTarget + 1);
166 EmitCondJumpTarget(X86::COND_B, FirstTarget);
167 EmitTailCall(FirstTarget + 1);
168 return;
169 }
170
171 if (NumTargets < 6) {
172 CmpTarget(FirstTarget + 1);
173 EmitCondJumpTarget(X86::COND_B, FirstTarget);
174 EmitCondJumpTarget(X86::COND_E, FirstTarget + 1);
175 EmitBranchFunnel(FirstTarget + 2, NumTargets - 2);
176 return;
177 }
178
179 auto *ThenMBB = CreateMBB();
180 CmpTarget(FirstTarget + (NumTargets / 2));
181 EmitCondJump(X86::COND_B, ThenMBB);
182 EmitCondJumpTarget(X86::COND_E, FirstTarget + (NumTargets / 2));
183 EmitBranchFunnel(FirstTarget + (NumTargets / 2) + 1,
184 NumTargets - (NumTargets / 2) - 1);
185
186 MF->insert(MBBI: InsPt, MBB: ThenMBB);
187 MBB = ThenMBB;
188 MBBI = MBB->end();
189 EmitBranchFunnel(FirstTarget, NumTargets / 2);
190 };
191
192 EmitBranchFunnel(0, (JTInst->getNumOperands() - 2) / 2);
193 for (auto P : TargetMBBs) {
194 MF->insert(MBBI: InsPt, MBB: P.first);
195 BuildMI(BB: P.first, MIMD: DL, MCID: TII->get(Opcode: X86::TAILJMPd64))
196 .add(MO: JTInst->getOperand(i: 3 + 2 * P.second));
197 }
198 JTMBB->erase(I: JTInst);
199}
200
201void X86ExpandPseudoImpl::expandCALL_RVMARKER(
202 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
203 // Expand CALL_RVMARKER pseudo to call instruction, followed by the special
204 //"movq %rax, %rdi" marker.
205 MachineInstr &MI = *MBBI;
206
207 MachineInstr *OriginalCall;
208 assert((MI.getOperand(1).isGlobal() || MI.getOperand(1).isReg()) &&
209 "invalid operand for regular call");
210 unsigned Opc = -1;
211 if (MI.getOpcode() == X86::CALL64m_RVMARKER)
212 Opc = X86::CALL64m;
213 else if (MI.getOpcode() == X86::CALL64r_RVMARKER)
214 Opc = X86::CALL64r;
215 else if (MI.getOpcode() == X86::CALL64pcrel32_RVMARKER)
216 Opc = X86::CALL64pcrel32;
217 else
218 llvm_unreachable("unexpected opcode");
219
220 OriginalCall = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc)).getInstr();
221 bool RAXImplicitDead = false;
222 for (MachineOperand &Op : llvm::drop_begin(RangeOrContainer: MI.operands())) {
223 // RAX may be 'implicit dead', if there are no other users of the return
224 // value. We introduce a new use, so change it to 'implicit def'.
225 if (Op.isReg() && Op.isImplicit() && Op.isDead() &&
226 TRI->regsOverlap(RegA: Op.getReg(), RegB: X86::RAX)) {
227 Op.setIsDead(false);
228 Op.setIsDef(true);
229 RAXImplicitDead = true;
230 }
231 OriginalCall->addOperand(Op);
232 }
233
234 // Emit marker "movq %rax, %rdi". %rdi is not callee-saved, so it cannot be
235 // live across the earlier call. The call to the ObjC runtime function returns
236 // the first argument, so the value of %rax is unchanged after the ObjC
237 // runtime call. On Windows targets, the runtime call follows the regular
238 // x64 calling convention and expects the first argument in %rcx.
239 auto TargetReg = STI->getTargetTriple().isOSWindows() ? X86::RCX : X86::RDI;
240 auto *Marker = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: X86::MOV64rr))
241 .addReg(RegNo: TargetReg, Flags: RegState::Define)
242 .addReg(RegNo: X86::RAX)
243 .getInstr();
244 if (MI.shouldUpdateAdditionalCallInfo())
245 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: Marker);
246
247 // Emit call to ObjC runtime.
248 const uint32_t *RegMask =
249 TRI->getCallPreservedMask(MF: *MBB.getParent(), CallingConv::C);
250 MachineInstr *RtCall =
251 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: X86::CALL64pcrel32))
252 .addGlobalAddress(GV: MI.getOperand(i: 0).getGlobal(), Offset: 0, TargetFlags: 0)
253 .addRegMask(Mask: RegMask)
254 .addReg(RegNo: X86::RAX,
255 Flags: RegState::Implicit |
256 (RAXImplicitDead ? (RegState::Dead | RegState::Define)
257 : RegState::Define))
258 .getInstr();
259 MI.eraseFromParent();
260
261 auto &TM = MBB.getParent()->getTarget();
262 // On Darwin platforms, wrap the expanded sequence in a bundle to prevent
263 // later optimizations from breaking up the sequence.
264 if (TM.getTargetTriple().isOSDarwin())
265 finalizeBundle(MBB, FirstMI: OriginalCall->getIterator(),
266 LastMI: std::next(x: RtCall->getIterator()));
267}
268
269/// If \p MBBI is a pseudo instruction, this method expands
270/// it to the corresponding (sequence of) actual instruction(s).
271/// \returns true if \p MBBI has been expanded.
272bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
273 MachineBasicBlock::iterator MBBI) {
274 MachineInstr &MI = *MBBI;
275 unsigned Opcode = MI.getOpcode();
276 const DebugLoc &DL = MBBI->getDebugLoc();
277#define GET_EGPR_IF_ENABLED(OPC) (STI->hasEGPR() ? OPC##_EVEX : OPC)
278 switch (Opcode) {
279 default:
280 return false;
281 case X86::TCRETURNdi:
282 case X86::TCRETURNdicc:
283 case X86::TCRETURNri:
284 case X86::TCRETURN_WIN64ri:
285 case X86::TCRETURN_HIPE32ri:
286 case X86::TCRETURNmi:
287 case X86::TCRETURNdi64:
288 case X86::TCRETURNdi64cc:
289 case X86::TCRETURNri64:
290 case X86::TCRETURNri64_ImpCall:
291 case X86::TCRETURNmi64:
292 case X86::TCRETURN_WINmi64: {
293 bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||
294 Opcode == X86::TCRETURN_WINmi64;
295 MachineOperand &JumpTarget = MBBI->getOperand(i: 0);
296 MachineOperand &StackAdjust = MBBI->getOperand(i: isMem ? X86::AddrNumOperands
297 : 1);
298 assert(StackAdjust.isImm() && "Expecting immediate value.");
299
300 // Adjust stack pointer.
301 int StackAdj = StackAdjust.getImm();
302 int MaxTCDelta = X86FI->getTCReturnAddrDelta();
303 int64_t Offset = 0;
304 assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
305
306 // Incoporate the retaddr area.
307 Offset = StackAdj - MaxTCDelta;
308 assert(Offset >= 0 && "Offset should never be negative");
309
310 if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) {
311 assert(Offset == 0 && "Conditional tail call cannot adjust the stack.");
312 }
313
314 if (Offset) {
315 // Check for possible merge with preceding ADD instruction.
316 Offset = X86FL->mergeSPAdd(MBB, MBBI, AddOffset: Offset, doMergeWithPrevious: true);
317 X86FL->emitSPUpdate(MBB, MBBI, DL, NumBytes: Offset, /*InEpilogue=*/true);
318 }
319
320 // Use this predicate to set REX prefix for X86_64 targets.
321 bool IsX64 = STI->isTargetWin64() || STI->isTargetUEFI64();
322 // Jump to label or value in register.
323 if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||
324 Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {
325 unsigned Op;
326 switch (Opcode) {
327 case X86::TCRETURNdi:
328 Op = X86::TAILJMPd;
329 break;
330 case X86::TCRETURNdicc:
331 Op = X86::TAILJMPd_CC;
332 break;
333 case X86::TCRETURNdi64cc:
334 assert(!MBB.getParent()->hasWinCFI() &&
335 "Conditional tail calls confuse "
336 "the Win64 unwinder.");
337 Op = X86::TAILJMPd64_CC;
338 break;
339 default:
340 // Note: Win64 uses REX prefixes indirect jumps out of functions, but
341 // not direct ones.
342 Op = X86::TAILJMPd64;
343 break;
344 }
345 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Op));
346 if (JumpTarget.isGlobal()) {
347 MIB.addGlobalAddress(GV: JumpTarget.getGlobal(), Offset: JumpTarget.getOffset(),
348 TargetFlags: JumpTarget.getTargetFlags());
349 } else {
350 assert(JumpTarget.isSymbol());
351 MIB.addExternalSymbol(FnName: JumpTarget.getSymbolName(),
352 TargetFlags: JumpTarget.getTargetFlags());
353 }
354 if (Op == X86::TAILJMPd_CC || Op == X86::TAILJMPd64_CC) {
355 MIB.addImm(Val: MBBI->getOperand(i: 2).getImm());
356 }
357
358 } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||
359 Opcode == X86::TCRETURN_WINmi64) {
360 unsigned Op = (Opcode == X86::TCRETURNmi)
361 ? X86::TAILJMPm
362 : (IsX64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
363 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Op));
364 for (unsigned i = 0; i != X86::AddrNumOperands; ++i)
365 MIB.add(MO: MBBI->getOperand(i));
366 } else if (Opcode == X86::TCRETURNri64 ||
367 Opcode == X86::TCRETURNri64_ImpCall ||
368 Opcode == X86::TCRETURN_WIN64ri) {
369 JumpTarget.setIsKill();
370 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
371 MCID: TII->get(Opcode: IsX64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
372 .add(MO: JumpTarget);
373 } else {
374 assert(!IsX64 && "Win64 and UEFI64 require REX for indirect jumps.");
375 JumpTarget.setIsKill();
376 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::TAILJMPr))
377 .add(MO: JumpTarget);
378 }
379
380 MachineInstr &NewMI = *std::prev(x: MBBI);
381 NewMI.copyImplicitOps(MF&: *MBBI->getParent()->getParent(), MI: *MBBI);
382 NewMI.setCFIType(MF&: *MBB.getParent(), Type: MI.getCFIType());
383
384 // Update the call info.
385 if (MBBI->isCandidateForAdditionalCallInfo())
386 MBB.getParent()->moveAdditionalCallInfo(Old: &*MBBI, New: &NewMI);
387
388 // Delete the pseudo instruction TCRETURN.
389 MBB.erase(I: MBBI);
390
391 return true;
392 }
393 case X86::EH_RETURN:
394 case X86::EH_RETURN64: {
395 MachineOperand &DestAddr = MBBI->getOperand(i: 0);
396 assert(DestAddr.isReg() && "Offset should be in register!");
397 const bool Uses64BitFramePtr = STI->isTarget64BitLP64();
398 Register StackPtr = TRI->getStackRegister();
399 BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
400 MCID: TII->get(Opcode: Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), DestReg: StackPtr)
401 .addReg(RegNo: DestAddr.getReg());
402 if (STI->hasSHSTK()) {
403 unsigned PopOpcode = STI->is64Bit() ? X86::POP64r : X86::POP32r;
404 unsigned JumpOpcode = X86::JMP32r;
405 if (Uses64BitFramePtr)
406 JumpOpcode = STI->isTargetWin64() || STI->isTargetUEFI64()
407 ? X86::JMP64r_REX
408 : X86::JMP64r;
409 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: PopOpcode))
410 .addReg(RegNo: DestAddr.getReg(), Flags: RegState::Define);
411 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: JumpOpcode)).addReg(RegNo: DestAddr.getReg());
412 MBB.erase(I: MBBI);
413 }
414 // The EH_RETURN pseudo is really removed during the MC Lowering.
415 return true;
416 }
417 case X86::IRET: {
418 // Adjust stack to erase error code
419 int64_t StackAdj = MBBI->getOperand(i: 0).getImm();
420 X86FL->emitSPUpdate(MBB, MBBI, DL, NumBytes: StackAdj, InEpilogue: true);
421 // Replace pseudo with machine iret
422 unsigned RetOp = STI->is64Bit() ? X86::IRET64 : X86::IRET32;
423 // Use UIRET if UINTR is present (except for building kernel)
424 if (STI->is64Bit() && STI->hasUINTR() &&
425 MBB.getParent()->getTarget().getCodeModel() != CodeModel::Kernel)
426 RetOp = X86::UIRET;
427 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: RetOp));
428 MBB.erase(I: MBBI);
429 return true;
430 }
431 case X86::RET: {
432 // Adjust stack to erase error code
433 int64_t StackAdj = MBBI->getOperand(i: 0).getImm();
434 MachineInstrBuilder MIB;
435 if (StackAdj == 0) {
436 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
437 MCID: TII->get(Opcode: STI->is64Bit() ? X86::RET64 : X86::RET32));
438 } else if (isUInt<16>(x: StackAdj)) {
439 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL,
440 MCID: TII->get(Opcode: STI->is64Bit() ? X86::RETI64 : X86::RETI32))
441 .addImm(Val: StackAdj);
442 } else {
443 assert(!STI->is64Bit() &&
444 "shouldn't need to do this for x86_64 targets!");
445 // A ret can only handle immediates as big as 2**16-1. If we need to pop
446 // off bytes before the return address, we must do it manually.
447 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::POP32r)).addReg(RegNo: X86::ECX, Flags: RegState::Define);
448 X86FL->emitSPUpdate(MBB, MBBI, DL, NumBytes: StackAdj, /*InEpilogue=*/true);
449 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::PUSH32r)).addReg(RegNo: X86::ECX);
450 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::RET32));
451 }
452 for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; ++I)
453 MIB.add(MO: MBBI->getOperand(i: I));
454 MBB.erase(I: MBBI);
455 return true;
456 }
457 case X86::LCMPXCHG16B_SAVE_RBX: {
458 // Perform the following transformation.
459 // SaveRbx = pseudocmpxchg Addr, <4 opds for the address>, InArg, SaveRbx
460 // =>
461 // RBX = InArg
462 // actualcmpxchg Addr
463 // RBX = SaveRbx
464 const MachineOperand &InArg = MBBI->getOperand(i: 6);
465 Register SaveRbx = MBBI->getOperand(i: 7).getReg();
466
467 // Copy the input argument of the pseudo into the argument of the
468 // actual instruction.
469 // NOTE: We don't copy the kill flag since the input might be the same reg
470 // as one of the other operands of LCMPXCHG16B.
471 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::RBX, SrcReg: InArg.getReg(), KillSrc: false);
472 // Create the actual instruction.
473 MachineInstr *NewInstr = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::LCMPXCHG16B));
474 // Copy the operands related to the address. If we access a frame variable,
475 // we need to replace the RBX base with SaveRbx, as RBX has another value.
476 const MachineOperand &Base = MBBI->getOperand(i: 1);
477 if (Base.getReg() == X86::RBX || Base.getReg() == X86::EBX)
478 NewInstr->addOperand(Op: MachineOperand::CreateReg(
479 Reg: Base.getReg() == X86::RBX
480 ? SaveRbx
481 : Register(TRI->getSubReg(Reg: SaveRbx, Idx: X86::sub_32bit)),
482 /*IsDef=*/isDef: false));
483 else
484 NewInstr->addOperand(Op: Base);
485 for (unsigned Idx = 1 + 1; Idx < 1 + X86::AddrNumOperands; ++Idx)
486 NewInstr->addOperand(Op: MBBI->getOperand(i: Idx));
487 NewInstr->getOperand(i: 7).setIsDead(MBBI->getOperand(i: 11).isDead());
488 // Finally, restore the value of RBX.
489 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::RBX, SrcReg: SaveRbx,
490 /*SrcIsKill*/ KillSrc: true);
491
492 // Delete the pseudo.
493 MBBI->eraseFromParent();
494 return true;
495 }
496 // Loading/storing mask pairs requires two kmov operations. The second one of
497 // these needs a 2 byte displacement relative to the specified address (with
498 // 32 bit spill size). The pairs of 1bit masks up to 16 bit masks all use the
499 // same spill size, they all are stored using MASKPAIR16STORE, loaded using
500 // MASKPAIR16LOAD.
501 //
502 // The displacement value might wrap around in theory, thus the asserts in
503 // both cases.
504 case X86::MASKPAIR16LOAD: {
505 int64_t Disp = MBBI->getOperand(i: 1 + X86::AddrDisp).getImm();
506 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");
507 Register Reg = MBBI->getOperand(i: 0).getReg();
508 bool DstIsDead = MBBI->getOperand(i: 0).isDead();
509 Register Reg0 = TRI->getSubReg(Reg, Idx: X86::sub_mask_0);
510 Register Reg1 = TRI->getSubReg(Reg, Idx: X86::sub_mask_1);
511
512 auto MIBLo =
513 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm)))
514 .addReg(RegNo: Reg0, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
515 auto MIBHi =
516 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWkm)))
517 .addReg(RegNo: Reg1, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
518
519 for (int i = 0; i < X86::AddrNumOperands; ++i) {
520 MIBLo.add(MO: MBBI->getOperand(i: 1 + i));
521 if (i == X86::AddrDisp)
522 MIBHi.addImm(Val: Disp + 2);
523 else
524 MIBHi.add(MO: MBBI->getOperand(i: 1 + i));
525 }
526
527 // Split the memory operand, adjusting the offset and size for the halves.
528 MachineMemOperand *OldMMO = MBBI->memoperands().front();
529 MachineFunction *MF = MBB.getParent();
530 MachineMemOperand *MMOLo = MF->getMachineMemOperand(MMO: OldMMO, Offset: 0, Size: 2);
531 MachineMemOperand *MMOHi = MF->getMachineMemOperand(MMO: OldMMO, Offset: 2, Size: 2);
532
533 MIBLo.setMemRefs(MMOLo);
534 MIBHi.setMemRefs(MMOHi);
535
536 // Delete the pseudo.
537 MBB.erase(I: MBBI);
538 return true;
539 }
540 case X86::MASKPAIR16STORE: {
541 int64_t Disp = MBBI->getOperand(i: X86::AddrDisp).getImm();
542 assert(Disp >= 0 && Disp <= INT32_MAX - 2 && "Unexpected displacement");
543 Register Reg = MBBI->getOperand(i: X86::AddrNumOperands).getReg();
544 bool SrcIsKill = MBBI->getOperand(i: X86::AddrNumOperands).isKill();
545 Register Reg0 = TRI->getSubReg(Reg, Idx: X86::sub_mask_0);
546 Register Reg1 = TRI->getSubReg(Reg, Idx: X86::sub_mask_1);
547
548 auto MIBLo =
549 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk)));
550 auto MIBHi =
551 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(GET_EGPR_IF_ENABLED(X86::KMOVWmk)));
552
553 for (int i = 0; i < X86::AddrNumOperands; ++i) {
554 MIBLo.add(MO: MBBI->getOperand(i));
555 if (i == X86::AddrDisp)
556 MIBHi.addImm(Val: Disp + 2);
557 else
558 MIBHi.add(MO: MBBI->getOperand(i));
559 }
560 MIBLo.addReg(RegNo: Reg0, Flags: getKillRegState(B: SrcIsKill));
561 MIBHi.addReg(RegNo: Reg1, Flags: getKillRegState(B: SrcIsKill));
562
563 // Split the memory operand, adjusting the offset and size for the halves.
564 MachineMemOperand *OldMMO = MBBI->memoperands().front();
565 MachineFunction *MF = MBB.getParent();
566 MachineMemOperand *MMOLo = MF->getMachineMemOperand(MMO: OldMMO, Offset: 0, Size: 2);
567 MachineMemOperand *MMOHi = MF->getMachineMemOperand(MMO: OldMMO, Offset: 2, Size: 2);
568
569 MIBLo.setMemRefs(MMOLo);
570 MIBHi.setMemRefs(MMOHi);
571
572 // Delete the pseudo.
573 MBB.erase(I: MBBI);
574 return true;
575 }
576 case X86::MWAITX_SAVE_RBX: {
577 // Perform the following transformation.
578 // SaveRbx = pseudomwaitx InArg, SaveRbx
579 // =>
580 // [E|R]BX = InArg
581 // actualmwaitx
582 // [E|R]BX = SaveRbx
583 const MachineOperand &InArg = MBBI->getOperand(i: 1);
584 // Copy the input argument of the pseudo into the argument of the
585 // actual instruction.
586 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::EBX, SrcReg: InArg.getReg(), KillSrc: InArg.isKill());
587 // Create the actual instruction.
588 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: X86::MWAITXrrr));
589 // Finally, restore the value of RBX.
590 Register SaveRbx = MBBI->getOperand(i: 2).getReg();
591 TII->copyPhysReg(MBB, MI: MBBI, DL, DestReg: X86::RBX, SrcReg: SaveRbx, /*SrcIsKill*/ KillSrc: true);
592 // Delete the pseudo.
593 MBBI->eraseFromParent();
594 return true;
595 }
596 case TargetOpcode::ICALL_BRANCH_FUNNEL:
597 expandICallBranchFunnel(MBB: &MBB, MBBI);
598 return true;
599 case X86::PLDTILECFGV: {
600 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::LDTILECFG)));
601 return true;
602 }
603 case X86::PTILELOADDV:
604 case X86::PTILELOADDT1V:
605 case X86::PTILELOADDRSV:
606 case X86::PTILELOADDRST1V:
607 case X86::PTCVTROWD2PSrteV:
608 case X86::PTCVTROWD2PSrtiV:
609 case X86::PTCVTROWPS2BF16HrteV:
610 case X86::PTCVTROWPS2BF16HrtiV:
611 case X86::PTCVTROWPS2BF16LrteV:
612 case X86::PTCVTROWPS2BF16LrtiV:
613 case X86::PTCVTROWPS2PHHrteV:
614 case X86::PTCVTROWPS2PHHrtiV:
615 case X86::PTCVTROWPS2PHLrteV:
616 case X86::PTCVTROWPS2PHLrtiV:
617 case X86::PTILEMOVROWrteV:
618 case X86::PTILEMOVROWrtiV: {
619 for (unsigned i = 2; i > 0; --i)
620 MI.removeOperand(OpNo: i);
621 unsigned Opc;
622 switch (Opcode) {
623 case X86::PTILELOADDRSV:
624 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRS);
625 break;
626 case X86::PTILELOADDRST1V:
627 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDRST1);
628 break;
629 case X86::PTILELOADDV:
630 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
631 break;
632 case X86::PTILELOADDT1V:
633 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1);
634 break;
635 case X86::PTCVTROWD2PSrteV:
636 Opc = X86::TCVTROWD2PSrte;
637 break;
638 case X86::PTCVTROWD2PSrtiV:
639 Opc = X86::TCVTROWD2PSrti;
640 break;
641 case X86::PTCVTROWPS2BF16HrteV:
642 Opc = X86::TCVTROWPS2BF16Hrte;
643 break;
644 case X86::PTCVTROWPS2BF16HrtiV:
645 Opc = X86::TCVTROWPS2BF16Hrti;
646 break;
647 case X86::PTCVTROWPS2BF16LrteV:
648 Opc = X86::TCVTROWPS2BF16Lrte;
649 break;
650 case X86::PTCVTROWPS2BF16LrtiV:
651 Opc = X86::TCVTROWPS2BF16Lrti;
652 break;
653 case X86::PTCVTROWPS2PHHrteV:
654 Opc = X86::TCVTROWPS2PHHrte;
655 break;
656 case X86::PTCVTROWPS2PHHrtiV:
657 Opc = X86::TCVTROWPS2PHHrti;
658 break;
659 case X86::PTCVTROWPS2PHLrteV:
660 Opc = X86::TCVTROWPS2PHLrte;
661 break;
662 case X86::PTCVTROWPS2PHLrtiV:
663 Opc = X86::TCVTROWPS2PHLrti;
664 break;
665 case X86::PTILEMOVROWrteV:
666 Opc = X86::TILEMOVROWrte;
667 break;
668 case X86::PTILEMOVROWrtiV:
669 Opc = X86::TILEMOVROWrti;
670 break;
671 default:
672 llvm_unreachable("Unexpected Opcode");
673 }
674 MI.setDesc(TII->get(Opcode: Opc));
675 return true;
676 }
677 case X86::PTCMMIMFP16PSV:
678 case X86::PTCMMRLFP16PSV:
679 case X86::PTDPBSSDV:
680 case X86::PTDPBSUDV:
681 case X86::PTDPBUSDV:
682 case X86::PTDPBUUDV:
683 case X86::PTDPBF16PSV:
684 case X86::PTDPFP16PSV:
685 case X86::PTDPBF8PSV:
686 case X86::PTDPBHF8PSV:
687 case X86::PTDPHBF8PSV:
688 case X86::PTDPHF8PSV: {
689 MI.untieRegOperand(OpIdx: 4);
690 for (unsigned i = 3; i > 0; --i)
691 MI.removeOperand(OpNo: i);
692 unsigned Opc;
693 switch (Opcode) {
694 // clang-format off
695 case X86::PTCMMIMFP16PSV: Opc = X86::TCMMIMFP16PS; break;
696 case X86::PTCMMRLFP16PSV: Opc = X86::TCMMRLFP16PS; break;
697 case X86::PTDPBSSDV: Opc = X86::TDPBSSD; break;
698 case X86::PTDPBSUDV: Opc = X86::TDPBSUD; break;
699 case X86::PTDPBUSDV: Opc = X86::TDPBUSD; break;
700 case X86::PTDPBUUDV: Opc = X86::TDPBUUD; break;
701 case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS; break;
702 case X86::PTDPFP16PSV: Opc = X86::TDPFP16PS; break;
703 case X86::PTDPBF8PSV: Opc = X86::TDPBF8PS; break;
704 case X86::PTDPBHF8PSV: Opc = X86::TDPBHF8PS; break;
705 case X86::PTDPHBF8PSV: Opc = X86::TDPHBF8PS; break;
706 case X86::PTDPHF8PSV: Opc = X86::TDPHF8PS; break;
707 // clang-format on
708 default:
709 llvm_unreachable("Unexpected Opcode");
710 }
711 MI.setDesc(TII->get(Opcode: Opc));
712 MI.tieOperands(DefIdx: 0, UseIdx: 1);
713 return true;
714 }
715 case X86::PTILESTOREDV: {
716 for (int i = 1; i >= 0; --i)
717 MI.removeOperand(OpNo: i);
718 MI.setDesc(TII->get(GET_EGPR_IF_ENABLED(X86::TILESTORED)));
719 return true;
720 }
721#undef GET_EGPR_IF_ENABLED
722 case X86::PTILEZEROV: {
723 for (int i = 2; i > 0; --i) // Remove row, col
724 MI.removeOperand(OpNo: i);
725 MI.setDesc(TII->get(Opcode: X86::TILEZERO));
726 return true;
727 }
728 case X86::CALL64pcrel32_RVMARKER:
729 case X86::CALL64r_RVMARKER:
730 case X86::CALL64m_RVMARKER:
731 expandCALL_RVMARKER(MBB, MBBI);
732 return true;
733 case X86::CALL64r_ImpCall:
734 MI.setDesc(TII->get(Opcode: X86::CALL64r));
735 return true;
736 case X86::ADD32mi_ND:
737 case X86::ADD64mi32_ND:
738 case X86::SUB32mi_ND:
739 case X86::SUB64mi32_ND:
740 case X86::AND32mi_ND:
741 case X86::AND64mi32_ND:
742 case X86::OR32mi_ND:
743 case X86::OR64mi32_ND:
744 case X86::XOR32mi_ND:
745 case X86::XOR64mi32_ND:
746 case X86::ADC32mi_ND:
747 case X86::ADC64mi32_ND:
748 case X86::SBB32mi_ND:
749 case X86::SBB64mi32_ND: {
750 // It's possible for an EVEX-encoded legacy instruction to reach the 15-byte
751 // instruction length limit: 4 bytes of EVEX prefix + 1 byte of opcode + 1
752 // byte of ModRM + 1 byte of SIB + 4 bytes of displacement + 4 bytes of
753 // immediate = 15 bytes in total, e.g.
754 //
755 // subq $184, %fs:257(%rbx, %rcx), %rax
756 //
757 // In such a case, no additional (ADSIZE or segment override) prefix can be
758 // used. To resolve the issue, we split the “long” instruction into 2
759 // instructions:
760 //
761 // movq %fs:257(%rbx, %rcx)，%rax
762 // subq $184, %rax
763 //
764 // Therefore we consider the OPmi_ND to be a pseudo instruction to some
765 // extent.
766 const MachineOperand &ImmOp =
767 MI.getOperand(i: MI.getNumExplicitOperands() - 1);
768 // If the immediate is a expr, conservatively estimate 4 bytes.
769 if (ImmOp.isImm() && isInt<8>(x: ImmOp.getImm()))
770 return false;
771 int MemOpNo = X86::getFirstAddrOperandIdx(MI);
772 const MachineOperand &DispOp = MI.getOperand(i: MemOpNo + X86::AddrDisp);
773 Register Base = MI.getOperand(i: MemOpNo + X86::AddrBaseReg).getReg();
774 // If the displacement is a expr, conservatively estimate 4 bytes.
775 if (Base && DispOp.isImm() && isInt<8>(x: DispOp.getImm()))
776 return false;
777 // There can only be one of three: SIB, segment override register, ADSIZE
778 Register Index = MI.getOperand(i: MemOpNo + X86::AddrIndexReg).getReg();
779 unsigned Count = !!MI.getOperand(i: MemOpNo + X86::AddrSegmentReg).getReg();
780 if (X86II::needSIB(BaseReg: Base, IndexReg: Index, /*In64BitMode=*/true))
781 ++Count;
782 if (getX86MCRegisterClass(RC: X86::GR32RegClassID).contains(Reg: Base) ||
783 getX86MCRegisterClass(RC: X86::GR32RegClassID).contains(Reg: Index))
784 ++Count;
785 if (Count < 2)
786 return false;
787 unsigned Opc, LoadOpc;
788 switch (Opcode) {
789#define MI_TO_RI(OP) \
790 case X86::OP##32mi_ND: \
791 Opc = X86::OP##32ri; \
792 LoadOpc = X86::MOV32rm; \
793 break; \
794 case X86::OP##64mi32_ND: \
795 Opc = X86::OP##64ri32; \
796 LoadOpc = X86::MOV64rm; \
797 break;
798
799 default:
800 llvm_unreachable("Unexpected Opcode");
801 MI_TO_RI(ADD);
802 MI_TO_RI(SUB);
803 MI_TO_RI(AND);
804 MI_TO_RI(OR);
805 MI_TO_RI(XOR);
806 MI_TO_RI(ADC);
807 MI_TO_RI(SBB);
808#undef MI_TO_RI
809 }
810 // Insert OPri.
811 Register DestReg = MI.getOperand(i: 0).getReg();
812 BuildMI(BB&: MBB, I: std::next(x: MBBI), MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg)
813 .addReg(RegNo: DestReg)
814 .add(MO: ImmOp);
815 // Change OPmi_ND to MOVrm.
816 for (unsigned I = MI.getNumImplicitOperands() + 1; I != 0; --I)
817 MI.removeOperand(OpNo: MI.getNumOperands() - 1);
818 MI.setDesc(TII->get(Opcode: LoadOpc));
819 return true;
820 }
821 }
822 llvm_unreachable("Previous switch has a fallthrough?");
823}
824
825// This function creates additional block for storing varargs guarded
826// registers. It adds check for %al into entry block, to skip
827// GuardedRegsBlk if xmm registers should not be stored.
828//
829// EntryBlk[VAStartPseudoInstr] EntryBlk
830// | | .
831// | | .
832// | | GuardedRegsBlk
833// | => | .
834// | | .
835// | TailBlk
836// | |
837// | |
838//
839void X86ExpandPseudoImpl::expandVastartSaveXmmRegs(
840 MachineBasicBlock *EntryBlk,
841 MachineBasicBlock::iterator VAStartPseudoInstr) const {
842 assert(VAStartPseudoInstr->getOpcode() == X86::VASTART_SAVE_XMM_REGS);
843
844 MachineFunction *Func = EntryBlk->getParent();
845 const TargetInstrInfo *TII = STI->getInstrInfo();
846 const DebugLoc &DL = VAStartPseudoInstr->getDebugLoc();
847 Register CountReg = VAStartPseudoInstr->getOperand(i: 0).getReg();
848
849 // Calculate liveins for newly created blocks.
850 LivePhysRegs LiveRegs(*STI->getRegisterInfo());
851 SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers;
852
853 LiveRegs.addLiveIns(MBB: *EntryBlk);
854 for (MachineInstr &MI : EntryBlk->instrs()) {
855 if (MI.getOpcode() == VAStartPseudoInstr->getOpcode())
856 break;
857
858 LiveRegs.stepForward(MI, Clobbers);
859 }
860
861 // Create the new basic blocks. One block contains all the XMM stores,
862 // and another block is the final destination regardless of whether any
863 // stores were performed.
864 const BasicBlock *LLVMBlk = EntryBlk->getBasicBlock();
865 MachineFunction::iterator EntryBlkIter = ++EntryBlk->getIterator();
866 MachineBasicBlock *GuardedRegsBlk = Func->CreateMachineBasicBlock(BB: LLVMBlk);
867 MachineBasicBlock *TailBlk = Func->CreateMachineBasicBlock(BB: LLVMBlk);
868 Func->insert(MBBI: EntryBlkIter, MBB: GuardedRegsBlk);
869 Func->insert(MBBI: EntryBlkIter, MBB: TailBlk);
870
871 // Transfer the remainder of EntryBlk and its successor edges to TailBlk.
872 TailBlk->splice(Where: TailBlk->begin(), Other: EntryBlk,
873 From: std::next(x: MachineBasicBlock::iterator(VAStartPseudoInstr)),
874 To: EntryBlk->end());
875 TailBlk->transferSuccessorsAndUpdatePHIs(FromMBB: EntryBlk);
876
877 uint64_t FrameOffset = VAStartPseudoInstr->getOperand(i: 4).getImm();
878 uint64_t VarArgsRegsOffset = VAStartPseudoInstr->getOperand(i: 6).getImm();
879
880 // TODO: add support for YMM and ZMM here.
881 unsigned MOVOpc = STI->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
882
883 // In the XMM save block, save all the XMM argument registers.
884 for (int64_t OpndIdx = 7, RegIdx = 0;
885 OpndIdx < VAStartPseudoInstr->getNumOperands() - 1;
886 OpndIdx++, RegIdx++) {
887 auto NewMI = BuildMI(BB: GuardedRegsBlk, MIMD: DL, MCID: TII->get(Opcode: MOVOpc));
888 for (int i = 0; i < X86::AddrNumOperands; ++i) {
889 if (i == X86::AddrDisp)
890 NewMI.addImm(Val: FrameOffset + VarArgsRegsOffset + RegIdx * 16);
891 else
892 NewMI.add(MO: VAStartPseudoInstr->getOperand(i: i + 1));
893 }
894 NewMI.addReg(RegNo: VAStartPseudoInstr->getOperand(i: OpndIdx).getReg());
895 assert(VAStartPseudoInstr->getOperand(OpndIdx).getReg().isPhysical());
896 }
897
898 // The original block will now fall through to the GuardedRegsBlk.
899 EntryBlk->addSuccessor(Succ: GuardedRegsBlk);
900 // The GuardedRegsBlk will fall through to the TailBlk.
901 GuardedRegsBlk->addSuccessor(Succ: TailBlk);
902
903 if (!STI->isCallingConvWin64(CC: Func->getFunction().getCallingConv())) {
904 // If %al is 0, branch around the XMM save block.
905 BuildMI(BB: EntryBlk, MIMD: DL, MCID: TII->get(Opcode: X86::TEST8rr))
906 .addReg(RegNo: CountReg)
907 .addReg(RegNo: CountReg);
908 BuildMI(BB: EntryBlk, MIMD: DL, MCID: TII->get(Opcode: X86::JCC_1))
909 .addMBB(MBB: TailBlk)
910 .addImm(Val: X86::COND_E);
911 EntryBlk->addSuccessor(Succ: TailBlk);
912 }
913
914 // Add liveins to the created block.
915 addLiveIns(MBB&: *GuardedRegsBlk, LiveRegs);
916 addLiveIns(MBB&: *TailBlk, LiveRegs);
917
918 // Delete the pseudo.
919 VAStartPseudoInstr->eraseFromParent();
920}
921
922/// Expand all pseudo instructions contained in \p MBB.
923/// \returns true if any expansion occurred for \p MBB.
924bool X86ExpandPseudoImpl::expandMBB(MachineBasicBlock &MBB) {
925 bool Modified = false;
926
927 // MBBI may be invalidated by the expansion.
928 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
929 while (MBBI != E) {
930 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
931 Modified |= expandMI(MBB, MBBI);
932 MBBI = NMBBI;
933 }
934
935 return Modified;
936}
937
938bool X86ExpandPseudoImpl::expandPseudosWhichAffectControlFlow(
939 MachineFunction &MF) {
940 // Currently pseudo which affects control flow is only
941 // X86::VASTART_SAVE_XMM_REGS which is located in Entry block.
942 // So we do not need to evaluate other blocks.
943 for (MachineInstr &Instr : MF.front().instrs()) {
944 if (Instr.getOpcode() == X86::VASTART_SAVE_XMM_REGS) {
945 expandVastartSaveXmmRegs(EntryBlk: &(MF.front()), VAStartPseudoInstr: Instr);
946 return true;
947 }
948 }
949
950 return false;
951}
952
953bool X86ExpandPseudoImpl::runOnMachineFunction(MachineFunction &MF) {
954 STI = &MF.getSubtarget<X86Subtarget>();
955 TII = STI->getInstrInfo();
956 TRI = STI->getRegisterInfo();
957 X86FI = MF.getInfo<X86MachineFunctionInfo>();
958 X86FL = STI->getFrameLowering();
959
960 bool Modified = expandPseudosWhichAffectControlFlow(MF);
961
962 for (MachineBasicBlock &MBB : MF)
963 Modified |= expandMBB(MBB);
964 return Modified;
965}
966
967/// Returns an instance of the pseudo instruction expansion pass.
968FunctionPass *llvm::createX86ExpandPseudoLegacyPass() {
969 return new X86ExpandPseudoLegacy();
970}
971
972bool X86ExpandPseudoLegacy::runOnMachineFunction(MachineFunction &MF) {
973 X86ExpandPseudoImpl Impl;
974 return Impl.runOnMachineFunction(MF);
975}
976
977PreservedAnalyses
978X86ExpandPseudoPass::run(MachineFunction &MF,
979 MachineFunctionAnalysisManager &MFAM) {
980 X86ExpandPseudoImpl Impl;
981 bool Changed = Impl.runOnMachineFunction(MF);
982 if (!Changed)
983 return PreservedAnalyses::all();
984
985 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
986 PA.preserveSet<CFGAnalyses>();
987 return PA;
988}
989