1//===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling, if-conversion, and other late
11// optimizations. This pass should be run after register allocation but before
12// the post-regalloc scheduling pass.
13//
14//===----------------------------------------------------------------------===//
15
16#include "ARM.h"
17#include "ARMBaseInstrInfo.h"
18#include "ARMBaseRegisterInfo.h"
19#include "ARMConstantPoolValue.h"
20#include "ARMMachineFunctionInfo.h"
21#include "ARMSubtarget.h"
22#include "MCTargetDesc/ARMAddressingModes.h"
23#include "llvm/CodeGen/LivePhysRegs.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunctionPass.h"
26#include "llvm/CodeGen/MachineJumpTableInfo.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/Support/Debug.h"
29
30#include <atomic>
31
32using namespace llvm;
33
34#define DEBUG_TYPE "arm-pseudo"
35
36static cl::opt<bool>
37VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
38 cl::desc("Verify machine code after expanding ARM pseudos"));
39
40#define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"
41
42namespace {
43 class ARMExpandPseudo : public MachineFunctionPass {
44 public:
45 static char ID;
46 ARMExpandPseudo() : MachineFunctionPass(ID) {}
47
48 const ARMBaseInstrInfo *TII;
49 const TargetRegisterInfo *TRI;
50 const ARMSubtarget *STI;
51 ARMFunctionInfo *AFI;
52
53 bool runOnMachineFunction(MachineFunction &Fn) override;
54
55 MachineFunctionProperties getRequiredProperties() const override {
56 return MachineFunctionProperties().setNoVRegs();
57 }
58
59 StringRef getPassName() const override {
60 return ARM_EXPAND_PSEUDO_NAME;
61 }
62
63 private:
64 bool ExpandMI(MachineBasicBlock &MBB,
65 MachineBasicBlock::iterator MBBI,
66 MachineBasicBlock::iterator &NextMBBI);
67 bool ExpandMBB(MachineBasicBlock &MBB);
68 void ExpandVLD(MachineBasicBlock::iterator &MBBI);
69 void ExpandVST(MachineBasicBlock::iterator &MBBI);
70 void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);
71 void ExpandVTBL(MachineBasicBlock::iterator &MBBI,
72 unsigned Opc, bool IsExt);
73 void ExpandMQQPRLoadStore(MachineBasicBlock::iterator &MBBI);
74 void ExpandTMOV32BitImm(MachineBasicBlock &MBB,
75 MachineBasicBlock::iterator &MBBI);
76 void ExpandMOV32BitImm(MachineBasicBlock &MBB,
77 MachineBasicBlock::iterator &MBBI);
78 void CMSEClearGPRegs(MachineBasicBlock &MBB,
79 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
80 const SmallVectorImpl<unsigned> &ClearRegs,
81 unsigned ClobberReg);
82 MachineBasicBlock &CMSEClearFPRegs(MachineBasicBlock &MBB,
83 MachineBasicBlock::iterator MBBI);
84 MachineBasicBlock &CMSEClearFPRegsV8(MachineBasicBlock &MBB,
85 MachineBasicBlock::iterator MBBI,
86 const BitVector &ClearRegs);
87 MachineBasicBlock &CMSEClearFPRegsV81(MachineBasicBlock &MBB,
88 MachineBasicBlock::iterator MBBI,
89 const BitVector &ClearRegs);
90 void CMSESaveClearFPRegs(MachineBasicBlock &MBB,
91 MachineBasicBlock::iterator MBBI, DebugLoc &DL,
92 const LivePhysRegs &LiveRegs,
93 SmallVectorImpl<unsigned> &AvailableRegs);
94 void CMSESaveClearFPRegsV8(MachineBasicBlock &MBB,
95 MachineBasicBlock::iterator MBBI, DebugLoc &DL,
96 const LivePhysRegs &LiveRegs,
97 SmallVectorImpl<unsigned> &ScratchRegs);
98 void CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,
99 MachineBasicBlock::iterator MBBI, DebugLoc &DL,
100 const LivePhysRegs &LiveRegs);
101 void CMSERestoreFPRegs(MachineBasicBlock &MBB,
102 MachineBasicBlock::iterator MBBI, DebugLoc &DL,
103 SmallVectorImpl<unsigned> &AvailableRegs);
104 void CMSERestoreFPRegsV8(MachineBasicBlock &MBB,
105 MachineBasicBlock::iterator MBBI, DebugLoc &DL,
106 SmallVectorImpl<unsigned> &AvailableRegs);
107 void CMSERestoreFPRegsV81(MachineBasicBlock &MBB,
108 MachineBasicBlock::iterator MBBI, DebugLoc &DL,
109 SmallVectorImpl<unsigned> &AvailableRegs);
110 bool ExpandCMP_SWAP(MachineBasicBlock &MBB,
111 MachineBasicBlock::iterator MBBI, unsigned LdrexOp,
112 unsigned StrexOp, unsigned UxtOp,
113 MachineBasicBlock::iterator &NextMBBI);
114
115 bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
116 MachineBasicBlock::iterator MBBI,
117 MachineBasicBlock::iterator &NextMBBI);
118 };
119 char ARMExpandPseudo::ID = 0;
120}
121
122INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,
123 false)
124
125namespace {
126 // Constants for register spacing in NEON load/store instructions.
127 // For quad-register load-lane and store-lane pseudo instructors, the
128 // spacing is initially assumed to be EvenDblSpc, and that is changed to
129 // OddDblSpc depending on the lane number operand.
130 enum NEONRegSpacing {
131 SingleSpc,
132 SingleLowSpc , // Single spacing, low registers, three and four vectors.
133 SingleHighQSpc, // Single spacing, high registers, four vectors.
134 SingleHighTSpc, // Single spacing, high registers, three vectors.
135 EvenDblSpc,
136 OddDblSpc
137 };
138
139 // Entries for NEON load/store information table. The table is sorted by
140 // PseudoOpc for fast binary-search lookups.
141 struct NEONLdStTableEntry {
142 uint16_t PseudoOpc;
143 uint16_t RealOpc;
144 bool IsLoad;
145 bool isUpdating;
146 bool hasWritebackOperand;
147 uint8_t RegSpacing; // One of type NEONRegSpacing
148 uint8_t NumRegs; // D registers loaded or stored
149 uint8_t RegElts; // elements per D register; used for lane ops
150 // FIXME: Temporary flag to denote whether the real instruction takes
151 // a single register (like the encoding) or all of the registers in
152 // the list (like the asm syntax and the isel DAG). When all definitions
153 // are converted to take only the single encoded register, this will
154 // go away.
155 bool copyAllListRegs;
156
157 // Comparison methods for binary search of the table.
158 bool operator<(const NEONLdStTableEntry &TE) const {
159 return PseudoOpc < TE.PseudoOpc;
160 }
161 friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {
162 return TE.PseudoOpc < PseudoOpc;
163 }
164 [[maybe_unused]] friend bool operator<(unsigned PseudoOpc,
165 const NEONLdStTableEntry &TE) {
166 return PseudoOpc < TE.PseudoOpc;
167 }
168 };
169}
170
171static const NEONLdStTableEntry NEONLdStTable[] = {
172{ .PseudoOpc: ARM::VLD1LNq16Pseudo, .RealOpc: ARM::VLD1LNd16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 4 ,.copyAllListRegs: true},
173{ .PseudoOpc: ARM::VLD1LNq16Pseudo_UPD, .RealOpc: ARM::VLD1LNd16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 4 ,.copyAllListRegs: true},
174{ .PseudoOpc: ARM::VLD1LNq32Pseudo, .RealOpc: ARM::VLD1LNd32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 2 ,.copyAllListRegs: true},
175{ .PseudoOpc: ARM::VLD1LNq32Pseudo_UPD, .RealOpc: ARM::VLD1LNd32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 2 ,.copyAllListRegs: true},
176{ .PseudoOpc: ARM::VLD1LNq8Pseudo, .RealOpc: ARM::VLD1LNd8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 8 ,.copyAllListRegs: true},
177{ .PseudoOpc: ARM::VLD1LNq8Pseudo_UPD, .RealOpc: ARM::VLD1LNd8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 8 ,.copyAllListRegs: true},
178
179{ .PseudoOpc: ARM::VLD1d16QPseudo, .RealOpc: ARM::VLD1d16Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
180{ .PseudoOpc: ARM::VLD1d16QPseudoWB_fixed, .RealOpc: ARM::VLD1d16Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
181{ .PseudoOpc: ARM::VLD1d16QPseudoWB_register, .RealOpc: ARM::VLD1d16Qwb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
182{ .PseudoOpc: ARM::VLD1d16TPseudo, .RealOpc: ARM::VLD1d16T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
183{ .PseudoOpc: ARM::VLD1d16TPseudoWB_fixed, .RealOpc: ARM::VLD1d16Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
184{ .PseudoOpc: ARM::VLD1d16TPseudoWB_register, .RealOpc: ARM::VLD1d16Twb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
185
186{ .PseudoOpc: ARM::VLD1d32QPseudo, .RealOpc: ARM::VLD1d32Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
187{ .PseudoOpc: ARM::VLD1d32QPseudoWB_fixed, .RealOpc: ARM::VLD1d32Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
188{ .PseudoOpc: ARM::VLD1d32QPseudoWB_register, .RealOpc: ARM::VLD1d32Qwb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
189{ .PseudoOpc: ARM::VLD1d32TPseudo, .RealOpc: ARM::VLD1d32T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
190{ .PseudoOpc: ARM::VLD1d32TPseudoWB_fixed, .RealOpc: ARM::VLD1d32Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
191{ .PseudoOpc: ARM::VLD1d32TPseudoWB_register, .RealOpc: ARM::VLD1d32Twb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
192
193{ .PseudoOpc: ARM::VLD1d64QPseudo, .RealOpc: ARM::VLD1d64Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
194{ .PseudoOpc: ARM::VLD1d64QPseudoWB_fixed, .RealOpc: ARM::VLD1d64Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
195{ .PseudoOpc: ARM::VLD1d64QPseudoWB_register, .RealOpc: ARM::VLD1d64Qwb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
196{ .PseudoOpc: ARM::VLD1d64TPseudo, .RealOpc: ARM::VLD1d64T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
197{ .PseudoOpc: ARM::VLD1d64TPseudoWB_fixed, .RealOpc: ARM::VLD1d64Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
198{ .PseudoOpc: ARM::VLD1d64TPseudoWB_register, .RealOpc: ARM::VLD1d64Twb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
199
200{ .PseudoOpc: ARM::VLD1d8QPseudo, .RealOpc: ARM::VLD1d8Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
201{ .PseudoOpc: ARM::VLD1d8QPseudoWB_fixed, .RealOpc: ARM::VLD1d8Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
202{ .PseudoOpc: ARM::VLD1d8QPseudoWB_register, .RealOpc: ARM::VLD1d8Qwb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
203{ .PseudoOpc: ARM::VLD1d8TPseudo, .RealOpc: ARM::VLD1d8T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
204{ .PseudoOpc: ARM::VLD1d8TPseudoWB_fixed, .RealOpc: ARM::VLD1d8Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
205{ .PseudoOpc: ARM::VLD1d8TPseudoWB_register, .RealOpc: ARM::VLD1d8Twb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
206
207{ .PseudoOpc: ARM::VLD1q16HighQPseudo, .RealOpc: ARM::VLD1d16Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
208{ .PseudoOpc: ARM::VLD1q16HighQPseudo_UPD, .RealOpc: ARM::VLD1d16Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
209{ .PseudoOpc: ARM::VLD1q16HighTPseudo, .RealOpc: ARM::VLD1d16T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
210{ .PseudoOpc: ARM::VLD1q16HighTPseudo_UPD, .RealOpc: ARM::VLD1d16Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
211{ .PseudoOpc: ARM::VLD1q16LowQPseudo_UPD, .RealOpc: ARM::VLD1d16Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
212{ .PseudoOpc: ARM::VLD1q16LowTPseudo_UPD, .RealOpc: ARM::VLD1d16Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
213
214{ .PseudoOpc: ARM::VLD1q32HighQPseudo, .RealOpc: ARM::VLD1d32Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
215{ .PseudoOpc: ARM::VLD1q32HighQPseudo_UPD, .RealOpc: ARM::VLD1d32Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
216{ .PseudoOpc: ARM::VLD1q32HighTPseudo, .RealOpc: ARM::VLD1d32T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
217{ .PseudoOpc: ARM::VLD1q32HighTPseudo_UPD, .RealOpc: ARM::VLD1d32Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
218{ .PseudoOpc: ARM::VLD1q32LowQPseudo_UPD, .RealOpc: ARM::VLD1d32Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
219{ .PseudoOpc: ARM::VLD1q32LowTPseudo_UPD, .RealOpc: ARM::VLD1d32Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
220
221{ .PseudoOpc: ARM::VLD1q64HighQPseudo, .RealOpc: ARM::VLD1d64Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
222{ .PseudoOpc: ARM::VLD1q64HighQPseudo_UPD, .RealOpc: ARM::VLD1d64Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
223{ .PseudoOpc: ARM::VLD1q64HighTPseudo, .RealOpc: ARM::VLD1d64T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
224{ .PseudoOpc: ARM::VLD1q64HighTPseudo_UPD, .RealOpc: ARM::VLD1d64Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
225{ .PseudoOpc: ARM::VLD1q64LowQPseudo_UPD, .RealOpc: ARM::VLD1d64Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
226{ .PseudoOpc: ARM::VLD1q64LowTPseudo_UPD, .RealOpc: ARM::VLD1d64Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
227
228{ .PseudoOpc: ARM::VLD1q8HighQPseudo, .RealOpc: ARM::VLD1d8Q, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
229{ .PseudoOpc: ARM::VLD1q8HighQPseudo_UPD, .RealOpc: ARM::VLD1d8Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
230{ .PseudoOpc: ARM::VLD1q8HighTPseudo, .RealOpc: ARM::VLD1d8T, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
231{ .PseudoOpc: ARM::VLD1q8HighTPseudo_UPD, .RealOpc: ARM::VLD1d8Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
232{ .PseudoOpc: ARM::VLD1q8LowQPseudo_UPD, .RealOpc: ARM::VLD1d8Qwb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
233{ .PseudoOpc: ARM::VLD1q8LowTPseudo_UPD, .RealOpc: ARM::VLD1d8Twb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
234
235{ .PseudoOpc: ARM::VLD2DUPq16EvenPseudo, .RealOpc: ARM::VLD2DUPd16x2, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: false},
236{ .PseudoOpc: ARM::VLD2DUPq16OddPseudo, .RealOpc: ARM::VLD2DUPd16x2, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: false},
237{ .PseudoOpc: ARM::VLD2DUPq16OddPseudoWB_fixed, .RealOpc: ARM::VLD2DUPd16x2wb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: false},
238{ .PseudoOpc: ARM::VLD2DUPq16OddPseudoWB_register, .RealOpc: ARM::VLD2DUPd16x2wb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: false},
239{ .PseudoOpc: ARM::VLD2DUPq32EvenPseudo, .RealOpc: ARM::VLD2DUPd32x2, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: false},
240{ .PseudoOpc: ARM::VLD2DUPq32OddPseudo, .RealOpc: ARM::VLD2DUPd32x2, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: false},
241{ .PseudoOpc: ARM::VLD2DUPq32OddPseudoWB_fixed, .RealOpc: ARM::VLD2DUPd32x2wb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: false},
242{ .PseudoOpc: ARM::VLD2DUPq32OddPseudoWB_register, .RealOpc: ARM::VLD2DUPd32x2wb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: false},
243{ .PseudoOpc: ARM::VLD2DUPq8EvenPseudo, .RealOpc: ARM::VLD2DUPd8x2, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: false},
244{ .PseudoOpc: ARM::VLD2DUPq8OddPseudo, .RealOpc: ARM::VLD2DUPd8x2, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: false},
245{ .PseudoOpc: ARM::VLD2DUPq8OddPseudoWB_fixed, .RealOpc: ARM::VLD2DUPd8x2wb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: false},
246{ .PseudoOpc: ARM::VLD2DUPq8OddPseudoWB_register, .RealOpc: ARM::VLD2DUPd8x2wb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: false},
247
248{ .PseudoOpc: ARM::VLD2LNd16Pseudo, .RealOpc: ARM::VLD2LNd16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: true},
249{ .PseudoOpc: ARM::VLD2LNd16Pseudo_UPD, .RealOpc: ARM::VLD2LNd16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: true},
250{ .PseudoOpc: ARM::VLD2LNd32Pseudo, .RealOpc: ARM::VLD2LNd32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: true},
251{ .PseudoOpc: ARM::VLD2LNd32Pseudo_UPD, .RealOpc: ARM::VLD2LNd32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: true},
252{ .PseudoOpc: ARM::VLD2LNd8Pseudo, .RealOpc: ARM::VLD2LNd8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: true},
253{ .PseudoOpc: ARM::VLD2LNd8Pseudo_UPD, .RealOpc: ARM::VLD2LNd8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: true},
254{ .PseudoOpc: ARM::VLD2LNq16Pseudo, .RealOpc: ARM::VLD2LNq16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: true},
255{ .PseudoOpc: ARM::VLD2LNq16Pseudo_UPD, .RealOpc: ARM::VLD2LNq16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: true},
256{ .PseudoOpc: ARM::VLD2LNq32Pseudo, .RealOpc: ARM::VLD2LNq32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: true},
257{ .PseudoOpc: ARM::VLD2LNq32Pseudo_UPD, .RealOpc: ARM::VLD2LNq32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: true},
258
259{ .PseudoOpc: ARM::VLD2q16Pseudo, .RealOpc: ARM::VLD2q16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
260{ .PseudoOpc: ARM::VLD2q16PseudoWB_fixed, .RealOpc: ARM::VLD2q16wb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
261{ .PseudoOpc: ARM::VLD2q16PseudoWB_register, .RealOpc: ARM::VLD2q16wb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
262{ .PseudoOpc: ARM::VLD2q32Pseudo, .RealOpc: ARM::VLD2q32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
263{ .PseudoOpc: ARM::VLD2q32PseudoWB_fixed, .RealOpc: ARM::VLD2q32wb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
264{ .PseudoOpc: ARM::VLD2q32PseudoWB_register, .RealOpc: ARM::VLD2q32wb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
265{ .PseudoOpc: ARM::VLD2q8Pseudo, .RealOpc: ARM::VLD2q8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
266{ .PseudoOpc: ARM::VLD2q8PseudoWB_fixed, .RealOpc: ARM::VLD2q8wb_fixed, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
267{ .PseudoOpc: ARM::VLD2q8PseudoWB_register, .RealOpc: ARM::VLD2q8wb_register, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
268
269{ .PseudoOpc: ARM::VLD3DUPd16Pseudo, .RealOpc: ARM::VLD3DUPd16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4,.copyAllListRegs: true},
270{ .PseudoOpc: ARM::VLD3DUPd16Pseudo_UPD, .RealOpc: ARM::VLD3DUPd16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4,.copyAllListRegs: true},
271{ .PseudoOpc: ARM::VLD3DUPd32Pseudo, .RealOpc: ARM::VLD3DUPd32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2,.copyAllListRegs: true},
272{ .PseudoOpc: ARM::VLD3DUPd32Pseudo_UPD, .RealOpc: ARM::VLD3DUPd32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2,.copyAllListRegs: true},
273{ .PseudoOpc: ARM::VLD3DUPd8Pseudo, .RealOpc: ARM::VLD3DUPd8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8,.copyAllListRegs: true},
274{ .PseudoOpc: ARM::VLD3DUPd8Pseudo_UPD, .RealOpc: ARM::VLD3DUPd8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8,.copyAllListRegs: true},
275{ .PseudoOpc: ARM::VLD3DUPq16EvenPseudo, .RealOpc: ARM::VLD3DUPq16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
276{ .PseudoOpc: ARM::VLD3DUPq16OddPseudo, .RealOpc: ARM::VLD3DUPq16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
277{ .PseudoOpc: ARM::VLD3DUPq16OddPseudo_UPD, .RealOpc: ARM::VLD3DUPq16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
278{ .PseudoOpc: ARM::VLD3DUPq32EvenPseudo, .RealOpc: ARM::VLD3DUPq32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
279{ .PseudoOpc: ARM::VLD3DUPq32OddPseudo, .RealOpc: ARM::VLD3DUPq32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
280{ .PseudoOpc: ARM::VLD3DUPq32OddPseudo_UPD, .RealOpc: ARM::VLD3DUPq32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
281{ .PseudoOpc: ARM::VLD3DUPq8EvenPseudo, .RealOpc: ARM::VLD3DUPq8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
282{ .PseudoOpc: ARM::VLD3DUPq8OddPseudo, .RealOpc: ARM::VLD3DUPq8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
283{ .PseudoOpc: ARM::VLD3DUPq8OddPseudo_UPD, .RealOpc: ARM::VLD3DUPq8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
284
285{ .PseudoOpc: ARM::VLD3LNd16Pseudo, .RealOpc: ARM::VLD3LNd16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
286{ .PseudoOpc: ARM::VLD3LNd16Pseudo_UPD, .RealOpc: ARM::VLD3LNd16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
287{ .PseudoOpc: ARM::VLD3LNd32Pseudo, .RealOpc: ARM::VLD3LNd32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
288{ .PseudoOpc: ARM::VLD3LNd32Pseudo_UPD, .RealOpc: ARM::VLD3LNd32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
289{ .PseudoOpc: ARM::VLD3LNd8Pseudo, .RealOpc: ARM::VLD3LNd8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
290{ .PseudoOpc: ARM::VLD3LNd8Pseudo_UPD, .RealOpc: ARM::VLD3LNd8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
291{ .PseudoOpc: ARM::VLD3LNq16Pseudo, .RealOpc: ARM::VLD3LNq16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
292{ .PseudoOpc: ARM::VLD3LNq16Pseudo_UPD, .RealOpc: ARM::VLD3LNq16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
293{ .PseudoOpc: ARM::VLD3LNq32Pseudo, .RealOpc: ARM::VLD3LNq32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
294{ .PseudoOpc: ARM::VLD3LNq32Pseudo_UPD, .RealOpc: ARM::VLD3LNq32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
295
296{ .PseudoOpc: ARM::VLD3d16Pseudo, .RealOpc: ARM::VLD3d16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
297{ .PseudoOpc: ARM::VLD3d16Pseudo_UPD, .RealOpc: ARM::VLD3d16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
298{ .PseudoOpc: ARM::VLD3d32Pseudo, .RealOpc: ARM::VLD3d32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
299{ .PseudoOpc: ARM::VLD3d32Pseudo_UPD, .RealOpc: ARM::VLD3d32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
300{ .PseudoOpc: ARM::VLD3d8Pseudo, .RealOpc: ARM::VLD3d8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
301{ .PseudoOpc: ARM::VLD3d8Pseudo_UPD, .RealOpc: ARM::VLD3d8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
302
303{ .PseudoOpc: ARM::VLD3q16Pseudo_UPD, .RealOpc: ARM::VLD3q16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
304{ .PseudoOpc: ARM::VLD3q16oddPseudo, .RealOpc: ARM::VLD3q16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
305{ .PseudoOpc: ARM::VLD3q16oddPseudo_UPD, .RealOpc: ARM::VLD3q16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
306{ .PseudoOpc: ARM::VLD3q32Pseudo_UPD, .RealOpc: ARM::VLD3q32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
307{ .PseudoOpc: ARM::VLD3q32oddPseudo, .RealOpc: ARM::VLD3q32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
308{ .PseudoOpc: ARM::VLD3q32oddPseudo_UPD, .RealOpc: ARM::VLD3q32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
309{ .PseudoOpc: ARM::VLD3q8Pseudo_UPD, .RealOpc: ARM::VLD3q8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
310{ .PseudoOpc: ARM::VLD3q8oddPseudo, .RealOpc: ARM::VLD3q8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
311{ .PseudoOpc: ARM::VLD3q8oddPseudo_UPD, .RealOpc: ARM::VLD3q8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
312
313{ .PseudoOpc: ARM::VLD4DUPd16Pseudo, .RealOpc: ARM::VLD4DUPd16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4,.copyAllListRegs: true},
314{ .PseudoOpc: ARM::VLD4DUPd16Pseudo_UPD, .RealOpc: ARM::VLD4DUPd16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4,.copyAllListRegs: true},
315{ .PseudoOpc: ARM::VLD4DUPd32Pseudo, .RealOpc: ARM::VLD4DUPd32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2,.copyAllListRegs: true},
316{ .PseudoOpc: ARM::VLD4DUPd32Pseudo_UPD, .RealOpc: ARM::VLD4DUPd32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2,.copyAllListRegs: true},
317{ .PseudoOpc: ARM::VLD4DUPd8Pseudo, .RealOpc: ARM::VLD4DUPd8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8,.copyAllListRegs: true},
318{ .PseudoOpc: ARM::VLD4DUPd8Pseudo_UPD, .RealOpc: ARM::VLD4DUPd8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8,.copyAllListRegs: true},
319{ .PseudoOpc: ARM::VLD4DUPq16EvenPseudo, .RealOpc: ARM::VLD4DUPq16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
320{ .PseudoOpc: ARM::VLD4DUPq16OddPseudo, .RealOpc: ARM::VLD4DUPq16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
321{ .PseudoOpc: ARM::VLD4DUPq16OddPseudo_UPD, .RealOpc: ARM::VLD4DUPq16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
322{ .PseudoOpc: ARM::VLD4DUPq32EvenPseudo, .RealOpc: ARM::VLD4DUPq32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
323{ .PseudoOpc: ARM::VLD4DUPq32OddPseudo, .RealOpc: ARM::VLD4DUPq32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
324{ .PseudoOpc: ARM::VLD4DUPq32OddPseudo_UPD, .RealOpc: ARM::VLD4DUPq32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
325{ .PseudoOpc: ARM::VLD4DUPq8EvenPseudo, .RealOpc: ARM::VLD4DUPq8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
326{ .PseudoOpc: ARM::VLD4DUPq8OddPseudo, .RealOpc: ARM::VLD4DUPq8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
327{ .PseudoOpc: ARM::VLD4DUPq8OddPseudo_UPD, .RealOpc: ARM::VLD4DUPq8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
328
329{ .PseudoOpc: ARM::VLD4LNd16Pseudo, .RealOpc: ARM::VLD4LNd16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
330{ .PseudoOpc: ARM::VLD4LNd16Pseudo_UPD, .RealOpc: ARM::VLD4LNd16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
331{ .PseudoOpc: ARM::VLD4LNd32Pseudo, .RealOpc: ARM::VLD4LNd32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
332{ .PseudoOpc: ARM::VLD4LNd32Pseudo_UPD, .RealOpc: ARM::VLD4LNd32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
333{ .PseudoOpc: ARM::VLD4LNd8Pseudo, .RealOpc: ARM::VLD4LNd8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
334{ .PseudoOpc: ARM::VLD4LNd8Pseudo_UPD, .RealOpc: ARM::VLD4LNd8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
335{ .PseudoOpc: ARM::VLD4LNq16Pseudo, .RealOpc: ARM::VLD4LNq16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
336{ .PseudoOpc: ARM::VLD4LNq16Pseudo_UPD, .RealOpc: ARM::VLD4LNq16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
337{ .PseudoOpc: ARM::VLD4LNq32Pseudo, .RealOpc: ARM::VLD4LNq32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
338{ .PseudoOpc: ARM::VLD4LNq32Pseudo_UPD, .RealOpc: ARM::VLD4LNq32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
339
340{ .PseudoOpc: ARM::VLD4d16Pseudo, .RealOpc: ARM::VLD4d16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
341{ .PseudoOpc: ARM::VLD4d16Pseudo_UPD, .RealOpc: ARM::VLD4d16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
342{ .PseudoOpc: ARM::VLD4d32Pseudo, .RealOpc: ARM::VLD4d32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
343{ .PseudoOpc: ARM::VLD4d32Pseudo_UPD, .RealOpc: ARM::VLD4d32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
344{ .PseudoOpc: ARM::VLD4d8Pseudo, .RealOpc: ARM::VLD4d8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
345{ .PseudoOpc: ARM::VLD4d8Pseudo_UPD, .RealOpc: ARM::VLD4d8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
346
347{ .PseudoOpc: ARM::VLD4q16Pseudo_UPD, .RealOpc: ARM::VLD4q16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
348{ .PseudoOpc: ARM::VLD4q16oddPseudo, .RealOpc: ARM::VLD4q16, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
349{ .PseudoOpc: ARM::VLD4q16oddPseudo_UPD, .RealOpc: ARM::VLD4q16_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
350{ .PseudoOpc: ARM::VLD4q32Pseudo_UPD, .RealOpc: ARM::VLD4q32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
351{ .PseudoOpc: ARM::VLD4q32oddPseudo, .RealOpc: ARM::VLD4q32, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
352{ .PseudoOpc: ARM::VLD4q32oddPseudo_UPD, .RealOpc: ARM::VLD4q32_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
353{ .PseudoOpc: ARM::VLD4q8Pseudo_UPD, .RealOpc: ARM::VLD4q8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
354{ .PseudoOpc: ARM::VLD4q8oddPseudo, .RealOpc: ARM::VLD4q8, .IsLoad: true, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
355{ .PseudoOpc: ARM::VLD4q8oddPseudo_UPD, .RealOpc: ARM::VLD4q8_UPD, .IsLoad: true, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
356
357{ .PseudoOpc: ARM::VST1LNq16Pseudo, .RealOpc: ARM::VST1LNd16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 4 ,.copyAllListRegs: true},
358{ .PseudoOpc: ARM::VST1LNq16Pseudo_UPD, .RealOpc: ARM::VST1LNd16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 4 ,.copyAllListRegs: true},
359{ .PseudoOpc: ARM::VST1LNq32Pseudo, .RealOpc: ARM::VST1LNd32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 2 ,.copyAllListRegs: true},
360{ .PseudoOpc: ARM::VST1LNq32Pseudo_UPD, .RealOpc: ARM::VST1LNd32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 2 ,.copyAllListRegs: true},
361{ .PseudoOpc: ARM::VST1LNq8Pseudo, .RealOpc: ARM::VST1LNd8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 8 ,.copyAllListRegs: true},
362{ .PseudoOpc: ARM::VST1LNq8Pseudo_UPD, .RealOpc: ARM::VST1LNd8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 1, .RegElts: 8 ,.copyAllListRegs: true},
363
364{ .PseudoOpc: ARM::VST1d16QPseudo, .RealOpc: ARM::VST1d16Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
365{ .PseudoOpc: ARM::VST1d16QPseudoWB_fixed, .RealOpc: ARM::VST1d16Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
366{ .PseudoOpc: ARM::VST1d16QPseudoWB_register, .RealOpc: ARM::VST1d16Qwb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
367{ .PseudoOpc: ARM::VST1d16TPseudo, .RealOpc: ARM::VST1d16T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
368{ .PseudoOpc: ARM::VST1d16TPseudoWB_fixed, .RealOpc: ARM::VST1d16Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
369{ .PseudoOpc: ARM::VST1d16TPseudoWB_register, .RealOpc: ARM::VST1d16Twb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
370
371{ .PseudoOpc: ARM::VST1d32QPseudo, .RealOpc: ARM::VST1d32Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
372{ .PseudoOpc: ARM::VST1d32QPseudoWB_fixed, .RealOpc: ARM::VST1d32Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
373{ .PseudoOpc: ARM::VST1d32QPseudoWB_register, .RealOpc: ARM::VST1d32Qwb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
374{ .PseudoOpc: ARM::VST1d32TPseudo, .RealOpc: ARM::VST1d32T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
375{ .PseudoOpc: ARM::VST1d32TPseudoWB_fixed, .RealOpc: ARM::VST1d32Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
376{ .PseudoOpc: ARM::VST1d32TPseudoWB_register, .RealOpc: ARM::VST1d32Twb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
377
378{ .PseudoOpc: ARM::VST1d64QPseudo, .RealOpc: ARM::VST1d64Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
379{ .PseudoOpc: ARM::VST1d64QPseudoWB_fixed, .RealOpc: ARM::VST1d64Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
380{ .PseudoOpc: ARM::VST1d64QPseudoWB_register, .RealOpc: ARM::VST1d64Qwb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
381{ .PseudoOpc: ARM::VST1d64TPseudo, .RealOpc: ARM::VST1d64T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
382{ .PseudoOpc: ARM::VST1d64TPseudoWB_fixed, .RealOpc: ARM::VST1d64Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
383{ .PseudoOpc: ARM::VST1d64TPseudoWB_register, .RealOpc: ARM::VST1d64Twb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
384
385{ .PseudoOpc: ARM::VST1d8QPseudo, .RealOpc: ARM::VST1d8Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
386{ .PseudoOpc: ARM::VST1d8QPseudoWB_fixed, .RealOpc: ARM::VST1d8Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
387{ .PseudoOpc: ARM::VST1d8QPseudoWB_register, .RealOpc: ARM::VST1d8Qwb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
388{ .PseudoOpc: ARM::VST1d8TPseudo, .RealOpc: ARM::VST1d8T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
389{ .PseudoOpc: ARM::VST1d8TPseudoWB_fixed, .RealOpc: ARM::VST1d8Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
390{ .PseudoOpc: ARM::VST1d8TPseudoWB_register, .RealOpc: ARM::VST1d8Twb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
391
392{ .PseudoOpc: ARM::VST1q16HighQPseudo, .RealOpc: ARM::VST1d16Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
393{ .PseudoOpc: ARM::VST1q16HighQPseudo_UPD, .RealOpc: ARM::VST1d16Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
394{ .PseudoOpc: ARM::VST1q16HighTPseudo, .RealOpc: ARM::VST1d16T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
395{ .PseudoOpc: ARM::VST1q16HighTPseudo_UPD, .RealOpc: ARM::VST1d16Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
396{ .PseudoOpc: ARM::VST1q16LowQPseudo_UPD, .RealOpc: ARM::VST1d16Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
397{ .PseudoOpc: ARM::VST1q16LowTPseudo_UPD, .RealOpc: ARM::VST1d16Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: false},
398
399{ .PseudoOpc: ARM::VST1q32HighQPseudo, .RealOpc: ARM::VST1d32Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
400{ .PseudoOpc: ARM::VST1q32HighQPseudo_UPD, .RealOpc: ARM::VST1d32Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
401{ .PseudoOpc: ARM::VST1q32HighTPseudo, .RealOpc: ARM::VST1d32T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
402{ .PseudoOpc: ARM::VST1q32HighTPseudo_UPD, .RealOpc: ARM::VST1d32Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
403{ .PseudoOpc: ARM::VST1q32LowQPseudo_UPD, .RealOpc: ARM::VST1d32Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
404{ .PseudoOpc: ARM::VST1q32LowTPseudo_UPD, .RealOpc: ARM::VST1d32Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: false},
405
406{ .PseudoOpc: ARM::VST1q64HighQPseudo, .RealOpc: ARM::VST1d64Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
407{ .PseudoOpc: ARM::VST1q64HighQPseudo_UPD, .RealOpc: ARM::VST1d64Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
408{ .PseudoOpc: ARM::VST1q64HighTPseudo, .RealOpc: ARM::VST1d64T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
409{ .PseudoOpc: ARM::VST1q64HighTPseudo_UPD, .RealOpc: ARM::VST1d64Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
410{ .PseudoOpc: ARM::VST1q64LowQPseudo_UPD, .RealOpc: ARM::VST1d64Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 1 ,.copyAllListRegs: false},
411{ .PseudoOpc: ARM::VST1q64LowTPseudo_UPD, .RealOpc: ARM::VST1d64Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 1 ,.copyAllListRegs: false},
412
413{ .PseudoOpc: ARM::VST1q8HighQPseudo, .RealOpc: ARM::VST1d8Q, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
414{ .PseudoOpc: ARM::VST1q8HighQPseudo_UPD, .RealOpc: ARM::VST1d8Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighQSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
415{ .PseudoOpc: ARM::VST1q8HighTPseudo, .RealOpc: ARM::VST1d8T, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
416{ .PseudoOpc: ARM::VST1q8HighTPseudo_UPD, .RealOpc: ARM::VST1d8Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleHighTSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
417{ .PseudoOpc: ARM::VST1q8LowQPseudo_UPD, .RealOpc: ARM::VST1d8Qwb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
418{ .PseudoOpc: ARM::VST1q8LowTPseudo_UPD, .RealOpc: ARM::VST1d8Twb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleLowSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: false},
419
420{ .PseudoOpc: ARM::VST2LNd16Pseudo, .RealOpc: ARM::VST2LNd16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: true},
421{ .PseudoOpc: ARM::VST2LNd16Pseudo_UPD, .RealOpc: ARM::VST2LNd16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 4 ,.copyAllListRegs: true},
422{ .PseudoOpc: ARM::VST2LNd32Pseudo, .RealOpc: ARM::VST2LNd32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: true},
423{ .PseudoOpc: ARM::VST2LNd32Pseudo_UPD, .RealOpc: ARM::VST2LNd32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 2 ,.copyAllListRegs: true},
424{ .PseudoOpc: ARM::VST2LNd8Pseudo, .RealOpc: ARM::VST2LNd8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: true},
425{ .PseudoOpc: ARM::VST2LNd8Pseudo_UPD, .RealOpc: ARM::VST2LNd8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 2, .RegElts: 8 ,.copyAllListRegs: true},
426{ .PseudoOpc: ARM::VST2LNq16Pseudo, .RealOpc: ARM::VST2LNq16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 4,.copyAllListRegs: true},
427{ .PseudoOpc: ARM::VST2LNq16Pseudo_UPD, .RealOpc: ARM::VST2LNq16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 4,.copyAllListRegs: true},
428{ .PseudoOpc: ARM::VST2LNq32Pseudo, .RealOpc: ARM::VST2LNq32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 2,.copyAllListRegs: true},
429{ .PseudoOpc: ARM::VST2LNq32Pseudo_UPD, .RealOpc: ARM::VST2LNq32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 2, .RegElts: 2,.copyAllListRegs: true},
430
431{ .PseudoOpc: ARM::VST2q16Pseudo, .RealOpc: ARM::VST2q16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
432{ .PseudoOpc: ARM::VST2q16PseudoWB_fixed, .RealOpc: ARM::VST2q16wb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
433{ .PseudoOpc: ARM::VST2q16PseudoWB_register, .RealOpc: ARM::VST2q16wb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: false},
434{ .PseudoOpc: ARM::VST2q32Pseudo, .RealOpc: ARM::VST2q32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
435{ .PseudoOpc: ARM::VST2q32PseudoWB_fixed, .RealOpc: ARM::VST2q32wb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
436{ .PseudoOpc: ARM::VST2q32PseudoWB_register, .RealOpc: ARM::VST2q32wb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: false},
437{ .PseudoOpc: ARM::VST2q8Pseudo, .RealOpc: ARM::VST2q8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
438{ .PseudoOpc: ARM::VST2q8PseudoWB_fixed, .RealOpc: ARM::VST2q8wb_fixed, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
439{ .PseudoOpc: ARM::VST2q8PseudoWB_register, .RealOpc: ARM::VST2q8wb_register, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: false},
440
441{ .PseudoOpc: ARM::VST3LNd16Pseudo, .RealOpc: ARM::VST3LNd16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
442{ .PseudoOpc: ARM::VST3LNd16Pseudo_UPD, .RealOpc: ARM::VST3LNd16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
443{ .PseudoOpc: ARM::VST3LNd32Pseudo, .RealOpc: ARM::VST3LNd32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
444{ .PseudoOpc: ARM::VST3LNd32Pseudo_UPD, .RealOpc: ARM::VST3LNd32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
445{ .PseudoOpc: ARM::VST3LNd8Pseudo, .RealOpc: ARM::VST3LNd8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
446{ .PseudoOpc: ARM::VST3LNd8Pseudo_UPD, .RealOpc: ARM::VST3LNd8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
447{ .PseudoOpc: ARM::VST3LNq16Pseudo, .RealOpc: ARM::VST3LNq16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 4,.copyAllListRegs: true},
448{ .PseudoOpc: ARM::VST3LNq16Pseudo_UPD, .RealOpc: ARM::VST3LNq16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 4,.copyAllListRegs: true},
449{ .PseudoOpc: ARM::VST3LNq32Pseudo, .RealOpc: ARM::VST3LNq32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 2,.copyAllListRegs: true},
450{ .PseudoOpc: ARM::VST3LNq32Pseudo_UPD, .RealOpc: ARM::VST3LNq32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 2,.copyAllListRegs: true},
451
452{ .PseudoOpc: ARM::VST3d16Pseudo, .RealOpc: ARM::VST3d16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
453{ .PseudoOpc: ARM::VST3d16Pseudo_UPD, .RealOpc: ARM::VST3d16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
454{ .PseudoOpc: ARM::VST3d32Pseudo, .RealOpc: ARM::VST3d32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
455{ .PseudoOpc: ARM::VST3d32Pseudo_UPD, .RealOpc: ARM::VST3d32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
456{ .PseudoOpc: ARM::VST3d8Pseudo, .RealOpc: ARM::VST3d8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
457{ .PseudoOpc: ARM::VST3d8Pseudo_UPD, .RealOpc: ARM::VST3d8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
458
459{ .PseudoOpc: ARM::VST3q16Pseudo_UPD, .RealOpc: ARM::VST3q16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
460{ .PseudoOpc: ARM::VST3q16oddPseudo, .RealOpc: ARM::VST3q16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
461{ .PseudoOpc: ARM::VST3q16oddPseudo_UPD, .RealOpc: ARM::VST3q16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 4 ,.copyAllListRegs: true},
462{ .PseudoOpc: ARM::VST3q32Pseudo_UPD, .RealOpc: ARM::VST3q32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
463{ .PseudoOpc: ARM::VST3q32oddPseudo, .RealOpc: ARM::VST3q32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
464{ .PseudoOpc: ARM::VST3q32oddPseudo_UPD, .RealOpc: ARM::VST3q32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 2 ,.copyAllListRegs: true},
465{ .PseudoOpc: ARM::VST3q8Pseudo_UPD, .RealOpc: ARM::VST3q8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
466{ .PseudoOpc: ARM::VST3q8oddPseudo, .RealOpc: ARM::VST3q8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
467{ .PseudoOpc: ARM::VST3q8oddPseudo_UPD, .RealOpc: ARM::VST3q8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 3, .RegElts: 8 ,.copyAllListRegs: true},
468
469{ .PseudoOpc: ARM::VST4LNd16Pseudo, .RealOpc: ARM::VST4LNd16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
470{ .PseudoOpc: ARM::VST4LNd16Pseudo_UPD, .RealOpc: ARM::VST4LNd16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
471{ .PseudoOpc: ARM::VST4LNd32Pseudo, .RealOpc: ARM::VST4LNd32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
472{ .PseudoOpc: ARM::VST4LNd32Pseudo_UPD, .RealOpc: ARM::VST4LNd32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
473{ .PseudoOpc: ARM::VST4LNd8Pseudo, .RealOpc: ARM::VST4LNd8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
474{ .PseudoOpc: ARM::VST4LNd8Pseudo_UPD, .RealOpc: ARM::VST4LNd8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
475{ .PseudoOpc: ARM::VST4LNq16Pseudo, .RealOpc: ARM::VST4LNq16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 4,.copyAllListRegs: true},
476{ .PseudoOpc: ARM::VST4LNq16Pseudo_UPD, .RealOpc: ARM::VST4LNq16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 4,.copyAllListRegs: true},
477{ .PseudoOpc: ARM::VST4LNq32Pseudo, .RealOpc: ARM::VST4LNq32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 2,.copyAllListRegs: true},
478{ .PseudoOpc: ARM::VST4LNq32Pseudo_UPD, .RealOpc: ARM::VST4LNq32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 2,.copyAllListRegs: true},
479
480{ .PseudoOpc: ARM::VST4d16Pseudo, .RealOpc: ARM::VST4d16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
481{ .PseudoOpc: ARM::VST4d16Pseudo_UPD, .RealOpc: ARM::VST4d16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
482{ .PseudoOpc: ARM::VST4d32Pseudo, .RealOpc: ARM::VST4d32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
483{ .PseudoOpc: ARM::VST4d32Pseudo_UPD, .RealOpc: ARM::VST4d32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
484{ .PseudoOpc: ARM::VST4d8Pseudo, .RealOpc: ARM::VST4d8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
485{ .PseudoOpc: ARM::VST4d8Pseudo_UPD, .RealOpc: ARM::VST4d8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: SingleSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
486
487{ .PseudoOpc: ARM::VST4q16Pseudo_UPD, .RealOpc: ARM::VST4q16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
488{ .PseudoOpc: ARM::VST4q16oddPseudo, .RealOpc: ARM::VST4q16, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
489{ .PseudoOpc: ARM::VST4q16oddPseudo_UPD, .RealOpc: ARM::VST4q16_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 4 ,.copyAllListRegs: true},
490{ .PseudoOpc: ARM::VST4q32Pseudo_UPD, .RealOpc: ARM::VST4q32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
491{ .PseudoOpc: ARM::VST4q32oddPseudo, .RealOpc: ARM::VST4q32, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
492{ .PseudoOpc: ARM::VST4q32oddPseudo_UPD, .RealOpc: ARM::VST4q32_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 2 ,.copyAllListRegs: true},
493{ .PseudoOpc: ARM::VST4q8Pseudo_UPD, .RealOpc: ARM::VST4q8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: EvenDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
494{ .PseudoOpc: ARM::VST4q8oddPseudo, .RealOpc: ARM::VST4q8, .IsLoad: false, .isUpdating: false, .hasWritebackOperand: false, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true},
495{ .PseudoOpc: ARM::VST4q8oddPseudo_UPD, .RealOpc: ARM::VST4q8_UPD, .IsLoad: false, .isUpdating: true, .hasWritebackOperand: true, .RegSpacing: OddDblSpc, .NumRegs: 4, .RegElts: 8 ,.copyAllListRegs: true}
496};
497
498/// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON
499/// load or store pseudo instruction.
500static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {
501#ifndef NDEBUG
502 // Make sure the table is sorted.
503 static std::atomic<bool> TableChecked(false);
504 if (!TableChecked.load(std::memory_order_relaxed)) {
505 assert(llvm::is_sorted(NEONLdStTable) && "NEONLdStTable is not sorted!");
506 TableChecked.store(true, std::memory_order_relaxed);
507 }
508#endif
509
510 auto I = llvm::lower_bound(Range: NEONLdStTable, Value&: Opcode);
511 if (I != std::end(arr: NEONLdStTable) && I->PseudoOpc == Opcode)
512 return I;
513 return nullptr;
514}
515
516/// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,
517/// corresponding to the specified register spacing. Not all of the results
518/// are necessarily valid, e.g., a Q register only has 2 D subregisters.
519static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,
520 const TargetRegisterInfo *TRI, MCRegister &D0,
521 MCRegister &D1, MCRegister &D2, MCRegister &D3) {
522 if (RegSpc == SingleSpc || RegSpc == SingleLowSpc) {
523 D0 = TRI->getSubReg(Reg, Idx: ARM::dsub_0);
524 D1 = TRI->getSubReg(Reg, Idx: ARM::dsub_1);
525 D2 = TRI->getSubReg(Reg, Idx: ARM::dsub_2);
526 D3 = TRI->getSubReg(Reg, Idx: ARM::dsub_3);
527 } else if (RegSpc == SingleHighQSpc) {
528 D0 = TRI->getSubReg(Reg, Idx: ARM::dsub_4);
529 D1 = TRI->getSubReg(Reg, Idx: ARM::dsub_5);
530 D2 = TRI->getSubReg(Reg, Idx: ARM::dsub_6);
531 D3 = TRI->getSubReg(Reg, Idx: ARM::dsub_7);
532 } else if (RegSpc == SingleHighTSpc) {
533 D0 = TRI->getSubReg(Reg, Idx: ARM::dsub_3);
534 D1 = TRI->getSubReg(Reg, Idx: ARM::dsub_4);
535 D2 = TRI->getSubReg(Reg, Idx: ARM::dsub_5);
536 D3 = TRI->getSubReg(Reg, Idx: ARM::dsub_6);
537 } else if (RegSpc == EvenDblSpc) {
538 D0 = TRI->getSubReg(Reg, Idx: ARM::dsub_0);
539 D1 = TRI->getSubReg(Reg, Idx: ARM::dsub_2);
540 D2 = TRI->getSubReg(Reg, Idx: ARM::dsub_4);
541 D3 = TRI->getSubReg(Reg, Idx: ARM::dsub_6);
542 } else {
543 assert(RegSpc == OddDblSpc && "unknown register spacing");
544 D0 = TRI->getSubReg(Reg, Idx: ARM::dsub_1);
545 D1 = TRI->getSubReg(Reg, Idx: ARM::dsub_3);
546 D2 = TRI->getSubReg(Reg, Idx: ARM::dsub_5);
547 D3 = TRI->getSubReg(Reg, Idx: ARM::dsub_7);
548 }
549}
550
551/// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register
552/// operands to real VLD instructions with D register operands.
553void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
554 MachineInstr &MI = *MBBI;
555 MachineBasicBlock &MBB = *MI.getParent();
556 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
557
558 const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(Opcode: MI.getOpcode());
559 assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
560 NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
561 unsigned NumRegs = TableEntry->NumRegs;
562
563 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
564 MCID: TII->get(Opcode: TableEntry->RealOpc));
565 unsigned OpIdx = 0;
566
567 bool DstIsDead = MI.getOperand(i: OpIdx).isDead();
568 Register DstReg = MI.getOperand(i: OpIdx++).getReg();
569
570 bool IsVLD2DUP = TableEntry->RealOpc == ARM::VLD2DUPd8x2 ||
571 TableEntry->RealOpc == ARM::VLD2DUPd16x2 ||
572 TableEntry->RealOpc == ARM::VLD2DUPd32x2 ||
573 TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||
574 TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||
575 TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed ||
576 TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_register ||
577 TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_register ||
578 TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_register;
579
580 if (IsVLD2DUP) {
581 unsigned SubRegIndex;
582 if (RegSpc == EvenDblSpc) {
583 SubRegIndex = ARM::dsub_0;
584 } else {
585 assert(RegSpc == OddDblSpc && "Unexpected spacing!");
586 SubRegIndex = ARM::dsub_1;
587 }
588 Register SubReg = TRI->getSubReg(Reg: DstReg, Idx: SubRegIndex);
589 MCRegister DstRegPair =
590 TRI->getMatchingSuperReg(Reg: SubReg, SubIdx: ARM::dsub_0, RC: &ARM::DPairSpcRegClass);
591 MIB.addReg(RegNo: DstRegPair, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
592 } else {
593 MCRegister D0, D1, D2, D3;
594 GetDSubRegs(Reg: DstReg, RegSpc, TRI, D0, D1, D2, D3);
595 MIB.addReg(RegNo: D0, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
596 if (NumRegs > 1 && TableEntry->copyAllListRegs)
597 MIB.addReg(RegNo: D1, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
598 if (NumRegs > 2 && TableEntry->copyAllListRegs)
599 MIB.addReg(RegNo: D2, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
600 if (NumRegs > 3 && TableEntry->copyAllListRegs)
601 MIB.addReg(RegNo: D3, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
602 }
603
604 if (TableEntry->isUpdating)
605 MIB.add(MO: MI.getOperand(i: OpIdx++));
606
607 // Copy the addrmode6 operands.
608 MIB.add(MO: MI.getOperand(i: OpIdx++));
609 MIB.add(MO: MI.getOperand(i: OpIdx++));
610
611 // Copy the am6offset operand.
612 if (TableEntry->hasWritebackOperand) {
613 // TODO: The writing-back pseudo instructions we translate here are all
614 // defined to take am6offset nodes that are capable to represent both fixed
615 // and register forms. Some real instructions, however, do not rely on
616 // am6offset and have separate definitions for such forms. When this is the
617 // case, fixed forms do not take any offset nodes, so here we skip them for
618 // such instructions. Once all real and pseudo writing-back instructions are
619 // rewritten without use of am6offset nodes, this code will go away.
620 const MachineOperand &AM6Offset = MI.getOperand(i: OpIdx++);
621 if (TableEntry->RealOpc == ARM::VLD1d8Qwb_fixed ||
622 TableEntry->RealOpc == ARM::VLD1d16Qwb_fixed ||
623 TableEntry->RealOpc == ARM::VLD1d32Qwb_fixed ||
624 TableEntry->RealOpc == ARM::VLD1d64Qwb_fixed ||
625 TableEntry->RealOpc == ARM::VLD1d8Twb_fixed ||
626 TableEntry->RealOpc == ARM::VLD1d16Twb_fixed ||
627 TableEntry->RealOpc == ARM::VLD1d32Twb_fixed ||
628 TableEntry->RealOpc == ARM::VLD1d64Twb_fixed ||
629 TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||
630 TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||
631 TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed) {
632 assert(AM6Offset.getReg() == 0 &&
633 "A fixed writing-back pseudo instruction provides an offset "
634 "register!");
635 } else {
636 MIB.add(MO: AM6Offset);
637 }
638 }
639
640 // For an instruction writing double-spaced subregs, the pseudo instruction
641 // has an extra operand that is a use of the super-register. Record the
642 // operand index and skip over it.
643 unsigned SrcOpIdx = 0;
644 if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc || RegSpc == SingleLowSpc ||
645 RegSpc == SingleHighQSpc || RegSpc == SingleHighTSpc)
646 SrcOpIdx = OpIdx++;
647
648 // Copy the predicate operands.
649 MIB.add(MO: MI.getOperand(i: OpIdx++));
650 MIB.add(MO: MI.getOperand(i: OpIdx++));
651
652 // Copy the super-register source operand used for double-spaced subregs over
653 // to the new instruction as an implicit operand.
654 if (SrcOpIdx != 0) {
655 MachineOperand MO = MI.getOperand(i: SrcOpIdx);
656 MO.setImplicit(true);
657 MIB.add(MO);
658 }
659 // Add an implicit def for the super-register.
660 MIB.addReg(RegNo: DstReg, Flags: RegState::ImplicitDefine | getDeadRegState(B: DstIsDead));
661 MIB.copyImplicitOps(OtherMI: MI);
662
663 // Transfer memoperands.
664 MIB.cloneMemRefs(OtherMI: MI);
665 MI.eraseFromParent();
666 LLVM_DEBUG(dbgs() << "To: "; MIB.getInstr()->dump(););
667}
668
669/// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register
670/// operands to real VST instructions with D register operands.
671void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
672 MachineInstr &MI = *MBBI;
673 MachineBasicBlock &MBB = *MI.getParent();
674 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
675
676 const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(Opcode: MI.getOpcode());
677 assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
678 NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
679 unsigned NumRegs = TableEntry->NumRegs;
680
681 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
682 MCID: TII->get(Opcode: TableEntry->RealOpc));
683 unsigned OpIdx = 0;
684 if (TableEntry->isUpdating)
685 MIB.add(MO: MI.getOperand(i: OpIdx++));
686
687 // Copy the addrmode6 operands.
688 MIB.add(MO: MI.getOperand(i: OpIdx++));
689 MIB.add(MO: MI.getOperand(i: OpIdx++));
690
691 if (TableEntry->hasWritebackOperand) {
692 // TODO: The writing-back pseudo instructions we translate here are all
693 // defined to take am6offset nodes that are capable to represent both fixed
694 // and register forms. Some real instructions, however, do not rely on
695 // am6offset and have separate definitions for such forms. When this is the
696 // case, fixed forms do not take any offset nodes, so here we skip them for
697 // such instructions. Once all real and pseudo writing-back instructions are
698 // rewritten without use of am6offset nodes, this code will go away.
699 const MachineOperand &AM6Offset = MI.getOperand(i: OpIdx++);
700 if (TableEntry->RealOpc == ARM::VST1d8Qwb_fixed ||
701 TableEntry->RealOpc == ARM::VST1d16Qwb_fixed ||
702 TableEntry->RealOpc == ARM::VST1d32Qwb_fixed ||
703 TableEntry->RealOpc == ARM::VST1d64Qwb_fixed ||
704 TableEntry->RealOpc == ARM::VST1d8Twb_fixed ||
705 TableEntry->RealOpc == ARM::VST1d16Twb_fixed ||
706 TableEntry->RealOpc == ARM::VST1d32Twb_fixed ||
707 TableEntry->RealOpc == ARM::VST1d64Twb_fixed) {
708 assert(AM6Offset.getReg() == 0 &&
709 "A fixed writing-back pseudo instruction provides an offset "
710 "register!");
711 } else {
712 MIB.add(MO: AM6Offset);
713 }
714 }
715
716 bool SrcIsKill = MI.getOperand(i: OpIdx).isKill();
717 bool SrcIsUndef = MI.getOperand(i: OpIdx).isUndef();
718 Register SrcReg = MI.getOperand(i: OpIdx++).getReg();
719 MCRegister D0, D1, D2, D3;
720 GetDSubRegs(Reg: SrcReg, RegSpc, TRI, D0, D1, D2, D3);
721 MIB.addReg(RegNo: D0, Flags: getUndefRegState(B: SrcIsUndef));
722 if (NumRegs > 1 && TableEntry->copyAllListRegs)
723 MIB.addReg(RegNo: D1, Flags: getUndefRegState(B: SrcIsUndef));
724 if (NumRegs > 2 && TableEntry->copyAllListRegs)
725 MIB.addReg(RegNo: D2, Flags: getUndefRegState(B: SrcIsUndef));
726 if (NumRegs > 3 && TableEntry->copyAllListRegs)
727 MIB.addReg(RegNo: D3, Flags: getUndefRegState(B: SrcIsUndef));
728
729 // Copy the predicate operands.
730 MIB.add(MO: MI.getOperand(i: OpIdx++));
731 MIB.add(MO: MI.getOperand(i: OpIdx++));
732
733 if (SrcIsKill && !SrcIsUndef) // Add an implicit kill for the super-reg.
734 MIB->addRegisterKilled(IncomingReg: SrcReg, RegInfo: TRI, AddIfNotFound: true);
735 else if (!SrcIsUndef)
736 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit); // Add implicit uses for src reg.
737 MIB.copyImplicitOps(OtherMI: MI);
738
739 // Transfer memoperands.
740 MIB.cloneMemRefs(OtherMI: MI);
741 MI.eraseFromParent();
742 LLVM_DEBUG(dbgs() << "To: "; MIB.getInstr()->dump(););
743}
744
745/// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
746/// register operands to real instructions with D register operands.
747void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
748 MachineInstr &MI = *MBBI;
749 MachineBasicBlock &MBB = *MI.getParent();
750 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
751
752 const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(Opcode: MI.getOpcode());
753 assert(TableEntry && "NEONLdStTable lookup failed");
754 NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
755 unsigned NumRegs = TableEntry->NumRegs;
756 unsigned RegElts = TableEntry->RegElts;
757
758 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
759 MCID: TII->get(Opcode: TableEntry->RealOpc));
760 unsigned OpIdx = 0;
761 // The lane operand is always the 3rd from last operand, before the 2
762 // predicate operands.
763 unsigned Lane = MI.getOperand(i: MI.getDesc().getNumOperands() - 3).getImm();
764
765 // Adjust the lane and spacing as needed for Q registers.
766 assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");
767 if (RegSpc == EvenDblSpc && Lane >= RegElts) {
768 RegSpc = OddDblSpc;
769 Lane -= RegElts;
770 }
771 assert(Lane < RegElts && "out of range lane for VLD/VST-lane");
772
773 MCRegister D0, D1, D2, D3;
774 unsigned DstReg = 0;
775 bool DstIsDead = false;
776 if (TableEntry->IsLoad) {
777 DstIsDead = MI.getOperand(i: OpIdx).isDead();
778 DstReg = MI.getOperand(i: OpIdx++).getReg();
779 GetDSubRegs(Reg: DstReg, RegSpc, TRI, D0, D1, D2, D3);
780 MIB.addReg(RegNo: D0, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
781 if (NumRegs > 1)
782 MIB.addReg(RegNo: D1, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
783 if (NumRegs > 2)
784 MIB.addReg(RegNo: D2, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
785 if (NumRegs > 3)
786 MIB.addReg(RegNo: D3, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
787 }
788
789 if (TableEntry->isUpdating)
790 MIB.add(MO: MI.getOperand(i: OpIdx++));
791
792 // Copy the addrmode6 operands.
793 MIB.add(MO: MI.getOperand(i: OpIdx++));
794 MIB.add(MO: MI.getOperand(i: OpIdx++));
795 // Copy the am6offset operand.
796 if (TableEntry->hasWritebackOperand)
797 MIB.add(MO: MI.getOperand(i: OpIdx++));
798
799 // Grab the super-register source.
800 MachineOperand MO = MI.getOperand(i: OpIdx++);
801 if (!TableEntry->IsLoad)
802 GetDSubRegs(Reg: MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);
803
804 // Add the subregs as sources of the new instruction.
805 RegState SrcFlags =
806 (getUndefRegState(B: MO.isUndef()) | getKillRegState(B: MO.isKill()));
807 MIB.addReg(RegNo: D0, Flags: SrcFlags);
808 if (NumRegs > 1)
809 MIB.addReg(RegNo: D1, Flags: SrcFlags);
810 if (NumRegs > 2)
811 MIB.addReg(RegNo: D2, Flags: SrcFlags);
812 if (NumRegs > 3)
813 MIB.addReg(RegNo: D3, Flags: SrcFlags);
814
815 // Add the lane number operand.
816 MIB.addImm(Val: Lane);
817 OpIdx += 1;
818
819 // Copy the predicate operands.
820 MIB.add(MO: MI.getOperand(i: OpIdx++));
821 MIB.add(MO: MI.getOperand(i: OpIdx++));
822
823 // Copy the super-register source to be an implicit source.
824 MO.setImplicit(true);
825 MIB.add(MO);
826 if (TableEntry->IsLoad)
827 // Add an implicit def for the super-register.
828 MIB.addReg(RegNo: DstReg, Flags: RegState::ImplicitDefine | getDeadRegState(B: DstIsDead));
829 MIB.copyImplicitOps(OtherMI: MI);
830 // Transfer memoperands.
831 MIB.cloneMemRefs(OtherMI: MI);
832 MI.eraseFromParent();
833}
834
835/// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ
836/// register operands to real instructions with D register operands.
837void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,
838 unsigned Opc, bool IsExt) {
839 MachineInstr &MI = *MBBI;
840 MachineBasicBlock &MBB = *MI.getParent();
841 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
842
843 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc));
844 unsigned OpIdx = 0;
845
846 // Transfer the destination register operand.
847 MIB.add(MO: MI.getOperand(i: OpIdx++));
848 if (IsExt) {
849 MachineOperand VdSrc(MI.getOperand(i: OpIdx++));
850 MIB.add(MO: VdSrc);
851 }
852
853 bool SrcIsKill = MI.getOperand(i: OpIdx).isKill();
854 Register SrcReg = MI.getOperand(i: OpIdx++).getReg();
855 MCRegister D0, D1, D2, D3;
856 GetDSubRegs(Reg: SrcReg, RegSpc: SingleSpc, TRI, D0, D1, D2, D3);
857 MIB.addReg(RegNo: D0);
858
859 // Copy the other source register operand.
860 MachineOperand VmSrc(MI.getOperand(i: OpIdx++));
861 MIB.add(MO: VmSrc);
862
863 // Copy the predicate operands.
864 MIB.add(MO: MI.getOperand(i: OpIdx++));
865 MIB.add(MO: MI.getOperand(i: OpIdx++));
866
867 // Add an implicit kill and use for the super-reg.
868 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: SrcIsKill));
869 MIB.copyImplicitOps(OtherMI: MI);
870 MI.eraseFromParent();
871 LLVM_DEBUG(dbgs() << "To: "; MIB.getInstr()->dump(););
872}
873
874void ARMExpandPseudo::ExpandMQQPRLoadStore(MachineBasicBlock::iterator &MBBI) {
875 MachineInstr &MI = *MBBI;
876 MachineBasicBlock &MBB = *MI.getParent();
877 unsigned NewOpc =
878 MI.getOpcode() == ARM::MQQPRStore || MI.getOpcode() == ARM::MQQQQPRStore
879 ? ARM::VSTMDIA
880 : ARM::VLDMDIA;
881 MachineInstrBuilder MIB =
882 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc));
883
884 RegState Flags = getKillRegState(B: MI.getOperand(i: 0).isKill()) |
885 getDefRegState(B: MI.getOperand(i: 0).isDef());
886 Register SrcReg = MI.getOperand(i: 0).getReg();
887
888 // Copy the destination register.
889 MIB.add(MO: MI.getOperand(i: 1));
890 MIB.add(MOs: predOps(Pred: ARMCC::AL));
891 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_0), Flags);
892 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_1), Flags);
893 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_2), Flags);
894 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_3), Flags);
895 if (MI.getOpcode() == ARM::MQQQQPRStore ||
896 MI.getOpcode() == ARM::MQQQQPRLoad) {
897 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_4), Flags);
898 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_5), Flags);
899 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_6), Flags);
900 MIB.addReg(RegNo: TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_7), Flags);
901 }
902
903 if (NewOpc == ARM::VSTMDIA)
904 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
905
906 MIB.copyImplicitOps(OtherMI: MI);
907 MIB.cloneMemRefs(OtherMI: MI);
908 MI.eraseFromParent();
909}
910
911static bool IsAnAddressOperand(const MachineOperand &MO) {
912 // This check is overly conservative. Unless we are certain that the machine
913 // operand is not a symbol reference, we return that it is a symbol reference.
914 // This is important as the load pair may not be split up Windows.
915 switch (MO.getType()) {
916 case MachineOperand::MO_Register:
917 case MachineOperand::MO_Immediate:
918 case MachineOperand::MO_CImmediate:
919 case MachineOperand::MO_FPImmediate:
920 case MachineOperand::MO_ShuffleMask:
921 return false;
922 case MachineOperand::MO_MachineBasicBlock:
923 return true;
924 case MachineOperand::MO_FrameIndex:
925 return false;
926 case MachineOperand::MO_ConstantPoolIndex:
927 case MachineOperand::MO_TargetIndex:
928 case MachineOperand::MO_JumpTableIndex:
929 case MachineOperand::MO_ExternalSymbol:
930 case MachineOperand::MO_GlobalAddress:
931 case MachineOperand::MO_BlockAddress:
932 return true;
933 case MachineOperand::MO_RegisterMask:
934 case MachineOperand::MO_RegisterLiveOut:
935 case MachineOperand::MO_LaneMask:
936 return false;
937 case MachineOperand::MO_Metadata:
938 case MachineOperand::MO_MCSymbol:
939 return true;
940 case MachineOperand::MO_DbgInstrRef:
941 case MachineOperand::MO_CFIIndex:
942 return false;
943 case MachineOperand::MO_IntrinsicID:
944 case MachineOperand::MO_Predicate:
945 llvm_unreachable("should not exist post-isel");
946 }
947 llvm_unreachable("unhandled machine operand type");
948}
949
950static MachineOperand makeImplicit(const MachineOperand &MO) {
951 MachineOperand NewMO = MO;
952 NewMO.setImplicit();
953 return NewMO;
954}
955
956static MachineOperand getMovOperand(const MachineOperand &MO,
957 unsigned TargetFlag) {
958 unsigned TF = MO.getTargetFlags() | TargetFlag;
959 switch (MO.getType()) {
960 case MachineOperand::MO_Immediate: {
961 unsigned Imm = MO.getImm();
962 switch (TargetFlag) {
963 case ARMII::MO_HI_8_15:
964 Imm = (Imm >> 24) & 0xff;
965 break;
966 case ARMII::MO_HI_0_7:
967 Imm = (Imm >> 16) & 0xff;
968 break;
969 case ARMII::MO_LO_8_15:
970 Imm = (Imm >> 8) & 0xff;
971 break;
972 case ARMII::MO_LO_0_7:
973 Imm = Imm & 0xff;
974 break;
975 case ARMII::MO_HI16:
976 Imm = (Imm >> 16) & 0xffff;
977 break;
978 case ARMII::MO_LO16:
979 Imm = Imm & 0xffff;
980 break;
981 default:
982 llvm_unreachable("Only HI/LO target flags are expected");
983 }
984 return MachineOperand::CreateImm(Val: Imm);
985 }
986 case MachineOperand::MO_ExternalSymbol:
987 return MachineOperand::CreateES(SymName: MO.getSymbolName(), TargetFlags: TF);
988 case MachineOperand::MO_JumpTableIndex:
989 return MachineOperand::CreateJTI(Idx: MO.getIndex(), TargetFlags: TF);
990 default:
991 return MachineOperand::CreateGA(GV: MO.getGlobal(), Offset: MO.getOffset(), TargetFlags: TF);
992 }
993}
994
995void ARMExpandPseudo::ExpandTMOV32BitImm(MachineBasicBlock &MBB,
996 MachineBasicBlock::iterator &MBBI) {
997 MachineInstr &MI = *MBBI;
998 Register DstReg = MI.getOperand(i: 0).getReg();
999 bool DstIsDead = MI.getOperand(i: 0).isDead();
1000 const MachineOperand &MO = MI.getOperand(i: 1);
1001 unsigned MIFlags = MI.getFlags();
1002
1003 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
1004
1005 // Expand the mov into a sequence of mov/add+lsl of the individual bytes. We
1006 // want to avoid emitting any zero bytes, as they won't change the result, and
1007 // also don't want any pointless shifts, so instead of immediately emitting
1008 // the shift for a byte we keep track of how much we will need to shift and do
1009 // it before the next nonzero byte.
1010 unsigned PendingShift = 0;
1011 for (unsigned Byte = 0; Byte < 4; ++Byte) {
1012 unsigned Flag = Byte == 0 ? ARMII::MO_HI_8_15
1013 : Byte == 1 ? ARMII::MO_HI_0_7
1014 : Byte == 2 ? ARMII::MO_LO_8_15
1015 : ARMII::MO_LO_0_7;
1016 MachineOperand Operand = getMovOperand(MO, TargetFlag: Flag);
1017 bool ZeroImm = Operand.isImm() && Operand.getImm() == 0;
1018 unsigned Op = PendingShift ? ARM::tADDi8 : ARM::tMOVi8;
1019
1020 // Emit the pending shift if we're going to emit this byte or if we've
1021 // reached the end.
1022 if (PendingShift && (!ZeroImm || Byte == 3)) {
1023 MachineInstr *Lsl =
1024 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::tLSLri), DestReg: DstReg)
1025 .add(MO: t1CondCodeOp(isDead: true))
1026 .addReg(RegNo: DstReg)
1027 .addImm(Val: PendingShift)
1028 .add(MOs: predOps(Pred: ARMCC::AL))
1029 .setMIFlags(MIFlags);
1030 (void)Lsl;
1031 LLVM_DEBUG(dbgs() << "And: "; Lsl->dump(););
1032 PendingShift = 0;
1033 }
1034
1035 // Emit this byte if it's nonzero.
1036 if (!ZeroImm) {
1037 MachineInstrBuilder MIB =
1038 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Op), DestReg: DstReg)
1039 .add(MO: t1CondCodeOp(isDead: true));
1040 if (Op == ARM::tADDi8)
1041 MIB.addReg(RegNo: DstReg);
1042 MIB.add(MO: Operand);
1043 MIB.add(MOs: predOps(Pred: ARMCC::AL));
1044 MIB.setMIFlags(MIFlags);
1045 LLVM_DEBUG(dbgs() << (Op == ARM::tMOVi8 ? "To: " : "And:") << " ";
1046 MIB.getInstr()->dump(););
1047 }
1048
1049 // Don't accumulate the shift value if we've not yet seen a nonzero byte.
1050 if (PendingShift || !ZeroImm)
1051 PendingShift += 8;
1052 }
1053
1054 // The dest is dead on the last instruction we emitted if it was dead on the
1055 // original instruction.
1056 (--MBBI)->getOperand(i: 0).setIsDead(DstIsDead);
1057
1058 MI.eraseFromParent();
1059}
1060
1061void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
1062 MachineBasicBlock::iterator &MBBI) {
1063 MachineInstr &MI = *MBBI;
1064 unsigned Opcode = MI.getOpcode();
1065 Register PredReg;
1066 ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
1067 Register DstReg = MI.getOperand(i: 0).getReg();
1068 bool DstIsDead = MI.getOperand(i: 0).isDead();
1069 bool isCC = Opcode == ARM::MOVCCi32imm || Opcode == ARM::t2MOVCCi32imm;
1070 const MachineOperand &MO = MI.getOperand(i: isCC ? 2 : 1);
1071 bool RequiresBundling = STI->isTargetWindows() && IsAnAddressOperand(MO);
1072 MachineInstrBuilder LO16, HI16;
1073 LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
1074
1075 if (!STI->hasV6T2Ops() &&
1076 (Opcode == ARM::MOVi32imm || Opcode == ARM::MOVCCi32imm)) {
1077 // FIXME Windows CE supports older ARM CPUs
1078 assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
1079
1080 assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
1081 unsigned ImmVal = (unsigned)MO.getImm();
1082 unsigned SOImmValV1 = 0, SOImmValV2 = 0;
1083
1084 if (ARM_AM::isSOImmTwoPartVal(V: ImmVal)) { // Expand into a movi + orr.
1085 LO16 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::MOVi), DestReg: DstReg);
1086 HI16 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::ORRri))
1087 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead))
1088 .addReg(RegNo: DstReg);
1089 SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(V: ImmVal);
1090 SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(V: ImmVal);
1091 } else { // Expand into a mvn + sub.
1092 LO16 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::MVNi), DestReg: DstReg);
1093 HI16 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::SUBri))
1094 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead))
1095 .addReg(RegNo: DstReg);
1096 SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(V: -ImmVal);
1097 SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(V: -ImmVal);
1098 SOImmValV1 = ~(-SOImmValV1);
1099 }
1100
1101 unsigned MIFlags = MI.getFlags();
1102 LO16 = LO16.addImm(Val: SOImmValV1);
1103 HI16 = HI16.addImm(Val: SOImmValV2);
1104 LO16.cloneMemRefs(OtherMI: MI);
1105 HI16.cloneMemRefs(OtherMI: MI);
1106 LO16.setMIFlags(MIFlags);
1107 HI16.setMIFlags(MIFlags);
1108 LO16.addImm(Val: Pred).addReg(RegNo: PredReg).add(MO: condCodeOp());
1109 HI16.addImm(Val: Pred).addReg(RegNo: PredReg).add(MO: condCodeOp());
1110 if (isCC)
1111 LO16.add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
1112 LO16.copyImplicitOps(OtherMI: MI);
1113 HI16.copyImplicitOps(OtherMI: MI);
1114 MI.eraseFromParent();
1115 return;
1116 }
1117
1118 unsigned LO16Opc = 0;
1119 unsigned HI16Opc = 0;
1120 unsigned MIFlags = MI.getFlags();
1121 if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {
1122 LO16Opc = ARM::t2MOVi16;
1123 HI16Opc = ARM::t2MOVTi16;
1124 } else {
1125 LO16Opc = ARM::MOVi16;
1126 HI16Opc = ARM::MOVTi16;
1127 }
1128
1129 LO16 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LO16Opc), DestReg: DstReg);
1130 LO16.setMIFlags(MIFlags);
1131 LO16.add(MO: getMovOperand(MO, TargetFlag: ARMII::MO_LO16));
1132 LO16.cloneMemRefs(OtherMI: MI);
1133 LO16.addImm(Val: Pred).addReg(RegNo: PredReg);
1134 if (isCC)
1135 LO16.add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
1136 LO16.copyImplicitOps(OtherMI: MI);
1137 LLVM_DEBUG(dbgs() << "To: "; LO16.getInstr()->dump(););
1138
1139 MachineOperand HIOperand = getMovOperand(MO, TargetFlag: ARMII::MO_HI16);
1140 if (!(HIOperand.isImm() && HIOperand.getImm() == 0)) {
1141 HI16 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HI16Opc))
1142 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead))
1143 .addReg(RegNo: DstReg);
1144 HI16.setMIFlags(MIFlags);
1145 HI16.add(MO: HIOperand);
1146 HI16.cloneMemRefs(OtherMI: MI);
1147 HI16.addImm(Val: Pred).addReg(RegNo: PredReg);
1148 HI16.copyImplicitOps(OtherMI: MI);
1149 LLVM_DEBUG(dbgs() << "And: "; HI16.getInstr()->dump(););
1150 } else {
1151 LO16->getOperand(i: 0).setIsDead(DstIsDead);
1152 }
1153
1154 if (RequiresBundling)
1155 finalizeBundle(MBB, FirstMI: LO16->getIterator(), LastMI: MBBI->getIterator());
1156
1157 MI.eraseFromParent();
1158}
1159
1160// The size of the area, accessed by that VLSTM/VLLDM
1161// S0-S31 + FPSCR + 8 more bytes (VPR + pad, or just pad)
1162static const int CMSE_FP_SAVE_SIZE = 136;
1163
1164static void determineGPRegsToClear(const MachineInstr &MI,
1165 const std::initializer_list<unsigned> &Regs,
1166 SmallVectorImpl<unsigned> &ClearRegs) {
1167 SmallVector<unsigned, 4> OpRegs;
1168 for (const MachineOperand &Op : MI.operands()) {
1169 if (!Op.isReg() || !Op.isUse())
1170 continue;
1171 OpRegs.push_back(Elt: Op.getReg());
1172 }
1173 llvm::sort(C&: OpRegs);
1174
1175 std::set_difference(first1: Regs.begin(), last1: Regs.end(), first2: OpRegs.begin(), last2: OpRegs.end(),
1176 result: std::back_inserter(x&: ClearRegs));
1177}
1178
1179void ARMExpandPseudo::CMSEClearGPRegs(
1180 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1181 const DebugLoc &DL, const SmallVectorImpl<unsigned> &ClearRegs,
1182 unsigned ClobberReg) {
1183
1184 if (STI->hasV8_1MMainlineOps()) {
1185 // Clear the registers using the CLRM instruction.
1186 MachineInstrBuilder CLRM =
1187 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2CLRM)).add(MOs: predOps(Pred: ARMCC::AL));
1188 for (unsigned R : ClearRegs)
1189 CLRM.addReg(RegNo: R, Flags: RegState::Define);
1190 CLRM.addReg(RegNo: ARM::APSR, Flags: RegState::Define);
1191 CLRM.addReg(RegNo: ARM::CPSR, Flags: RegState::Define | RegState::Implicit);
1192 } else {
1193 // Clear the registers and flags by copying ClobberReg into them.
1194 // (Baseline can't do a high register clear in one instruction).
1195 for (unsigned Reg : ClearRegs) {
1196 if (Reg == ClobberReg)
1197 continue;
1198 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tMOVr), DestReg: Reg)
1199 .addReg(RegNo: ClobberReg)
1200 .add(MOs: predOps(Pred: ARMCC::AL));
1201 }
1202
1203 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2MSR_M))
1204 .addImm(Val: STI->hasDSP() ? 0xc00 : 0x800)
1205 .addReg(RegNo: ClobberReg)
1206 .add(MOs: predOps(Pred: ARMCC::AL));
1207 }
1208}
1209
1210// Find which FP registers need to be cleared. The parameter `ClearRegs` is
1211// initialised with all elements set to true, and this function resets all the
1212// bits, which correspond to register uses. Returns true if any floating point
1213// register is defined, false otherwise.
1214static bool determineFPRegsToClear(const MachineInstr &MI,
1215 BitVector &ClearRegs) {
1216 bool DefFP = false;
1217 for (const MachineOperand &Op : MI.operands()) {
1218 if (!Op.isReg())
1219 continue;
1220
1221 Register Reg = Op.getReg();
1222 if (Op.isDef()) {
1223 if ((Reg >= ARM::Q0 && Reg <= ARM::Q7) ||
1224 (Reg >= ARM::D0 && Reg <= ARM::D15) ||
1225 (Reg >= ARM::S0 && Reg <= ARM::S31))
1226 DefFP = true;
1227 continue;
1228 }
1229
1230 if (Reg >= ARM::Q0 && Reg <= ARM::Q7) {
1231 int R = Reg - ARM::Q0;
1232 ClearRegs.reset(I: R * 4, E: (R + 1) * 4);
1233 } else if (Reg >= ARM::D0 && Reg <= ARM::D15) {
1234 int R = Reg - ARM::D0;
1235 ClearRegs.reset(I: R * 2, E: (R + 1) * 2);
1236 } else if (Reg >= ARM::S0 && Reg <= ARM::S31) {
1237 ClearRegs[Reg - ARM::S0] = false;
1238 }
1239 }
1240 return DefFP;
1241}
1242
1243MachineBasicBlock &
1244ARMExpandPseudo::CMSEClearFPRegs(MachineBasicBlock &MBB,
1245 MachineBasicBlock::iterator MBBI) {
1246 BitVector ClearRegs(16, true);
1247 (void)determineFPRegsToClear(MI: *MBBI, ClearRegs);
1248
1249 if (STI->hasV8_1MMainlineOps())
1250 return CMSEClearFPRegsV81(MBB, MBBI, ClearRegs);
1251 else
1252 return CMSEClearFPRegsV8(MBB, MBBI, ClearRegs);
1253}
1254
1255// Clear the FP registers for v8.0-M, by copying over the content
1256// of LR. Uses R12 as a scratch register.
1257MachineBasicBlock &
1258ARMExpandPseudo::CMSEClearFPRegsV8(MachineBasicBlock &MBB,
1259 MachineBasicBlock::iterator MBBI,
1260 const BitVector &ClearRegs) {
1261 if (!STI->hasFPRegs())
1262 return MBB;
1263
1264 auto &RetI = *MBBI;
1265 const DebugLoc &DL = RetI.getDebugLoc();
1266
1267 // If optimising for minimum size, clear FP registers unconditionally.
1268 // Otherwise, check the CONTROL.SFPA (Secure Floating-Point Active) bit and
1269 // don't clear them if they belong to the non-secure state.
1270 MachineBasicBlock *ClearBB, *DoneBB;
1271 if (STI->hasMinSize()) {
1272 ClearBB = DoneBB = &MBB;
1273 } else {
1274 MachineFunction *MF = MBB.getParent();
1275 ClearBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1276 DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1277
1278 MF->insert(MBBI: ++MBB.getIterator(), MBB: ClearBB);
1279 MF->insert(MBBI: ++ClearBB->getIterator(), MBB: DoneBB);
1280
1281 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MBBI, To: MBB.end());
1282 DoneBB->transferSuccessors(FromMBB: &MBB);
1283 MBB.addSuccessor(Succ: ClearBB);
1284 MBB.addSuccessor(Succ: DoneBB);
1285 ClearBB->addSuccessor(Succ: DoneBB);
1286
1287 // At the new basic blocks we need to have live-in the registers, used
1288 // for the return value as well as LR, used to clear registers.
1289 for (const MachineOperand &Op : RetI.operands()) {
1290 if (!Op.isReg())
1291 continue;
1292 Register Reg = Op.getReg();
1293 if (Reg == ARM::NoRegister || Reg == ARM::LR)
1294 continue;
1295 assert(Reg.isPhysical() && "Unallocated register");
1296 ClearBB->addLiveIn(PhysReg: Reg);
1297 DoneBB->addLiveIn(PhysReg: Reg);
1298 }
1299 ClearBB->addLiveIn(PhysReg: ARM::LR);
1300 DoneBB->addLiveIn(PhysReg: ARM::LR);
1301
1302 // Read the CONTROL register.
1303 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: TII->get(Opcode: ARM::t2MRS_M), DestReg: ARM::R12)
1304 .addImm(Val: 20)
1305 .add(MOs: predOps(Pred: ARMCC::AL));
1306 // Check bit 3 (SFPA).
1307 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: TII->get(Opcode: ARM::t2TSTri))
1308 .addReg(RegNo: ARM::R12)
1309 .addImm(Val: 8)
1310 .add(MOs: predOps(Pred: ARMCC::AL));
1311 // If SFPA is clear, jump over ClearBB to DoneBB.
1312 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: TII->get(Opcode: ARM::tBcc))
1313 .addMBB(MBB: DoneBB)
1314 .addImm(Val: ARMCC::EQ)
1315 .addReg(RegNo: ARM::CPSR, Flags: RegState::Kill);
1316 }
1317
1318 // Emit the clearing sequence
1319 for (unsigned D = 0; D < 8; D++) {
1320 // Attempt to clear as double
1321 if (ClearRegs[D * 2 + 0] && ClearRegs[D * 2 + 1]) {
1322 unsigned Reg = ARM::D0 + D;
1323 BuildMI(BB: ClearBB, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVDRR), DestReg: Reg)
1324 .addReg(RegNo: ARM::LR)
1325 .addReg(RegNo: ARM::LR)
1326 .add(MOs: predOps(Pred: ARMCC::AL));
1327 } else {
1328 // Clear first part as single
1329 if (ClearRegs[D * 2 + 0]) {
1330 unsigned Reg = ARM::S0 + D * 2;
1331 BuildMI(BB: ClearBB, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVSR), DestReg: Reg)
1332 .addReg(RegNo: ARM::LR)
1333 .add(MOs: predOps(Pred: ARMCC::AL));
1334 }
1335 // Clear second part as single
1336 if (ClearRegs[D * 2 + 1]) {
1337 unsigned Reg = ARM::S0 + D * 2 + 1;
1338 BuildMI(BB: ClearBB, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVSR), DestReg: Reg)
1339 .addReg(RegNo: ARM::LR)
1340 .add(MOs: predOps(Pred: ARMCC::AL));
1341 }
1342 }
1343 }
1344
1345 // Clear FPSCR bits 0-4, 7, 28-31
1346 // The other bits are program global according to the AAPCS
1347 BuildMI(BB: ClearBB, MIMD: DL, MCID: TII->get(Opcode: ARM::VMRS), DestReg: ARM::R12)
1348 .add(MOs: predOps(Pred: ARMCC::AL));
1349 BuildMI(BB: ClearBB, MIMD: DL, MCID: TII->get(Opcode: ARM::t2BICri), DestReg: ARM::R12)
1350 .addReg(RegNo: ARM::R12)
1351 .addImm(Val: 0x0000009F)
1352 .add(MOs: predOps(Pred: ARMCC::AL))
1353 .add(MO: condCodeOp());
1354 BuildMI(BB: ClearBB, MIMD: DL, MCID: TII->get(Opcode: ARM::t2BICri), DestReg: ARM::R12)
1355 .addReg(RegNo: ARM::R12)
1356 .addImm(Val: 0xF0000000)
1357 .add(MOs: predOps(Pred: ARMCC::AL))
1358 .add(MO: condCodeOp());
1359 BuildMI(BB: ClearBB, MIMD: DL, MCID: TII->get(Opcode: ARM::VMSR))
1360 .addReg(RegNo: ARM::R12)
1361 .add(MOs: predOps(Pred: ARMCC::AL));
1362
1363 return *DoneBB;
1364}
1365
1366MachineBasicBlock &
1367ARMExpandPseudo::CMSEClearFPRegsV81(MachineBasicBlock &MBB,
1368 MachineBasicBlock::iterator MBBI,
1369 const BitVector &ClearRegs) {
1370 auto &RetI = *MBBI;
1371
1372 // Emit a sequence of VSCCLRM <sreglist> instructions, one instruction for
1373 // each contiguous sequence of S-registers.
1374 int Start = -1, End = -1;
1375 for (int S = 0, E = ClearRegs.size(); S != E; ++S) {
1376 if (ClearRegs[S] && S == End + 1) {
1377 End = S; // extend range
1378 continue;
1379 }
1380 // Emit current range.
1381 if (Start < End) {
1382 MachineInstrBuilder VSCCLRM =
1383 BuildMI(BB&: MBB, I: MBBI, MIMD: RetI.getDebugLoc(), MCID: TII->get(Opcode: ARM::VSCCLRMS))
1384 .add(MOs: predOps(Pred: ARMCC::AL));
1385 while (++Start <= End)
1386 VSCCLRM.addReg(RegNo: ARM::S0 + Start, Flags: RegState::Define);
1387 VSCCLRM.addReg(RegNo: ARM::VPR, Flags: RegState::Define);
1388 }
1389 Start = End = S;
1390 }
1391 // Emit last range.
1392 if (Start < End) {
1393 MachineInstrBuilder VSCCLRM =
1394 BuildMI(BB&: MBB, I: MBBI, MIMD: RetI.getDebugLoc(), MCID: TII->get(Opcode: ARM::VSCCLRMS))
1395 .add(MOs: predOps(Pred: ARMCC::AL));
1396 while (++Start <= End)
1397 VSCCLRM.addReg(RegNo: ARM::S0 + Start, Flags: RegState::Define);
1398 VSCCLRM.addReg(RegNo: ARM::VPR, Flags: RegState::Define);
1399 }
1400
1401 return MBB;
1402}
1403
1404void ARMExpandPseudo::CMSESaveClearFPRegs(
1405 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1406 const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {
1407 if (STI->hasV8_1MMainlineOps())
1408 CMSESaveClearFPRegsV81(MBB, MBBI, DL, LiveRegs);
1409 else if (STI->hasV8MMainlineOps())
1410 CMSESaveClearFPRegsV8(MBB, MBBI, DL, LiveRegs, ScratchRegs);
1411}
1412
1413// Save and clear FP registers if present
1414void ARMExpandPseudo::CMSESaveClearFPRegsV8(
1415 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1416 const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {
1417
1418 // Store an available register for FPSCR clearing
1419 assert(!ScratchRegs.empty());
1420 unsigned SpareReg = ScratchRegs.front();
1421
1422 // save space on stack for VLSTM
1423 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tSUBspi), DestReg: ARM::SP)
1424 .addReg(RegNo: ARM::SP)
1425 .addImm(Val: CMSE_FP_SAVE_SIZE >> 2)
1426 .add(MOs: predOps(Pred: ARMCC::AL));
1427
1428 // Use ScratchRegs to store the fp regs
1429 std::vector<std::tuple<unsigned, unsigned, unsigned>> ClearedFPRegs;
1430 std::vector<unsigned> NonclearedFPRegs;
1431 bool ReturnsFPReg = false;
1432 for (const MachineOperand &Op : MBBI->operands()) {
1433 if (Op.isReg() && Op.isUse()) {
1434 Register Reg = Op.getReg();
1435 assert(!ARM::DPRRegClass.contains(Reg) ||
1436 ARM::DPR_VFP2RegClass.contains(Reg));
1437 assert(!ARM::QPRRegClass.contains(Reg));
1438 if (ARM::DPR_VFP2RegClass.contains(Reg)) {
1439 if (ScratchRegs.size() >= 2) {
1440 unsigned SaveReg2 = ScratchRegs.pop_back_val();
1441 unsigned SaveReg1 = ScratchRegs.pop_back_val();
1442 ClearedFPRegs.emplace_back(args&: Reg, args&: SaveReg1, args&: SaveReg2);
1443
1444 // Save the fp register to the normal registers
1445 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVRRD))
1446 .addReg(RegNo: SaveReg1, Flags: RegState::Define)
1447 .addReg(RegNo: SaveReg2, Flags: RegState::Define)
1448 .addReg(RegNo: Reg)
1449 .add(MOs: predOps(Pred: ARMCC::AL));
1450 } else {
1451 NonclearedFPRegs.push_back(x: Reg);
1452 }
1453 } else if (ARM::SPRRegClass.contains(Reg)) {
1454 if (ScratchRegs.size() >= 1) {
1455 unsigned SaveReg = ScratchRegs.pop_back_val();
1456 ClearedFPRegs.emplace_back(args&: Reg, args&: SaveReg, args: 0);
1457
1458 // Save the fp register to the normal registers
1459 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVRS), DestReg: SaveReg)
1460 .addReg(RegNo: Reg)
1461 .add(MOs: predOps(Pred: ARMCC::AL));
1462 } else {
1463 NonclearedFPRegs.push_back(x: Reg);
1464 }
1465 }
1466 } else if (Op.isReg() && Op.isDef()) {
1467 Register Reg = Op.getReg();
1468 if (ARM::SPRRegClass.contains(Reg) || ARM::DPRRegClass.contains(Reg) ||
1469 ARM::QPRRegClass.contains(Reg))
1470 ReturnsFPReg = true;
1471 }
1472 }
1473
1474 bool PassesFPReg = (!NonclearedFPRegs.empty() || !ClearedFPRegs.empty());
1475
1476 if (PassesFPReg || ReturnsFPReg)
1477 assert(STI->hasFPRegs() && "Subtarget needs fpregs");
1478
1479 // CVE-2024-7883
1480 //
1481 // The VLLDM/VLSTM instructions set up lazy state preservation, but they
1482 // execute as NOPs if the FP register file is not considered to contain
1483 // secure data, represented by the CONTROL_S.SFPA bit. This means that the
1484 // state of CONTROL_S.SFPA must be the same when these two instructions are
1485 // executed. That might not be the case if we haven't used any FP
1486 // instructions before the VLSTM, so CONTROL_S.SFPA is clear, but do have one
1487 // before the VLLDM, which sets it..
1488 //
1489 // If we can't prove that SFPA will be the same for the VLSTM and VLLDM, we
1490 // execute a "vmov s0, s0" instruction before the VLSTM to ensure that
1491 // CONTROL_S.SFPA is set for both.
1492 //
1493 // That can only happen for callees which take no FP arguments (or we'd have
1494 // inserted a VMOV above) and which return values in FP regs (so that we need
1495 // to use a VMOV to back-up the return value before the VLLDM). It also can't
1496 // happen if the call is dominated by other existing floating-point
1497 // instructions, but we don't currently check for that case.
1498 //
1499 // These conditions mean that we only emit this instruction when using the
1500 // hard-float ABI, which means we can assume that FP instructions are
1501 // available, and don't need to make it conditional like we do for the
1502 // CVE-2021-35465 workaround.
1503 if (ReturnsFPReg && !PassesFPReg) {
1504 bool S0Dead = !LiveRegs.contains(Reg: ARM::S0);
1505 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVS))
1506 .addReg(RegNo: ARM::S0, Flags: RegState::Define | getDeadRegState(B: S0Dead))
1507 .addReg(RegNo: ARM::S0, Flags: getUndefRegState(B: S0Dead))
1508 .add(MOs: predOps(Pred: ARMCC::AL));
1509 }
1510
1511 // Lazy store all fp registers to the stack.
1512 // This executes as NOP in the absence of floating-point support.
1513 MachineInstrBuilder VLSTM =
1514 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLSTM))
1515 .addReg(RegNo: ARM::SP)
1516 .add(MOs: predOps(Pred: ARMCC::AL))
1517 .addImm(Val: 0); // Represents a pseoudo register list, has no effect on
1518 // the encoding.
1519 // Mark non-live registers as undef
1520 for (MachineOperand &MO : VLSTM->implicit_operands()) {
1521 if (MO.isReg() && !MO.isDef()) {
1522 Register Reg = MO.getReg();
1523 MO.setIsUndef(!LiveRegs.contains(Reg));
1524 }
1525 }
1526
1527 // Restore all arguments
1528 for (const auto &Regs : ClearedFPRegs) {
1529 unsigned Reg, SaveReg1, SaveReg2;
1530 std::tie(args&: Reg, args&: SaveReg1, args&: SaveReg2) = Regs;
1531 if (ARM::DPR_VFP2RegClass.contains(Reg))
1532 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVDRR), DestReg: Reg)
1533 .addReg(RegNo: SaveReg1)
1534 .addReg(RegNo: SaveReg2)
1535 .add(MOs: predOps(Pred: ARMCC::AL));
1536 else if (ARM::SPRRegClass.contains(Reg))
1537 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVSR), DestReg: Reg)
1538 .addReg(RegNo: SaveReg1)
1539 .add(MOs: predOps(Pred: ARMCC::AL));
1540 }
1541
1542 for (unsigned Reg : NonclearedFPRegs) {
1543 if (ARM::DPR_VFP2RegClass.contains(Reg)) {
1544 if (STI->isLittle()) {
1545 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLDRD), DestReg: Reg)
1546 .addReg(RegNo: ARM::SP)
1547 .addImm(Val: (Reg - ARM::D0) * 2)
1548 .add(MOs: predOps(Pred: ARMCC::AL));
1549 } else {
1550 // For big-endian targets we need to load the two subregisters of Reg
1551 // manually because VLDRD would load them in wrong order
1552 MCRegister SReg0 = TRI->getSubReg(Reg, Idx: ARM::ssub_0);
1553 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLDRS), DestReg: SReg0)
1554 .addReg(RegNo: ARM::SP)
1555 .addImm(Val: (Reg - ARM::D0) * 2)
1556 .add(MOs: predOps(Pred: ARMCC::AL));
1557 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLDRS), DestReg: SReg0 + 1)
1558 .addReg(RegNo: ARM::SP)
1559 .addImm(Val: (Reg - ARM::D0) * 2 + 1)
1560 .add(MOs: predOps(Pred: ARMCC::AL));
1561 }
1562 } else if (ARM::SPRRegClass.contains(Reg)) {
1563 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLDRS), DestReg: Reg)
1564 .addReg(RegNo: ARM::SP)
1565 .addImm(Val: Reg - ARM::S0)
1566 .add(MOs: predOps(Pred: ARMCC::AL));
1567 }
1568 }
1569 // restore FPSCR from stack and clear bits 0-4, 7, 28-31
1570 // The other bits are program global according to the AAPCS
1571 if (PassesFPReg) {
1572 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tLDRspi), DestReg: SpareReg)
1573 .addReg(RegNo: ARM::SP)
1574 .addImm(Val: 0x10)
1575 .add(MOs: predOps(Pred: ARMCC::AL));
1576 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2BICri), DestReg: SpareReg)
1577 .addReg(RegNo: SpareReg)
1578 .addImm(Val: 0x0000009F)
1579 .add(MOs: predOps(Pred: ARMCC::AL))
1580 .add(MO: condCodeOp());
1581 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2BICri), DestReg: SpareReg)
1582 .addReg(RegNo: SpareReg)
1583 .addImm(Val: 0xF0000000)
1584 .add(MOs: predOps(Pred: ARMCC::AL))
1585 .add(MO: condCodeOp());
1586 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMSR))
1587 .addReg(RegNo: SpareReg)
1588 .add(MOs: predOps(Pred: ARMCC::AL));
1589 // The ldr must happen after a floating point instruction. To prevent the
1590 // post-ra scheduler to mess with the order, we create a bundle.
1591 finalizeBundle(MBB, FirstMI: VLSTM->getIterator(), LastMI: MBBI->getIterator());
1592 }
1593}
1594
1595void ARMExpandPseudo::CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,
1596 MachineBasicBlock::iterator MBBI,
1597 DebugLoc &DL,
1598 const LivePhysRegs &LiveRegs) {
1599 BitVector ClearRegs(32, true);
1600 bool DefFP = determineFPRegsToClear(MI: *MBBI, ClearRegs);
1601
1602 // If the instruction does not write to a FP register and no elements were
1603 // removed from the set, then no FP registers were used to pass
1604 // arguments/returns.
1605 if (!DefFP && ClearRegs.count() == ClearRegs.size()) {
1606 // save space on stack for VLSTM
1607 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tSUBspi), DestReg: ARM::SP)
1608 .addReg(RegNo: ARM::SP)
1609 .addImm(Val: CMSE_FP_SAVE_SIZE >> 2)
1610 .add(MOs: predOps(Pred: ARMCC::AL));
1611
1612 // Lazy store all FP registers to the stack
1613 MachineInstrBuilder VLSTM =
1614 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLSTM))
1615 .addReg(RegNo: ARM::SP)
1616 .add(MOs: predOps(Pred: ARMCC::AL))
1617 .addImm(Val: 0); // Represents a pseoudo register list, has no effect on
1618 // the encoding.
1619 // Mark non-live registers as undef
1620 for (MachineOperand &MO : VLSTM->implicit_operands()) {
1621 if (MO.isReg() && !MO.isDef()) {
1622 Register Reg = MO.getReg();
1623 MO.setIsUndef(!LiveRegs.contains(Reg));
1624 }
1625 }
1626 } else {
1627 // Push all the callee-saved registers (s16-s31).
1628 MachineInstrBuilder VPUSH =
1629 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VSTMSDB_UPD), DestReg: ARM::SP)
1630 .addReg(RegNo: ARM::SP)
1631 .add(MOs: predOps(Pred: ARMCC::AL));
1632 for (unsigned Reg = ARM::S16; Reg <= ARM::S31; ++Reg)
1633 VPUSH.addReg(RegNo: Reg);
1634
1635 // Clear FP registers with a VSCCLRM.
1636 (void)CMSEClearFPRegsV81(MBB, MBBI, ClearRegs);
1637
1638 // Save floating-point context.
1639 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VSTR_FPCXTS_pre), DestReg: ARM::SP)
1640 .addReg(RegNo: ARM::SP)
1641 .addImm(Val: -8)
1642 .add(MOs: predOps(Pred: ARMCC::AL));
1643 }
1644}
1645
1646// Restore FP registers if present
1647void ARMExpandPseudo::CMSERestoreFPRegs(
1648 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1649 SmallVectorImpl<unsigned> &AvailableRegs) {
1650 if (STI->hasV8_1MMainlineOps())
1651 CMSERestoreFPRegsV81(MBB, MBBI, DL, AvailableRegs);
1652 else if (STI->hasV8MMainlineOps())
1653 CMSERestoreFPRegsV8(MBB, MBBI, DL, AvailableRegs);
1654}
1655
1656void ARMExpandPseudo::CMSERestoreFPRegsV8(
1657 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1658 SmallVectorImpl<unsigned> &AvailableRegs) {
1659
1660 // Keep a scratch register for the mitigation sequence.
1661 unsigned ScratchReg = ARM::NoRegister;
1662 if (STI->fixCMSE_CVE_2021_35465())
1663 ScratchReg = AvailableRegs.pop_back_val();
1664
1665 // Use AvailableRegs to store the fp regs
1666 std::vector<std::tuple<unsigned, unsigned, unsigned>> ClearedFPRegs;
1667 std::vector<unsigned> NonclearedFPRegs;
1668 for (const MachineOperand &Op : MBBI->operands()) {
1669 if (Op.isReg() && Op.isDef()) {
1670 Register Reg = Op.getReg();
1671 assert(!ARM::DPRRegClass.contains(Reg) ||
1672 ARM::DPR_VFP2RegClass.contains(Reg));
1673 assert(!ARM::QPRRegClass.contains(Reg));
1674 if (ARM::DPR_VFP2RegClass.contains(Reg)) {
1675 if (AvailableRegs.size() >= 2) {
1676 unsigned SaveReg2 = AvailableRegs.pop_back_val();
1677 unsigned SaveReg1 = AvailableRegs.pop_back_val();
1678 ClearedFPRegs.emplace_back(args&: Reg, args&: SaveReg1, args&: SaveReg2);
1679
1680 // Save the fp register to the normal registers
1681 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVRRD))
1682 .addReg(RegNo: SaveReg1, Flags: RegState::Define)
1683 .addReg(RegNo: SaveReg2, Flags: RegState::Define)
1684 .addReg(RegNo: Reg)
1685 .add(MOs: predOps(Pred: ARMCC::AL));
1686 } else {
1687 NonclearedFPRegs.push_back(x: Reg);
1688 }
1689 } else if (ARM::SPRRegClass.contains(Reg)) {
1690 if (AvailableRegs.size() >= 1) {
1691 unsigned SaveReg = AvailableRegs.pop_back_val();
1692 ClearedFPRegs.emplace_back(args&: Reg, args&: SaveReg, args: 0);
1693
1694 // Save the fp register to the normal registers
1695 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVRS), DestReg: SaveReg)
1696 .addReg(RegNo: Reg)
1697 .add(MOs: predOps(Pred: ARMCC::AL));
1698 } else {
1699 NonclearedFPRegs.push_back(x: Reg);
1700 }
1701 }
1702 }
1703 }
1704
1705 bool returnsFPReg = (!NonclearedFPRegs.empty() || !ClearedFPRegs.empty());
1706
1707 if (returnsFPReg)
1708 assert(STI->hasFPRegs() && "Subtarget needs fpregs");
1709
1710 // Push FP regs that cannot be restored via normal registers on the stack
1711 for (unsigned Reg : NonclearedFPRegs) {
1712 if (ARM::DPR_VFP2RegClass.contains(Reg))
1713 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VSTRD))
1714 .addReg(RegNo: Reg)
1715 .addReg(RegNo: ARM::SP)
1716 .addImm(Val: (Reg - ARM::D0) * 2)
1717 .add(MOs: predOps(Pred: ARMCC::AL));
1718 else if (ARM::SPRRegClass.contains(Reg))
1719 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VSTRS))
1720 .addReg(RegNo: Reg)
1721 .addReg(RegNo: ARM::SP)
1722 .addImm(Val: Reg - ARM::S0)
1723 .add(MOs: predOps(Pred: ARMCC::AL));
1724 }
1725
1726 // Lazy load fp regs from stack.
1727 // This executes as NOP in the absence of floating-point support.
1728 MachineInstrBuilder VLLDM =
1729 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLLDM))
1730 .addReg(RegNo: ARM::SP)
1731 .add(MOs: predOps(Pred: ARMCC::AL))
1732 .addImm(Val: 0); // Represents a pseoudo register list, has no effect on
1733 // the encoding.
1734
1735 if (STI->fixCMSE_CVE_2021_35465()) {
1736 auto Bundler = MIBundleBuilder(MBB, VLLDM);
1737 // Read the CONTROL register.
1738 Bundler.append(MI: BuildMI(MF&: *MBB.getParent(), MIMD: DL, MCID: TII->get(Opcode: ARM::t2MRS_M))
1739 .addReg(RegNo: ScratchReg, Flags: RegState::Define)
1740 .addImm(Val: 20)
1741 .add(MOs: predOps(Pred: ARMCC::AL)));
1742 // Check bit 3 (SFPA).
1743 Bundler.append(MI: BuildMI(MF&: *MBB.getParent(), MIMD: DL, MCID: TII->get(Opcode: ARM::t2TSTri))
1744 .addReg(RegNo: ScratchReg)
1745 .addImm(Val: 8)
1746 .add(MOs: predOps(Pred: ARMCC::AL)));
1747 // Emit the IT block.
1748 Bundler.append(MI: BuildMI(MF&: *MBB.getParent(), MIMD: DL, MCID: TII->get(Opcode: ARM::t2IT))
1749 .addImm(Val: ARMCC::NE)
1750 .addImm(Val: 8));
1751 // If SFPA is clear jump over to VLLDM, otherwise execute an instruction
1752 // which has no functional effect apart from causing context creation:
1753 // vmovne s0, s0. In the absence of FPU we emit .inst.w 0xeeb00a40,
1754 // which is defined as NOP if not executed.
1755 if (STI->hasFPRegs())
1756 Bundler.append(MI: BuildMI(MF&: *MBB.getParent(), MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVS))
1757 .addReg(RegNo: ARM::S0, Flags: RegState::Define)
1758 .addReg(RegNo: ARM::S0, Flags: RegState::Undef)
1759 .add(MOs: predOps(Pred: ARMCC::NE)));
1760 else
1761 Bundler.append(MI: BuildMI(MF&: *MBB.getParent(), MIMD: DL, MCID: TII->get(Opcode: ARM::INLINEASM))
1762 .addExternalSymbol(FnName: ".inst.w 0xeeb00a40")
1763 .addImm(Val: InlineAsm::Extra_HasSideEffects));
1764 finalizeBundle(MBB, FirstMI: Bundler.begin(), LastMI: Bundler.end());
1765 }
1766
1767 // Restore all FP registers via normal registers
1768 for (const auto &Regs : ClearedFPRegs) {
1769 unsigned Reg, SaveReg1, SaveReg2;
1770 std::tie(args&: Reg, args&: SaveReg1, args&: SaveReg2) = Regs;
1771 if (ARM::DPR_VFP2RegClass.contains(Reg))
1772 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVDRR), DestReg: Reg)
1773 .addReg(RegNo: SaveReg1)
1774 .addReg(RegNo: SaveReg2)
1775 .add(MOs: predOps(Pred: ARMCC::AL));
1776 else if (ARM::SPRRegClass.contains(Reg))
1777 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VMOVSR), DestReg: Reg)
1778 .addReg(RegNo: SaveReg1)
1779 .add(MOs: predOps(Pred: ARMCC::AL));
1780 }
1781
1782 // Pop the stack space
1783 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tADDspi), DestReg: ARM::SP)
1784 .addReg(RegNo: ARM::SP)
1785 .addImm(Val: CMSE_FP_SAVE_SIZE >> 2)
1786 .add(MOs: predOps(Pred: ARMCC::AL));
1787}
1788
1789static bool definesOrUsesFPReg(const MachineInstr &MI) {
1790 for (const MachineOperand &Op : MI.operands()) {
1791 if (!Op.isReg())
1792 continue;
1793 Register Reg = Op.getReg();
1794 if ((Reg >= ARM::Q0 && Reg <= ARM::Q7) ||
1795 (Reg >= ARM::D0 && Reg <= ARM::D15) ||
1796 (Reg >= ARM::S0 && Reg <= ARM::S31))
1797 return true;
1798 }
1799 return false;
1800}
1801
1802void ARMExpandPseudo::CMSERestoreFPRegsV81(
1803 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1804 SmallVectorImpl<unsigned> &AvailableRegs) {
1805 if (!definesOrUsesFPReg(MI: *MBBI)) {
1806 if (STI->fixCMSE_CVE_2021_35465()) {
1807 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VSCCLRMS))
1808 .add(MOs: predOps(Pred: ARMCC::AL))
1809 .addReg(RegNo: ARM::VPR, Flags: RegState::Define);
1810 }
1811
1812 // Load FP registers from stack.
1813 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLLDM))
1814 .addReg(RegNo: ARM::SP)
1815 .add(MOs: predOps(Pred: ARMCC::AL))
1816 .addImm(Val: 0); // Represents a pseoudo register list, has no effect on the
1817 // encoding.
1818
1819 // Pop the stack space
1820 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tADDspi), DestReg: ARM::SP)
1821 .addReg(RegNo: ARM::SP)
1822 .addImm(Val: CMSE_FP_SAVE_SIZE >> 2)
1823 .add(MOs: predOps(Pred: ARMCC::AL));
1824 } else {
1825 // Restore the floating point context.
1826 BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: ARM::VLDR_FPCXTS_post),
1827 DestReg: ARM::SP)
1828 .addReg(RegNo: ARM::SP)
1829 .addImm(Val: 8)
1830 .add(MOs: predOps(Pred: ARMCC::AL));
1831
1832 // Pop all the callee-saved registers (s16-s31).
1833 MachineInstrBuilder VPOP =
1834 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::VLDMSIA_UPD), DestReg: ARM::SP)
1835 .addReg(RegNo: ARM::SP)
1836 .add(MOs: predOps(Pred: ARMCC::AL));
1837 for (unsigned Reg = ARM::S16; Reg <= ARM::S31; ++Reg)
1838 VPOP.addReg(RegNo: Reg, Flags: RegState::Define);
1839 }
1840}
1841
1842/// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as
1843/// possible. This only gets used at -O0 so we don't care about efficiency of
1844/// the generated code.
1845bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
1846 MachineBasicBlock::iterator MBBI,
1847 unsigned LdrexOp, unsigned StrexOp,
1848 unsigned UxtOp,
1849 MachineBasicBlock::iterator &NextMBBI) {
1850 bool IsThumb = STI->isThumb();
1851 bool IsThumb1Only = STI->isThumb1Only();
1852 MachineInstr &MI = *MBBI;
1853 DebugLoc DL = MI.getDebugLoc();
1854 const MachineOperand &Dest = MI.getOperand(i: 0);
1855 Register TempReg = MI.getOperand(i: 1).getReg();
1856 // Duplicating undef operands into 2 instructions does not guarantee the same
1857 // value on both; However undef should be replaced by xzr anyway.
1858 assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
1859 Register AddrReg = MI.getOperand(i: 2).getReg();
1860 Register DesiredReg = MI.getOperand(i: 3).getReg();
1861 Register NewReg = MI.getOperand(i: 4).getReg();
1862
1863 if (IsThumb) {
1864 assert(STI->hasV8MBaselineOps() &&
1865 "CMP_SWAP not expected to be custom expanded for Thumb1");
1866 assert((UxtOp == 0 || UxtOp == ARM::tUXTB || UxtOp == ARM::tUXTH) &&
1867 "ARMv8-M.baseline does not have t2UXTB/t2UXTH");
1868 assert((UxtOp == 0 || ARM::tGPRRegClass.contains(DesiredReg)) &&
1869 "DesiredReg used for UXT op must be tGPR");
1870 }
1871
1872 MachineFunction *MF = MBB.getParent();
1873 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1874 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1875 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
1876
1877 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
1878 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
1879 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: DoneBB);
1880
1881 if (UxtOp) {
1882 MachineInstrBuilder MIB =
1883 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: UxtOp), DestReg: DesiredReg)
1884 .addReg(RegNo: DesiredReg, Flags: RegState::Kill);
1885 if (!IsThumb)
1886 MIB.addImm(Val: 0);
1887 MIB.add(MOs: predOps(Pred: ARMCC::AL));
1888 }
1889
1890 // .Lloadcmp:
1891 // ldrex rDest, [rAddr]
1892 // cmp rDest, rDesired
1893 // bne .Ldone
1894
1895 MachineInstrBuilder MIB;
1896 MIB = BuildMI(BB: LoadCmpBB, MIMD: DL, MCID: TII->get(Opcode: LdrexOp), DestReg: Dest.getReg());
1897 MIB.addReg(RegNo: AddrReg);
1898 if (LdrexOp == ARM::t2LDREX)
1899 MIB.addImm(Val: 0); // a 32-bit Thumb ldrex (only) allows an offset.
1900 MIB.add(MOs: predOps(Pred: ARMCC::AL));
1901
1902 unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
1903 BuildMI(BB: LoadCmpBB, MIMD: DL, MCID: TII->get(Opcode: CMPrr))
1904 .addReg(RegNo: Dest.getReg(), Flags: getKillRegState(B: Dest.isDead()))
1905 .addReg(RegNo: DesiredReg)
1906 .add(MOs: predOps(Pred: ARMCC::AL));
1907 unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
1908 BuildMI(BB: LoadCmpBB, MIMD: DL, MCID: TII->get(Opcode: Bcc))
1909 .addMBB(MBB: DoneBB)
1910 .addImm(Val: ARMCC::NE)
1911 .addReg(RegNo: ARM::CPSR, Flags: RegState::Kill);
1912 LoadCmpBB->addSuccessor(Succ: DoneBB);
1913 LoadCmpBB->addSuccessor(Succ: StoreBB);
1914
1915 // .Lstore:
1916 // strex rTempReg, rNew, [rAddr]
1917 // cmp rTempReg, #0
1918 // bne .Lloadcmp
1919 MIB = BuildMI(BB: StoreBB, MIMD: DL, MCID: TII->get(Opcode: StrexOp), DestReg: TempReg)
1920 .addReg(RegNo: NewReg)
1921 .addReg(RegNo: AddrReg);
1922 if (StrexOp == ARM::t2STREX)
1923 MIB.addImm(Val: 0); // a 32-bit Thumb strex (only) allows an offset.
1924 MIB.add(MOs: predOps(Pred: ARMCC::AL));
1925
1926 unsigned CMPri =
1927 IsThumb ? (IsThumb1Only ? ARM::tCMPi8 : ARM::t2CMPri) : ARM::CMPri;
1928 BuildMI(BB: StoreBB, MIMD: DL, MCID: TII->get(Opcode: CMPri))
1929 .addReg(RegNo: TempReg, Flags: RegState::Kill)
1930 .addImm(Val: 0)
1931 .add(MOs: predOps(Pred: ARMCC::AL));
1932 BuildMI(BB: StoreBB, MIMD: DL, MCID: TII->get(Opcode: Bcc))
1933 .addMBB(MBB: LoadCmpBB)
1934 .addImm(Val: ARMCC::NE)
1935 .addReg(RegNo: ARM::CPSR, Flags: RegState::Kill);
1936 StoreBB->addSuccessor(Succ: LoadCmpBB);
1937 StoreBB->addSuccessor(Succ: DoneBB);
1938
1939 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
1940 DoneBB->transferSuccessors(FromMBB: &MBB);
1941
1942 MBB.addSuccessor(Succ: LoadCmpBB);
1943
1944 NextMBBI = MBB.end();
1945 MI.eraseFromParent();
1946
1947 // Recompute livein lists.
1948 LivePhysRegs LiveRegs;
1949 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
1950 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
1951 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
1952 // Do an extra pass around the loop to get loop carried registers right.
1953 StoreBB->clearLiveIns();
1954 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
1955 LoadCmpBB->clearLiveIns();
1956 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
1957
1958 return true;
1959}
1960
1961/// ARM's ldrexd/strexd take a consecutive register pair (represented as a
1962/// single GPRPair register), Thumb's take two separate registers so we need to
1963/// extract the subregs from the pair.
1964static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,
1965 RegState Flags, bool IsThumb,
1966 const TargetRegisterInfo *TRI) {
1967 if (IsThumb) {
1968 Register RegLo = TRI->getSubReg(Reg: Reg.getReg(), Idx: ARM::gsub_0);
1969 Register RegHi = TRI->getSubReg(Reg: Reg.getReg(), Idx: ARM::gsub_1);
1970 MIB.addReg(RegNo: RegLo, Flags);
1971 MIB.addReg(RegNo: RegHi, Flags);
1972 } else
1973 MIB.addReg(RegNo: Reg.getReg(), Flags);
1974}
1975
1976/// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.
1977bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
1978 MachineBasicBlock::iterator MBBI,
1979 MachineBasicBlock::iterator &NextMBBI) {
1980 bool IsThumb = STI->isThumb();
1981 assert(!STI->isThumb1Only() && "CMP_SWAP_64 unsupported under Thumb1!");
1982 MachineInstr &MI = *MBBI;
1983 DebugLoc DL = MI.getDebugLoc();
1984 MachineOperand &Dest = MI.getOperand(i: 0);
1985 // Duplicating undef operands into 2 instructions does not guarantee the same
1986 // value on both; However undef should be replaced by xzr anyway.
1987 assert(!MI.getOperand(1).isUndef() && "cannot handle undef");
1988 Register AddrAndTempReg = MI.getOperand(i: 1).getReg();
1989 Register AddrReg = TRI->getSubReg(Reg: AddrAndTempReg, Idx: ARM::gsub_0);
1990 Register TempReg = TRI->getSubReg(Reg: AddrAndTempReg, Idx: ARM::gsub_1);
1991 assert(MI.getOperand(1).getReg() == MI.getOperand(2).getReg() &&
1992 "tied operands have different registers");
1993 Register DesiredReg = MI.getOperand(i: 3).getReg();
1994 MachineOperand New = MI.getOperand(i: 4);
1995 New.setIsKill(false);
1996
1997 Register DestLo = TRI->getSubReg(Reg: Dest.getReg(), Idx: ARM::gsub_0);
1998 Register DestHi = TRI->getSubReg(Reg: Dest.getReg(), Idx: ARM::gsub_1);
1999 Register DesiredLo = TRI->getSubReg(Reg: DesiredReg, Idx: ARM::gsub_0);
2000 Register DesiredHi = TRI->getSubReg(Reg: DesiredReg, Idx: ARM::gsub_1);
2001
2002 MachineFunction *MF = MBB.getParent();
2003 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2004 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2005 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
2006
2007 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
2008 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
2009 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: DoneBB);
2010
2011 // .Lloadcmp:
2012 // ldrexd rDestLo, rDestHi, [rAddr]
2013 // cmp rDestLo, rDesiredLo
2014 // sbcs dead rTempReg, rDestHi, rDesiredHi
2015 // bne .Ldone
2016 unsigned LDREXD = IsThumb ? ARM::t2LDREXD : ARM::LDREXD;
2017 MachineInstrBuilder MIB;
2018 MIB = BuildMI(BB: LoadCmpBB, MIMD: DL, MCID: TII->get(Opcode: LDREXD));
2019 addExclusiveRegPair(MIB, Reg&: Dest, Flags: RegState::Define, IsThumb, TRI);
2020 MIB.addReg(RegNo: AddrReg).add(MOs: predOps(Pred: ARMCC::AL));
2021
2022 unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
2023 BuildMI(BB: LoadCmpBB, MIMD: DL, MCID: TII->get(Opcode: CMPrr))
2024 .addReg(RegNo: DestLo, Flags: getKillRegState(B: Dest.isDead()))
2025 .addReg(RegNo: DesiredLo)
2026 .add(MOs: predOps(Pred: ARMCC::AL));
2027
2028 BuildMI(BB: LoadCmpBB, MIMD: DL, MCID: TII->get(Opcode: CMPrr))
2029 .addReg(RegNo: DestHi, Flags: getKillRegState(B: Dest.isDead()))
2030 .addReg(RegNo: DesiredHi)
2031 .addImm(Val: ARMCC::EQ).addReg(RegNo: ARM::CPSR, Flags: RegState::Kill);
2032
2033 unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
2034 BuildMI(BB: LoadCmpBB, MIMD: DL, MCID: TII->get(Opcode: Bcc))
2035 .addMBB(MBB: DoneBB)
2036 .addImm(Val: ARMCC::NE)
2037 .addReg(RegNo: ARM::CPSR, Flags: RegState::Kill);
2038 LoadCmpBB->addSuccessor(Succ: DoneBB);
2039 LoadCmpBB->addSuccessor(Succ: StoreBB);
2040
2041 // .Lstore:
2042 // strexd rTempReg, rNewLo, rNewHi, [rAddr]
2043 // cmp rTempReg, #0
2044 // bne .Lloadcmp
2045 unsigned STREXD = IsThumb ? ARM::t2STREXD : ARM::STREXD;
2046 MIB = BuildMI(BB: StoreBB, MIMD: DL, MCID: TII->get(Opcode: STREXD), DestReg: TempReg);
2047 RegState Flags = getKillRegState(B: New.isDead());
2048 addExclusiveRegPair(MIB, Reg&: New, Flags, IsThumb, TRI);
2049 MIB.addReg(RegNo: AddrReg).add(MOs: predOps(Pred: ARMCC::AL));
2050
2051 unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
2052 BuildMI(BB: StoreBB, MIMD: DL, MCID: TII->get(Opcode: CMPri))
2053 .addReg(RegNo: TempReg, Flags: RegState::Kill)
2054 .addImm(Val: 0)
2055 .add(MOs: predOps(Pred: ARMCC::AL));
2056 BuildMI(BB: StoreBB, MIMD: DL, MCID: TII->get(Opcode: Bcc))
2057 .addMBB(MBB: LoadCmpBB)
2058 .addImm(Val: ARMCC::NE)
2059 .addReg(RegNo: ARM::CPSR, Flags: RegState::Kill);
2060 StoreBB->addSuccessor(Succ: LoadCmpBB);
2061 StoreBB->addSuccessor(Succ: DoneBB);
2062
2063 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
2064 DoneBB->transferSuccessors(FromMBB: &MBB);
2065
2066 MBB.addSuccessor(Succ: LoadCmpBB);
2067
2068 NextMBBI = MBB.end();
2069 MI.eraseFromParent();
2070
2071 // Recompute livein lists.
2072 LivePhysRegs LiveRegs;
2073 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
2074 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
2075 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
2076 // Do an extra pass around the loop to get loop carried registers right.
2077 StoreBB->clearLiveIns();
2078 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
2079 LoadCmpBB->clearLiveIns();
2080 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
2081
2082 return true;
2083}
2084
2085static void CMSEPushCalleeSaves(const TargetInstrInfo &TII,
2086 MachineBasicBlock &MBB,
2087 MachineBasicBlock::iterator MBBI,
2088 Register JumpReg, const LivePhysRegs &LiveRegs,
2089 bool Thumb1Only) {
2090 const DebugLoc &DL = MBBI->getDebugLoc();
2091 if (Thumb1Only) { // push Lo and Hi regs separately
2092 MachineInstrBuilder PushMIB =
2093 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tPUSH)).add(MOs: predOps(Pred: ARMCC::AL));
2094 for (unsigned Reg = ARM::R4; Reg < ARM::R8; ++Reg) {
2095 PushMIB.addReg(
2096 RegNo: Reg, Flags: getUndefRegState(B: Reg != JumpReg && !LiveRegs.contains(Reg)));
2097 }
2098
2099 // Thumb1 can only tPUSH low regs, so we copy the high regs to the low
2100 // regs that we just saved and push the low regs again, taking care to
2101 // not clobber JumpReg. If JumpReg is one of the low registers, push first
2102 // the values of r9-r11, and then r8. That would leave them ordered in
2103 // memory, and allow us to later pop them with a single instructions.
2104 // FIXME: Could also use any of r0-r3 that are free (including in the
2105 // first PUSH above).
2106 for (unsigned LoReg = ARM::R7, HiReg = ARM::R11; LoReg >= ARM::R4;
2107 --LoReg) {
2108 if (JumpReg == LoReg)
2109 continue;
2110 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: LoReg)
2111 .addReg(RegNo: HiReg, Flags: getUndefRegState(B: !LiveRegs.contains(Reg: HiReg)))
2112 .add(MOs: predOps(Pred: ARMCC::AL));
2113 --HiReg;
2114 }
2115 MachineInstrBuilder PushMIB2 =
2116 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tPUSH)).add(MOs: predOps(Pred: ARMCC::AL));
2117 for (unsigned Reg = ARM::R4; Reg < ARM::R8; ++Reg) {
2118 if (Reg == JumpReg)
2119 continue;
2120 PushMIB2.addReg(RegNo: Reg, Flags: RegState::Kill);
2121 }
2122
2123 // If we couldn't use a low register for temporary storage (because it was
2124 // the JumpReg), use r4 or r5, whichever is not JumpReg. It has already been
2125 // saved.
2126 if (JumpReg >= ARM::R4 && JumpReg <= ARM::R7) {
2127 Register LoReg = JumpReg == ARM::R4 ? ARM::R5 : ARM::R4;
2128 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: LoReg)
2129 .addReg(RegNo: ARM::R8, Flags: getUndefRegState(B: !LiveRegs.contains(Reg: ARM::R8)))
2130 .add(MOs: predOps(Pred: ARMCC::AL));
2131 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tPUSH))
2132 .add(MOs: predOps(Pred: ARMCC::AL))
2133 .addReg(RegNo: LoReg, Flags: RegState::Kill);
2134 }
2135 } else { // push Lo and Hi registers with a single instruction
2136 MachineInstrBuilder PushMIB =
2137 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2STMDB_UPD), DestReg: ARM::SP)
2138 .addReg(RegNo: ARM::SP)
2139 .add(MOs: predOps(Pred: ARMCC::AL));
2140 for (unsigned Reg = ARM::R4; Reg < ARM::R12; ++Reg) {
2141 PushMIB.addReg(
2142 RegNo: Reg, Flags: getUndefRegState(B: Reg != JumpReg && !LiveRegs.contains(Reg)));
2143 }
2144 }
2145}
2146
2147static void CMSEPopCalleeSaves(const TargetInstrInfo &TII,
2148 MachineBasicBlock &MBB,
2149 MachineBasicBlock::iterator MBBI,
2150 bool Thumb1Only) {
2151 const DebugLoc &DL = MBBI->getDebugLoc();
2152 if (Thumb1Only) {
2153 MachineInstrBuilder PopMIB =
2154 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tPOP)).add(MOs: predOps(Pred: ARMCC::AL));
2155 for (int R = 0; R < 4; ++R) {
2156 PopMIB.addReg(RegNo: ARM::R4 + R, Flags: RegState::Define);
2157 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tMOVr), DestReg: ARM::R8 + R)
2158 .addReg(RegNo: ARM::R4 + R, Flags: RegState::Kill)
2159 .add(MOs: predOps(Pred: ARMCC::AL));
2160 }
2161 MachineInstrBuilder PopMIB2 =
2162 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::tPOP)).add(MOs: predOps(Pred: ARMCC::AL));
2163 for (int R = 0; R < 4; ++R)
2164 PopMIB2.addReg(RegNo: ARM::R4 + R, Flags: RegState::Define);
2165 } else { // pop Lo and Hi registers with a single instruction
2166 MachineInstrBuilder PopMIB =
2167 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII.get(Opcode: ARM::t2LDMIA_UPD), DestReg: ARM::SP)
2168 .addReg(RegNo: ARM::SP)
2169 .add(MOs: predOps(Pred: ARMCC::AL));
2170 for (unsigned Reg = ARM::R4; Reg < ARM::R12; ++Reg)
2171 PopMIB.addReg(RegNo: Reg, Flags: RegState::Define);
2172 }
2173}
2174
2175bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
2176 MachineBasicBlock::iterator MBBI,
2177 MachineBasicBlock::iterator &NextMBBI) {
2178 MachineInstr &MI = *MBBI;
2179 unsigned Opcode = MI.getOpcode();
2180 switch (Opcode) {
2181 default:
2182 return false;
2183
2184 case ARM::VBSPd:
2185 case ARM::VBSPq: {
2186 Register DstReg = MI.getOperand(i: 0).getReg();
2187 if (DstReg == MI.getOperand(i: 3).getReg()) {
2188 // Expand to VBIT
2189 unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBITd : ARM::VBITq;
2190 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
2191 .add(MO: MI.getOperand(i: 0))
2192 .add(MO: MI.getOperand(i: 3))
2193 .add(MO: MI.getOperand(i: 2))
2194 .add(MO: MI.getOperand(i: 1))
2195 .addImm(Val: MI.getOperand(i: 4).getImm())
2196 .add(MO: MI.getOperand(i: 5));
2197 } else if (DstReg == MI.getOperand(i: 2).getReg()) {
2198 // Expand to VBIF
2199 unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBIFd : ARM::VBIFq;
2200 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
2201 .add(MO: MI.getOperand(i: 0))
2202 .add(MO: MI.getOperand(i: 2))
2203 .add(MO: MI.getOperand(i: 3))
2204 .add(MO: MI.getOperand(i: 1))
2205 .addImm(Val: MI.getOperand(i: 4).getImm())
2206 .add(MO: MI.getOperand(i: 5));
2207 } else {
2208 // Expand to VBSL
2209 unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBSLd : ARM::VBSLq;
2210 if (DstReg == MI.getOperand(i: 1).getReg()) {
2211 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
2212 .add(MO: MI.getOperand(i: 0))
2213 .add(MO: MI.getOperand(i: 1))
2214 .add(MO: MI.getOperand(i: 2))
2215 .add(MO: MI.getOperand(i: 3))
2216 .addImm(Val: MI.getOperand(i: 4).getImm())
2217 .add(MO: MI.getOperand(i: 5));
2218 } else {
2219 // Use move to satisfy constraints
2220 unsigned MoveOpc = Opcode == ARM::VBSPd ? ARM::VORRd : ARM::VORRq;
2221 RegState MO1Flags = getRegState(RegOp: MI.getOperand(i: 1)) & ~RegState::Kill;
2222 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: MoveOpc))
2223 .addReg(RegNo: DstReg,
2224 Flags: RegState::Define |
2225 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable()))
2226 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: MO1Flags)
2227 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: MO1Flags)
2228 .addImm(Val: MI.getOperand(i: 4).getImm())
2229 .add(MO: MI.getOperand(i: 5));
2230 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
2231 .add(MO: MI.getOperand(i: 0))
2232 .addReg(RegNo: DstReg,
2233 Flags: RegState::Kill |
2234 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable()))
2235 .add(MO: MI.getOperand(i: 2))
2236 .add(MO: MI.getOperand(i: 3))
2237 .addImm(Val: MI.getOperand(i: 4).getImm())
2238 .add(MO: MI.getOperand(i: 5));
2239 }
2240 }
2241 MI.eraseFromParent();
2242 return true;
2243 }
2244
2245 case ARM::TCRETURNdi:
2246 case ARM::TCRETURNri:
2247 case ARM::TCRETURNrinotr12: {
2248 MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
2249 if (MBBI->getOpcode() == ARM::SEH_EpilogEnd)
2250 MBBI--;
2251 if (MBBI->getOpcode() == ARM::SEH_Nop_Ret)
2252 MBBI--;
2253 assert(MBBI->isReturn() &&
2254 "Can only insert epilog into returning blocks");
2255 unsigned RetOpcode = MBBI->getOpcode();
2256 DebugLoc dl = MBBI->getDebugLoc();
2257 const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
2258 MBB.getParent()->getSubtarget().getInstrInfo());
2259
2260 // Tail call return: adjust the stack pointer and jump to callee.
2261 MBBI = MBB.getLastNonDebugInstr();
2262 if (MBBI->getOpcode() == ARM::SEH_EpilogEnd)
2263 MBBI--;
2264 if (MBBI->getOpcode() == ARM::SEH_Nop_Ret)
2265 MBBI--;
2266 MachineOperand &JumpTarget = MBBI->getOperand(i: 0);
2267
2268 // Jump to label or value in register.
2269 if (RetOpcode == ARM::TCRETURNdi) {
2270 MachineFunction *MF = MBB.getParent();
2271 bool NeedsWinCFI = MF->getTarget().getMCAsmInfo()->usesWindowsCFI() &&
2272 MF->getFunction().needsUnwindTableEntry();
2273 unsigned TCOpcode =
2274 STI->isThumb()
2275 ? ((STI->isTargetMachO() || NeedsWinCFI) ? ARM::tTAILJMPd
2276 : ARM::tTAILJMPdND)
2277 : ARM::TAILJMPd;
2278 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: dl, MCID: TII.get(Opcode: TCOpcode));
2279 if (JumpTarget.isGlobal())
2280 MIB.addGlobalAddress(GV: JumpTarget.getGlobal(), Offset: JumpTarget.getOffset(),
2281 TargetFlags: JumpTarget.getTargetFlags());
2282 else {
2283 assert(JumpTarget.isSymbol());
2284 MIB.addExternalSymbol(FnName: JumpTarget.getSymbolName(),
2285 TargetFlags: JumpTarget.getTargetFlags());
2286 }
2287
2288 // Add the default predicate in Thumb mode.
2289 if (STI->isThumb())
2290 MIB.add(MOs: predOps(Pred: ARMCC::AL));
2291 } else if (RetOpcode == ARM::TCRETURNri ||
2292 RetOpcode == ARM::TCRETURNrinotr12) {
2293 unsigned Opcode =
2294 STI->isThumb() ? ARM::tTAILJMPr
2295 : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);
2296 BuildMI(BB&: MBB, I: MBBI, MIMD: dl,
2297 MCID: TII.get(Opcode))
2298 .addReg(RegNo: JumpTarget.getReg(), Flags: RegState::Kill);
2299 }
2300
2301 auto NewMI = std::prev(x: MBBI);
2302 for (unsigned i = 2, e = MBBI->getNumOperands(); i != e; ++i)
2303 NewMI->addOperand(Op: MBBI->getOperand(i));
2304
2305 NewMI->setCFIType(MF&: *MBB.getParent(), Type: MI.getCFIType());
2306
2307 // Update call info and delete the pseudo instruction TCRETURN.
2308 if (MI.isCandidateForAdditionalCallInfo())
2309 MI.getMF()->moveAdditionalCallInfo(Old: &MI, New: &*NewMI);
2310 // Copy nomerge flag over to new instruction.
2311 if (MI.getFlag(Flag: MachineInstr::NoMerge))
2312 NewMI->setFlag(MachineInstr::NoMerge);
2313 MBB.erase(I: MBBI);
2314
2315 MBBI = NewMI;
2316 return true;
2317 }
2318 case ARM::tBXNS_RET: {
2319 // For v8.0-M.Main we need to authenticate LR before clearing FPRs, which
2320 // uses R12 as a scratch register.
2321 if (!STI->hasV8_1MMainlineOps() && AFI->shouldSignReturnAddress())
2322 BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: ARM::t2AUT));
2323
2324 MachineBasicBlock &AfterBB = CMSEClearFPRegs(MBB, MBBI);
2325
2326 if (STI->hasV8_1MMainlineOps()) {
2327 // Restore the non-secure floating point context.
2328 BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(),
2329 MCID: TII->get(Opcode: ARM::VLDR_FPCXTNS_post), DestReg: ARM::SP)
2330 .addReg(RegNo: ARM::SP)
2331 .addImm(Val: 4)
2332 .add(MOs: predOps(Pred: ARMCC::AL));
2333
2334 if (AFI->shouldSignReturnAddress())
2335 BuildMI(BB&: AfterBB, I: AfterBB.end(), MIMD: DebugLoc(), MCID: TII->get(Opcode: ARM::t2AUT));
2336 }
2337
2338 // Clear all GPR that are not a use of the return instruction.
2339 assert(llvm::all_of(MBBI->operands(), [](const MachineOperand &Op) {
2340 return !Op.isReg() || Op.getReg() != ARM::R12;
2341 }));
2342 SmallVector<unsigned, 5> ClearRegs;
2343 determineGPRegsToClear(
2344 MI: *MBBI, Regs: {ARM::R0, ARM::R1, ARM::R2, ARM::R3, ARM::R12}, ClearRegs);
2345 CMSEClearGPRegs(MBB&: AfterBB, MBBI: AfterBB.end(), DL: MBBI->getDebugLoc(), ClearRegs,
2346 ClobberReg: ARM::LR);
2347
2348 MachineInstrBuilder NewMI =
2349 BuildMI(BB&: AfterBB, I: AfterBB.end(), MIMD: MBBI->getDebugLoc(),
2350 MCID: TII->get(Opcode: ARM::tBXNS))
2351 .addReg(RegNo: ARM::LR)
2352 .add(MOs: predOps(Pred: ARMCC::AL));
2353 for (const MachineOperand &Op : MI.operands())
2354 NewMI->addOperand(Op);
2355 MI.eraseFromParent();
2356 return true;
2357 }
2358 case ARM::tBLXNS_CALL: {
2359 DebugLoc DL = MBBI->getDebugLoc();
2360 Register JumpReg = MBBI->getOperand(i: 0).getReg();
2361
2362 // Figure out which registers are live at the point immediately before the
2363 // call. When we indiscriminately push a set of registers, the live
2364 // registers are added as ordinary use operands, whereas dead registers
2365 // are "undef".
2366 LivePhysRegs LiveRegs(*TRI);
2367 LiveRegs.addLiveOuts(MBB);
2368 for (const MachineInstr &MI : make_range(x: MBB.rbegin(), y: MBBI.getReverse()))
2369 LiveRegs.stepBackward(MI);
2370 LiveRegs.stepBackward(MI: *MBBI);
2371
2372 CMSEPushCalleeSaves(TII: *TII, MBB, MBBI, JumpReg, LiveRegs,
2373 Thumb1Only: AFI->isThumb1OnlyFunction());
2374
2375 SmallVector<unsigned, 16> ClearRegs;
2376 determineGPRegsToClear(MI: *MBBI,
2377 Regs: {ARM::R0, ARM::R1, ARM::R2, ARM::R3, ARM::R4,
2378 ARM::R5, ARM::R6, ARM::R7, ARM::R8, ARM::R9,
2379 ARM::R10, ARM::R11, ARM::R12},
2380 ClearRegs);
2381 auto OriginalClearRegs = ClearRegs;
2382
2383 // Get the first cleared register as a scratch (to use later with tBIC).
2384 // We need to use the first so we can ensure it is a low register.
2385 unsigned ScratchReg = ClearRegs.front();
2386
2387 // Clear LSB of JumpReg
2388 if (AFI->isThumb2Function()) {
2389 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::t2BICri), DestReg: JumpReg)
2390 .addReg(RegNo: JumpReg)
2391 .addImm(Val: 1)
2392 .add(MOs: predOps(Pred: ARMCC::AL))
2393 .add(MO: condCodeOp());
2394 } else {
2395 // We need to use an extra register to cope with 8M Baseline,
2396 // since we have saved all of the registers we are ok to trash a non
2397 // argument register here.
2398 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tMOVi8), DestReg: ScratchReg)
2399 .add(MO: condCodeOp())
2400 .addImm(Val: 1)
2401 .add(MOs: predOps(Pred: ARMCC::AL));
2402 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tBIC), DestReg: JumpReg)
2403 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define)
2404 .addReg(RegNo: JumpReg)
2405 .addReg(RegNo: ScratchReg)
2406 .add(MOs: predOps(Pred: ARMCC::AL));
2407 }
2408
2409 CMSESaveClearFPRegs(MBB, MBBI, DL, LiveRegs,
2410 ScratchRegs&: ClearRegs); // save+clear FP regs with ClearRegs
2411 CMSEClearGPRegs(MBB, MBBI, DL, ClearRegs, ClobberReg: JumpReg);
2412
2413 const MachineInstrBuilder NewCall =
2414 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: ARM::tBLXNSr))
2415 .add(MOs: predOps(Pred: ARMCC::AL))
2416 .addReg(RegNo: JumpReg, Flags: RegState::Kill);
2417
2418 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
2419 NewCall->addOperand(Op: MO);
2420 if (MI.isCandidateForAdditionalCallInfo())
2421 MI.getMF()->moveAdditionalCallInfo(Old: &MI, New: NewCall.getInstr());
2422
2423 CMSERestoreFPRegs(MBB, MBBI, DL, AvailableRegs&: OriginalClearRegs); // restore FP registers
2424
2425 CMSEPopCalleeSaves(TII: *TII, MBB, MBBI, Thumb1Only: AFI->isThumb1OnlyFunction());
2426
2427 MI.eraseFromParent();
2428 return true;
2429 }
2430 case ARM::VMOVHcc:
2431 case ARM::VMOVScc:
2432 case ARM::VMOVDcc: {
2433 unsigned newOpc = Opcode != ARM::VMOVDcc ? ARM::VMOVS : ARM::VMOVD;
2434 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: newOpc),
2435 DestReg: MI.getOperand(i: 1).getReg())
2436 .add(MO: MI.getOperand(i: 2))
2437 .addImm(Val: MI.getOperand(i: 3).getImm()) // 'pred'
2438 .add(MO: MI.getOperand(i: 4))
2439 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2440
2441 MI.eraseFromParent();
2442 return true;
2443 }
2444 case ARM::t2MOVCCr:
2445 case ARM::MOVCCr: {
2446 unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVr : ARM::MOVr;
2447 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc),
2448 DestReg: MI.getOperand(i: 1).getReg())
2449 .add(MO: MI.getOperand(i: 2))
2450 .addImm(Val: MI.getOperand(i: 3).getImm()) // 'pred'
2451 .add(MO: MI.getOperand(i: 4))
2452 .add(MO: condCodeOp()) // 's' bit
2453 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2454
2455 MI.eraseFromParent();
2456 return true;
2457 }
2458 case ARM::MOVCCsi: {
2459 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::MOVsi),
2460 DestReg: (MI.getOperand(i: 1).getReg()))
2461 .add(MO: MI.getOperand(i: 2))
2462 .addImm(Val: MI.getOperand(i: 3).getImm())
2463 .addImm(Val: MI.getOperand(i: 4).getImm()) // 'pred'
2464 .add(MO: MI.getOperand(i: 5))
2465 .add(MO: condCodeOp()) // 's' bit
2466 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2467
2468 MI.eraseFromParent();
2469 return true;
2470 }
2471 case ARM::MOVCCsr: {
2472 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::MOVsr),
2473 DestReg: (MI.getOperand(i: 1).getReg()))
2474 .add(MO: MI.getOperand(i: 2))
2475 .add(MO: MI.getOperand(i: 3))
2476 .addImm(Val: MI.getOperand(i: 4).getImm())
2477 .addImm(Val: MI.getOperand(i: 5).getImm()) // 'pred'
2478 .add(MO: MI.getOperand(i: 6))
2479 .add(MO: condCodeOp()) // 's' bit
2480 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2481
2482 MI.eraseFromParent();
2483 return true;
2484 }
2485 case ARM::t2MOVCCi16:
2486 case ARM::MOVCCi16: {
2487 unsigned NewOpc = AFI->isThumbFunction() ? ARM::t2MOVi16 : ARM::MOVi16;
2488 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc),
2489 DestReg: MI.getOperand(i: 1).getReg())
2490 .addImm(Val: MI.getOperand(i: 2).getImm())
2491 .addImm(Val: MI.getOperand(i: 3).getImm()) // 'pred'
2492 .add(MO: MI.getOperand(i: 4))
2493 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2494 MI.eraseFromParent();
2495 return true;
2496 }
2497 case ARM::t2MOVCCi:
2498 case ARM::MOVCCi: {
2499 unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVi : ARM::MOVi;
2500 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc),
2501 DestReg: MI.getOperand(i: 1).getReg())
2502 .addImm(Val: MI.getOperand(i: 2).getImm())
2503 .addImm(Val: MI.getOperand(i: 3).getImm()) // 'pred'
2504 .add(MO: MI.getOperand(i: 4))
2505 .add(MO: condCodeOp()) // 's' bit
2506 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2507
2508 MI.eraseFromParent();
2509 return true;
2510 }
2511 case ARM::t2MVNCCi:
2512 case ARM::MVNCCi: {
2513 unsigned Opc = AFI->isThumbFunction() ? ARM::t2MVNi : ARM::MVNi;
2514 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc),
2515 DestReg: MI.getOperand(i: 1).getReg())
2516 .addImm(Val: MI.getOperand(i: 2).getImm())
2517 .addImm(Val: MI.getOperand(i: 3).getImm()) // 'pred'
2518 .add(MO: MI.getOperand(i: 4))
2519 .add(MO: condCodeOp()) // 's' bit
2520 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2521
2522 MI.eraseFromParent();
2523 return true;
2524 }
2525 case ARM::t2MOVCClsl:
2526 case ARM::t2MOVCClsr:
2527 case ARM::t2MOVCCasr:
2528 case ARM::t2MOVCCror: {
2529 unsigned NewOpc;
2530 switch (Opcode) {
2531 case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break;
2532 case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break;
2533 case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break;
2534 case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break;
2535 default: llvm_unreachable("unexpeced conditional move");
2536 }
2537 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc),
2538 DestReg: MI.getOperand(i: 1).getReg())
2539 .add(MO: MI.getOperand(i: 2))
2540 .addImm(Val: MI.getOperand(i: 3).getImm())
2541 .addImm(Val: MI.getOperand(i: 4).getImm()) // 'pred'
2542 .add(MO: MI.getOperand(i: 5))
2543 .add(MO: condCodeOp()) // 's' bit
2544 .add(MO: makeImplicit(MO: MI.getOperand(i: 1)));
2545 MI.eraseFromParent();
2546 return true;
2547 }
2548 case ARM::Int_eh_sjlj_dispatchsetup: {
2549 MachineFunction &MF = *MI.getParent()->getParent();
2550 const ARMBaseRegisterInfo &RI = TII->getRegisterInfo();
2551 // For functions using a base pointer, we rematerialize it (via the frame
2552 // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it
2553 // for us. Otherwise, expand to nothing.
2554 if (RI.hasBasePointer(MF)) {
2555 int32_t NumBytes = AFI->getFramePtrSpillOffset();
2556 Register FramePtr = RI.getFrameRegister(MF);
2557 assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&
2558 "base pointer without frame pointer?");
2559
2560 if (AFI->isThumb2Function()) {
2561 emitT2RegPlusImmediate(MBB, MBBI, dl: MI.getDebugLoc(), DestReg: ARM::R6,
2562 BaseReg: FramePtr, NumBytes: -NumBytes, Pred: ARMCC::AL, PredReg: 0, TII: *TII);
2563 } else if (AFI->isThumbFunction()) {
2564 emitThumbRegPlusImmediate(MBB, MBBI, dl: MI.getDebugLoc(), DestReg: ARM::R6,
2565 BaseReg: FramePtr, NumBytes: -NumBytes, TII: *TII, MRI: RI);
2566 } else {
2567 emitARMRegPlusImmediate(MBB, MBBI, dl: MI.getDebugLoc(), DestReg: ARM::R6,
2568 BaseReg: FramePtr, NumBytes: -NumBytes, Pred: ARMCC::AL, PredReg: 0,
2569 TII: *TII);
2570 }
2571 // If there's dynamic realignment, adjust for it.
2572 if (RI.hasStackRealignment(MF)) {
2573 MachineFrameInfo &MFI = MF.getFrameInfo();
2574 Align MaxAlign = MFI.getMaxAlign();
2575 assert (!AFI->isThumb1OnlyFunction());
2576 // Emit bic r6, r6, MaxAlign
2577 assert(MaxAlign <= Align(256) &&
2578 "The BIC instruction cannot encode "
2579 "immediates larger than 256 with all lower "
2580 "bits set.");
2581 unsigned bicOpc = AFI->isThumbFunction() ?
2582 ARM::t2BICri : ARM::BICri;
2583 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: bicOpc), DestReg: ARM::R6)
2584 .addReg(RegNo: ARM::R6, Flags: RegState::Kill)
2585 .addImm(Val: MaxAlign.value() - 1)
2586 .add(MOs: predOps(Pred: ARMCC::AL))
2587 .add(MO: condCodeOp());
2588 }
2589 }
2590 MI.eraseFromParent();
2591 return true;
2592 }
2593
2594 case ARM::LSRs1:
2595 case ARM::ASRs1: {
2596 // These are just fancy MOVs instructions.
2597 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::MOVsi),
2598 DestReg: MI.getOperand(i: 0).getReg())
2599 .add(MO: MI.getOperand(i: 1))
2600 .addImm(Val: ARM_AM::getSORegOpc(
2601 ShOp: (Opcode == ARM::LSRs1 ? ARM_AM::lsr : ARM_AM::asr), Imm: 1))
2602 .add(MOs: predOps(Pred: ARMCC::AL))
2603 .addReg(RegNo: ARM::CPSR, Flags: RegState::Define);
2604 MI.eraseFromParent();
2605 return true;
2606 }
2607 case ARM::RRX: {
2608 // This encodes as "MOVs Rd, Rm, rrx
2609 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::MOVsi),
2610 DestReg: MI.getOperand(i: 0).getReg())
2611 .add(MO: MI.getOperand(i: 1))
2612 .addImm(Val: ARM_AM::getSORegOpc(ShOp: ARM_AM::rrx, Imm: 0))
2613 .add(MOs: predOps(Pred: ARMCC::AL))
2614 .add(MO: condCodeOp())
2615 .copyImplicitOps(OtherMI: MI);
2616 MI.eraseFromParent();
2617 return true;
2618 }
2619 case ARM::tTPsoft:
2620 case ARM::TPsoft: {
2621 const bool Thumb = Opcode == ARM::tTPsoft;
2622
2623 MachineInstrBuilder MIB;
2624 MachineFunction *MF = MBB.getParent();
2625 if (STI->genLongCalls()) {
2626 MachineConstantPool *MCP = MF->getConstantPool();
2627 unsigned PCLabelID = AFI->createPICLabelUId();
2628 MachineConstantPoolValue *CPV =
2629 ARMConstantPoolSymbol::Create(C&: MF->getFunction().getContext(),
2630 s: "__aeabi_read_tp", ID: PCLabelID, PCAdj: 0);
2631 Register Reg = MI.getOperand(i: 0).getReg();
2632 MIB =
2633 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
2634 MCID: TII->get(Opcode: Thumb ? ARM::tLDRpci : ARM::LDRi12), DestReg: Reg)
2635 .addConstantPoolIndex(Idx: MCP->getConstantPoolIndex(V: CPV, Alignment: Align(4)));
2636 if (!Thumb)
2637 MIB.addImm(Val: 0);
2638 MIB.add(MOs: predOps(Pred: ARMCC::AL));
2639
2640 MIB =
2641 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
2642 MCID: TII->get(Opcode: Thumb ? gettBLXrOpcode(MF: *MF) : getBLXOpcode(MF: *MF)));
2643 if (Thumb)
2644 MIB.add(MOs: predOps(Pred: ARMCC::AL));
2645 MIB.addReg(RegNo: Reg, Flags: RegState::Kill);
2646 } else {
2647 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
2648 MCID: TII->get(Opcode: Thumb ? ARM::tBL : ARM::BL));
2649 if (Thumb)
2650 MIB.add(MOs: predOps(Pred: ARMCC::AL));
2651 MIB.addExternalSymbol(FnName: "__aeabi_read_tp", TargetFlags: 0);
2652 }
2653
2654 MIB.cloneMemRefs(OtherMI: MI);
2655 MIB.copyImplicitOps(OtherMI: MI);
2656 // Update the call info.
2657 if (MI.isCandidateForAdditionalCallInfo())
2658 MF->moveAdditionalCallInfo(Old: &MI, New: &*MIB);
2659 MI.eraseFromParent();
2660 return true;
2661 }
2662 case ARM::tLDRpci_pic:
2663 case ARM::t2LDRpci_pic: {
2664 unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)
2665 ? ARM::tLDRpci : ARM::t2LDRpci;
2666 Register DstReg = MI.getOperand(i: 0).getReg();
2667 bool DstIsDead = MI.getOperand(i: 0).isDead();
2668 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewLdOpc), DestReg: DstReg)
2669 .add(MO: MI.getOperand(i: 1))
2670 .add(MOs: predOps(Pred: ARMCC::AL))
2671 .cloneMemRefs(OtherMI: MI)
2672 .copyImplicitOps(OtherMI: MI);
2673 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::tPICADD))
2674 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead))
2675 .addReg(RegNo: DstReg)
2676 .add(MO: MI.getOperand(i: 2))
2677 .copyImplicitOps(OtherMI: MI);
2678 MI.eraseFromParent();
2679 return true;
2680 }
2681
2682 case ARM::LDRLIT_ga_abs:
2683 case ARM::LDRLIT_ga_pcrel:
2684 case ARM::LDRLIT_ga_pcrel_ldr:
2685 case ARM::tLDRLIT_ga_abs:
2686 case ARM::t2LDRLIT_ga_pcrel:
2687 case ARM::tLDRLIT_ga_pcrel: {
2688 Register DstReg = MI.getOperand(i: 0).getReg();
2689 bool DstIsDead = MI.getOperand(i: 0).isDead();
2690 const MachineOperand &MO1 = MI.getOperand(i: 1);
2691 auto Flags = MO1.getTargetFlags();
2692 const GlobalValue *GV = MO1.getGlobal();
2693 bool IsARM = Opcode != ARM::tLDRLIT_ga_pcrel &&
2694 Opcode != ARM::tLDRLIT_ga_abs &&
2695 Opcode != ARM::t2LDRLIT_ga_pcrel;
2696 bool IsPIC =
2697 Opcode != ARM::LDRLIT_ga_abs && Opcode != ARM::tLDRLIT_ga_abs;
2698 unsigned LDRLITOpc = IsARM ? ARM::LDRi12 : ARM::tLDRpci;
2699 if (Opcode == ARM::t2LDRLIT_ga_pcrel)
2700 LDRLITOpc = ARM::t2LDRpci;
2701 unsigned PICAddOpc =
2702 IsARM
2703 ? (Opcode == ARM::LDRLIT_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
2704 : ARM::tPICADD;
2705
2706 // We need a new const-pool entry to load from.
2707 MachineConstantPool *MCP = MBB.getParent()->getConstantPool();
2708 unsigned ARMPCLabelIndex = 0;
2709 MachineConstantPoolValue *CPV;
2710
2711 if (IsPIC) {
2712 unsigned PCAdj = IsARM ? 8 : 4;
2713 auto Modifier = (Flags & ARMII::MO_GOT)
2714 ? ARMCP::GOT_PREL
2715 : ARMCP::no_modifier;
2716 ARMPCLabelIndex = AFI->createPICLabelUId();
2717 CPV = ARMConstantPoolConstant::Create(
2718 C: GV, ID: ARMPCLabelIndex, Kind: ARMCP::CPValue, PCAdj, Modifier,
2719 /*AddCurrentAddr*/ AddCurrentAddress: Modifier == ARMCP::GOT_PREL);
2720 } else
2721 CPV = ARMConstantPoolConstant::Create(GV, Modifier: ARMCP::no_modifier);
2722
2723 MachineInstrBuilder MIB =
2724 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LDRLITOpc), DestReg: DstReg)
2725 .addConstantPoolIndex(Idx: MCP->getConstantPoolIndex(V: CPV, Alignment: Align(4)));
2726 if (IsARM)
2727 MIB.addImm(Val: 0);
2728 MIB.add(MOs: predOps(Pred: ARMCC::AL));
2729
2730 if (IsPIC) {
2731 MachineInstrBuilder MIB =
2732 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: PICAddOpc))
2733 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead))
2734 .addReg(RegNo: DstReg)
2735 .addImm(Val: ARMPCLabelIndex);
2736
2737 if (IsARM)
2738 MIB.add(MOs: predOps(Pred: ARMCC::AL));
2739 }
2740
2741 MI.eraseFromParent();
2742 return true;
2743 }
2744 case ARM::MOV_ga_pcrel:
2745 case ARM::MOV_ga_pcrel_ldr:
2746 case ARM::t2MOV_ga_pcrel: {
2747 // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.
2748 unsigned LabelId = AFI->createPICLabelUId();
2749 Register DstReg = MI.getOperand(i: 0).getReg();
2750 bool DstIsDead = MI.getOperand(i: 0).isDead();
2751 const MachineOperand &MO1 = MI.getOperand(i: 1);
2752 const GlobalValue *GV = MO1.getGlobal();
2753 unsigned TF = MO1.getTargetFlags();
2754 bool isARM = Opcode != ARM::t2MOV_ga_pcrel;
2755 unsigned LO16Opc = isARM ? ARM::MOVi16_ga_pcrel : ARM::t2MOVi16_ga_pcrel;
2756 unsigned HI16Opc = isARM ? ARM::MOVTi16_ga_pcrel :ARM::t2MOVTi16_ga_pcrel;
2757 unsigned LO16TF = TF | ARMII::MO_LO16;
2758 unsigned HI16TF = TF | ARMII::MO_HI16;
2759 unsigned PICAddOpc = isARM
2760 ? (Opcode == ARM::MOV_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
2761 : ARM::tPICADD;
2762 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LO16Opc), DestReg: DstReg)
2763 .addGlobalAddress(GV, Offset: MO1.getOffset(), TargetFlags: TF | LO16TF)
2764 .addImm(Val: LabelId)
2765 .copyImplicitOps(OtherMI: MI);
2766
2767 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HI16Opc), DestReg: DstReg)
2768 .addReg(RegNo: DstReg)
2769 .addGlobalAddress(GV, Offset: MO1.getOffset(), TargetFlags: TF | HI16TF)
2770 .addImm(Val: LabelId)
2771 .copyImplicitOps(OtherMI: MI);
2772
2773 MachineInstrBuilder MIB3 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
2774 MCID: TII->get(Opcode: PICAddOpc))
2775 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead))
2776 .addReg(RegNo: DstReg).addImm(Val: LabelId);
2777 if (isARM) {
2778 MIB3.add(MOs: predOps(Pred: ARMCC::AL));
2779 if (Opcode == ARM::MOV_ga_pcrel_ldr)
2780 MIB3.cloneMemRefs(OtherMI: MI);
2781 }
2782 MIB3.copyImplicitOps(OtherMI: MI);
2783 MI.eraseFromParent();
2784 return true;
2785 }
2786
2787 case ARM::MOVi32imm:
2788 case ARM::MOVCCi32imm:
2789 case ARM::t2MOVi32imm:
2790 case ARM::t2MOVCCi32imm:
2791 ExpandMOV32BitImm(MBB, MBBI);
2792 return true;
2793
2794 case ARM::tMOVi32imm:
2795 ExpandTMOV32BitImm(MBB, MBBI);
2796 return true;
2797
2798 case ARM::tLEApcrelJT:
2799 // Inline jump tables are handled in ARMAsmPrinter.
2800 if (MI.getMF()->getJumpTableInfo()->getEntryKind() ==
2801 MachineJumpTableInfo::EK_Inline)
2802 return false;
2803
2804 // Use a 32-bit immediate move to generate the address of the jump table.
2805 assert(STI->isThumb() && "Non-inline jump tables expected only in thumb");
2806 ExpandTMOV32BitImm(MBB, MBBI);
2807 return true;
2808
2809 case ARM::SUBS_PC_LR: {
2810 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::SUBri), DestReg: ARM::PC)
2811 .addReg(RegNo: ARM::LR)
2812 .add(MO: MI.getOperand(i: 0))
2813 .add(MO: MI.getOperand(i: 1))
2814 .add(MO: MI.getOperand(i: 2))
2815 .addReg(RegNo: ARM::CPSR, Flags: RegState::Undef)
2816 .copyImplicitOps(OtherMI: MI);
2817 MI.eraseFromParent();
2818 return true;
2819 }
2820 case ARM::VLDMQIA: {
2821 unsigned NewOpc = ARM::VLDMDIA;
2822 MachineInstrBuilder MIB =
2823 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc));
2824 unsigned OpIdx = 0;
2825
2826 // Grab the Q register destination.
2827 bool DstIsDead = MI.getOperand(i: OpIdx).isDead();
2828 Register DstReg = MI.getOperand(i: OpIdx++).getReg();
2829
2830 // Copy the source register.
2831 MIB.add(MO: MI.getOperand(i: OpIdx++));
2832
2833 // Copy the predicate operands.
2834 MIB.add(MO: MI.getOperand(i: OpIdx++));
2835 MIB.add(MO: MI.getOperand(i: OpIdx++));
2836
2837 // Add the destination operands (D subregs).
2838 Register D0 = TRI->getSubReg(Reg: DstReg, Idx: ARM::dsub_0);
2839 Register D1 = TRI->getSubReg(Reg: DstReg, Idx: ARM::dsub_1);
2840 MIB.addReg(RegNo: D0, Flags: RegState::Define | getDeadRegState(B: DstIsDead))
2841 .addReg(RegNo: D1, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
2842
2843 // Add an implicit def for the super-register.
2844 MIB.addReg(RegNo: DstReg, Flags: RegState::ImplicitDefine | getDeadRegState(B: DstIsDead));
2845 MIB.copyImplicitOps(OtherMI: MI);
2846 MIB.cloneMemRefs(OtherMI: MI);
2847 MI.eraseFromParent();
2848 return true;
2849 }
2850
2851 case ARM::VSTMQIA: {
2852 unsigned NewOpc = ARM::VSTMDIA;
2853 MachineInstrBuilder MIB =
2854 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: NewOpc));
2855 unsigned OpIdx = 0;
2856
2857 // Grab the Q register source.
2858 bool SrcIsKill = MI.getOperand(i: OpIdx).isKill();
2859 Register SrcReg = MI.getOperand(i: OpIdx++).getReg();
2860
2861 // Copy the destination register.
2862 MachineOperand Dst(MI.getOperand(i: OpIdx++));
2863 MIB.add(MO: Dst);
2864
2865 // Copy the predicate operands.
2866 MIB.add(MO: MI.getOperand(i: OpIdx++));
2867 MIB.add(MO: MI.getOperand(i: OpIdx++));
2868
2869 // Add the source operands (D subregs).
2870 Register D0 = TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_0);
2871 Register D1 = TRI->getSubReg(Reg: SrcReg, Idx: ARM::dsub_1);
2872 MIB.addReg(RegNo: D0, Flags: getKillRegState(B: SrcIsKill))
2873 .addReg(RegNo: D1, Flags: getKillRegState(B: SrcIsKill));
2874
2875 if (SrcIsKill) // Add an implicit kill for the Q register.
2876 MIB->addRegisterKilled(IncomingReg: SrcReg, RegInfo: TRI, AddIfNotFound: true);
2877
2878 MIB.copyImplicitOps(OtherMI: MI);
2879 MIB.cloneMemRefs(OtherMI: MI);
2880 MI.eraseFromParent();
2881 return true;
2882 }
2883
2884 case ARM::VLD2q8Pseudo:
2885 case ARM::VLD2q16Pseudo:
2886 case ARM::VLD2q32Pseudo:
2887 case ARM::VLD2q8PseudoWB_fixed:
2888 case ARM::VLD2q16PseudoWB_fixed:
2889 case ARM::VLD2q32PseudoWB_fixed:
2890 case ARM::VLD2q8PseudoWB_register:
2891 case ARM::VLD2q16PseudoWB_register:
2892 case ARM::VLD2q32PseudoWB_register:
2893 case ARM::VLD3d8Pseudo:
2894 case ARM::VLD3d16Pseudo:
2895 case ARM::VLD3d32Pseudo:
2896 case ARM::VLD1d8TPseudo:
2897 case ARM::VLD1d8TPseudoWB_fixed:
2898 case ARM::VLD1d8TPseudoWB_register:
2899 case ARM::VLD1d16TPseudo:
2900 case ARM::VLD1d16TPseudoWB_fixed:
2901 case ARM::VLD1d16TPseudoWB_register:
2902 case ARM::VLD1d32TPseudo:
2903 case ARM::VLD1d32TPseudoWB_fixed:
2904 case ARM::VLD1d32TPseudoWB_register:
2905 case ARM::VLD1d64TPseudo:
2906 case ARM::VLD1d64TPseudoWB_fixed:
2907 case ARM::VLD1d64TPseudoWB_register:
2908 case ARM::VLD3d8Pseudo_UPD:
2909 case ARM::VLD3d16Pseudo_UPD:
2910 case ARM::VLD3d32Pseudo_UPD:
2911 case ARM::VLD3q8Pseudo_UPD:
2912 case ARM::VLD3q16Pseudo_UPD:
2913 case ARM::VLD3q32Pseudo_UPD:
2914 case ARM::VLD3q8oddPseudo:
2915 case ARM::VLD3q16oddPseudo:
2916 case ARM::VLD3q32oddPseudo:
2917 case ARM::VLD3q8oddPseudo_UPD:
2918 case ARM::VLD3q16oddPseudo_UPD:
2919 case ARM::VLD3q32oddPseudo_UPD:
2920 case ARM::VLD4d8Pseudo:
2921 case ARM::VLD4d16Pseudo:
2922 case ARM::VLD4d32Pseudo:
2923 case ARM::VLD1d8QPseudo:
2924 case ARM::VLD1d8QPseudoWB_fixed:
2925 case ARM::VLD1d8QPseudoWB_register:
2926 case ARM::VLD1d16QPseudo:
2927 case ARM::VLD1d16QPseudoWB_fixed:
2928 case ARM::VLD1d16QPseudoWB_register:
2929 case ARM::VLD1d32QPseudo:
2930 case ARM::VLD1d32QPseudoWB_fixed:
2931 case ARM::VLD1d32QPseudoWB_register:
2932 case ARM::VLD1d64QPseudo:
2933 case ARM::VLD1d64QPseudoWB_fixed:
2934 case ARM::VLD1d64QPseudoWB_register:
2935 case ARM::VLD1q8HighQPseudo:
2936 case ARM::VLD1q8HighQPseudo_UPD:
2937 case ARM::VLD1q8LowQPseudo_UPD:
2938 case ARM::VLD1q8HighTPseudo:
2939 case ARM::VLD1q8HighTPseudo_UPD:
2940 case ARM::VLD1q8LowTPseudo_UPD:
2941 case ARM::VLD1q16HighQPseudo:
2942 case ARM::VLD1q16HighQPseudo_UPD:
2943 case ARM::VLD1q16LowQPseudo_UPD:
2944 case ARM::VLD1q16HighTPseudo:
2945 case ARM::VLD1q16HighTPseudo_UPD:
2946 case ARM::VLD1q16LowTPseudo_UPD:
2947 case ARM::VLD1q32HighQPseudo:
2948 case ARM::VLD1q32HighQPseudo_UPD:
2949 case ARM::VLD1q32LowQPseudo_UPD:
2950 case ARM::VLD1q32HighTPseudo:
2951 case ARM::VLD1q32HighTPseudo_UPD:
2952 case ARM::VLD1q32LowTPseudo_UPD:
2953 case ARM::VLD1q64HighQPseudo:
2954 case ARM::VLD1q64HighQPseudo_UPD:
2955 case ARM::VLD1q64LowQPseudo_UPD:
2956 case ARM::VLD1q64HighTPseudo:
2957 case ARM::VLD1q64HighTPseudo_UPD:
2958 case ARM::VLD1q64LowTPseudo_UPD:
2959 case ARM::VLD4d8Pseudo_UPD:
2960 case ARM::VLD4d16Pseudo_UPD:
2961 case ARM::VLD4d32Pseudo_UPD:
2962 case ARM::VLD4q8Pseudo_UPD:
2963 case ARM::VLD4q16Pseudo_UPD:
2964 case ARM::VLD4q32Pseudo_UPD:
2965 case ARM::VLD4q8oddPseudo:
2966 case ARM::VLD4q16oddPseudo:
2967 case ARM::VLD4q32oddPseudo:
2968 case ARM::VLD4q8oddPseudo_UPD:
2969 case ARM::VLD4q16oddPseudo_UPD:
2970 case ARM::VLD4q32oddPseudo_UPD:
2971 case ARM::VLD3DUPd8Pseudo:
2972 case ARM::VLD3DUPd16Pseudo:
2973 case ARM::VLD3DUPd32Pseudo:
2974 case ARM::VLD3DUPd8Pseudo_UPD:
2975 case ARM::VLD3DUPd16Pseudo_UPD:
2976 case ARM::VLD3DUPd32Pseudo_UPD:
2977 case ARM::VLD4DUPd8Pseudo:
2978 case ARM::VLD4DUPd16Pseudo:
2979 case ARM::VLD4DUPd32Pseudo:
2980 case ARM::VLD4DUPd8Pseudo_UPD:
2981 case ARM::VLD4DUPd16Pseudo_UPD:
2982 case ARM::VLD4DUPd32Pseudo_UPD:
2983 case ARM::VLD2DUPq8EvenPseudo:
2984 case ARM::VLD2DUPq8OddPseudo:
2985 case ARM::VLD2DUPq16EvenPseudo:
2986 case ARM::VLD2DUPq16OddPseudo:
2987 case ARM::VLD2DUPq32EvenPseudo:
2988 case ARM::VLD2DUPq32OddPseudo:
2989 case ARM::VLD2DUPq8OddPseudoWB_fixed:
2990 case ARM::VLD2DUPq8OddPseudoWB_register:
2991 case ARM::VLD2DUPq16OddPseudoWB_fixed:
2992 case ARM::VLD2DUPq16OddPseudoWB_register:
2993 case ARM::VLD2DUPq32OddPseudoWB_fixed:
2994 case ARM::VLD2DUPq32OddPseudoWB_register:
2995 case ARM::VLD3DUPq8EvenPseudo:
2996 case ARM::VLD3DUPq8OddPseudo:
2997 case ARM::VLD3DUPq16EvenPseudo:
2998 case ARM::VLD3DUPq16OddPseudo:
2999 case ARM::VLD3DUPq32EvenPseudo:
3000 case ARM::VLD3DUPq32OddPseudo:
3001 case ARM::VLD3DUPq8OddPseudo_UPD:
3002 case ARM::VLD3DUPq16OddPseudo_UPD:
3003 case ARM::VLD3DUPq32OddPseudo_UPD:
3004 case ARM::VLD4DUPq8EvenPseudo:
3005 case ARM::VLD4DUPq8OddPseudo:
3006 case ARM::VLD4DUPq16EvenPseudo:
3007 case ARM::VLD4DUPq16OddPseudo:
3008 case ARM::VLD4DUPq32EvenPseudo:
3009 case ARM::VLD4DUPq32OddPseudo:
3010 case ARM::VLD4DUPq8OddPseudo_UPD:
3011 case ARM::VLD4DUPq16OddPseudo_UPD:
3012 case ARM::VLD4DUPq32OddPseudo_UPD:
3013 ExpandVLD(MBBI);
3014 return true;
3015
3016 case ARM::VST2q8Pseudo:
3017 case ARM::VST2q16Pseudo:
3018 case ARM::VST2q32Pseudo:
3019 case ARM::VST2q8PseudoWB_fixed:
3020 case ARM::VST2q16PseudoWB_fixed:
3021 case ARM::VST2q32PseudoWB_fixed:
3022 case ARM::VST2q8PseudoWB_register:
3023 case ARM::VST2q16PseudoWB_register:
3024 case ARM::VST2q32PseudoWB_register:
3025 case ARM::VST3d8Pseudo:
3026 case ARM::VST3d16Pseudo:
3027 case ARM::VST3d32Pseudo:
3028 case ARM::VST1d8TPseudo:
3029 case ARM::VST1d8TPseudoWB_fixed:
3030 case ARM::VST1d8TPseudoWB_register:
3031 case ARM::VST1d16TPseudo:
3032 case ARM::VST1d16TPseudoWB_fixed:
3033 case ARM::VST1d16TPseudoWB_register:
3034 case ARM::VST1d32TPseudo:
3035 case ARM::VST1d32TPseudoWB_fixed:
3036 case ARM::VST1d32TPseudoWB_register:
3037 case ARM::VST1d64TPseudo:
3038 case ARM::VST1d64TPseudoWB_fixed:
3039 case ARM::VST1d64TPseudoWB_register:
3040 case ARM::VST3d8Pseudo_UPD:
3041 case ARM::VST3d16Pseudo_UPD:
3042 case ARM::VST3d32Pseudo_UPD:
3043 case ARM::VST3q8Pseudo_UPD:
3044 case ARM::VST3q16Pseudo_UPD:
3045 case ARM::VST3q32Pseudo_UPD:
3046 case ARM::VST3q8oddPseudo:
3047 case ARM::VST3q16oddPseudo:
3048 case ARM::VST3q32oddPseudo:
3049 case ARM::VST3q8oddPseudo_UPD:
3050 case ARM::VST3q16oddPseudo_UPD:
3051 case ARM::VST3q32oddPseudo_UPD:
3052 case ARM::VST4d8Pseudo:
3053 case ARM::VST4d16Pseudo:
3054 case ARM::VST4d32Pseudo:
3055 case ARM::VST1d8QPseudo:
3056 case ARM::VST1d8QPseudoWB_fixed:
3057 case ARM::VST1d8QPseudoWB_register:
3058 case ARM::VST1d16QPseudo:
3059 case ARM::VST1d16QPseudoWB_fixed:
3060 case ARM::VST1d16QPseudoWB_register:
3061 case ARM::VST1d32QPseudo:
3062 case ARM::VST1d32QPseudoWB_fixed:
3063 case ARM::VST1d32QPseudoWB_register:
3064 case ARM::VST1d64QPseudo:
3065 case ARM::VST1d64QPseudoWB_fixed:
3066 case ARM::VST1d64QPseudoWB_register:
3067 case ARM::VST4d8Pseudo_UPD:
3068 case ARM::VST4d16Pseudo_UPD:
3069 case ARM::VST4d32Pseudo_UPD:
3070 case ARM::VST1q8HighQPseudo:
3071 case ARM::VST1q8LowQPseudo_UPD:
3072 case ARM::VST1q8HighTPseudo:
3073 case ARM::VST1q8LowTPseudo_UPD:
3074 case ARM::VST1q16HighQPseudo:
3075 case ARM::VST1q16LowQPseudo_UPD:
3076 case ARM::VST1q16HighTPseudo:
3077 case ARM::VST1q16LowTPseudo_UPD:
3078 case ARM::VST1q32HighQPseudo:
3079 case ARM::VST1q32LowQPseudo_UPD:
3080 case ARM::VST1q32HighTPseudo:
3081 case ARM::VST1q32LowTPseudo_UPD:
3082 case ARM::VST1q64HighQPseudo:
3083 case ARM::VST1q64LowQPseudo_UPD:
3084 case ARM::VST1q64HighTPseudo:
3085 case ARM::VST1q64LowTPseudo_UPD:
3086 case ARM::VST1q8HighTPseudo_UPD:
3087 case ARM::VST1q16HighTPseudo_UPD:
3088 case ARM::VST1q32HighTPseudo_UPD:
3089 case ARM::VST1q64HighTPseudo_UPD:
3090 case ARM::VST1q8HighQPseudo_UPD:
3091 case ARM::VST1q16HighQPseudo_UPD:
3092 case ARM::VST1q32HighQPseudo_UPD:
3093 case ARM::VST1q64HighQPseudo_UPD:
3094 case ARM::VST4q8Pseudo_UPD:
3095 case ARM::VST4q16Pseudo_UPD:
3096 case ARM::VST4q32Pseudo_UPD:
3097 case ARM::VST4q8oddPseudo:
3098 case ARM::VST4q16oddPseudo:
3099 case ARM::VST4q32oddPseudo:
3100 case ARM::VST4q8oddPseudo_UPD:
3101 case ARM::VST4q16oddPseudo_UPD:
3102 case ARM::VST4q32oddPseudo_UPD:
3103 ExpandVST(MBBI);
3104 return true;
3105
3106 case ARM::VLD1LNq8Pseudo:
3107 case ARM::VLD1LNq16Pseudo:
3108 case ARM::VLD1LNq32Pseudo:
3109 case ARM::VLD1LNq8Pseudo_UPD:
3110 case ARM::VLD1LNq16Pseudo_UPD:
3111 case ARM::VLD1LNq32Pseudo_UPD:
3112 case ARM::VLD2LNd8Pseudo:
3113 case ARM::VLD2LNd16Pseudo:
3114 case ARM::VLD2LNd32Pseudo:
3115 case ARM::VLD2LNq16Pseudo:
3116 case ARM::VLD2LNq32Pseudo:
3117 case ARM::VLD2LNd8Pseudo_UPD:
3118 case ARM::VLD2LNd16Pseudo_UPD:
3119 case ARM::VLD2LNd32Pseudo_UPD:
3120 case ARM::VLD2LNq16Pseudo_UPD:
3121 case ARM::VLD2LNq32Pseudo_UPD:
3122 case ARM::VLD3LNd8Pseudo:
3123 case ARM::VLD3LNd16Pseudo:
3124 case ARM::VLD3LNd32Pseudo:
3125 case ARM::VLD3LNq16Pseudo:
3126 case ARM::VLD3LNq32Pseudo:
3127 case ARM::VLD3LNd8Pseudo_UPD:
3128 case ARM::VLD3LNd16Pseudo_UPD:
3129 case ARM::VLD3LNd32Pseudo_UPD:
3130 case ARM::VLD3LNq16Pseudo_UPD:
3131 case ARM::VLD3LNq32Pseudo_UPD:
3132 case ARM::VLD4LNd8Pseudo:
3133 case ARM::VLD4LNd16Pseudo:
3134 case ARM::VLD4LNd32Pseudo:
3135 case ARM::VLD4LNq16Pseudo:
3136 case ARM::VLD4LNq32Pseudo:
3137 case ARM::VLD4LNd8Pseudo_UPD:
3138 case ARM::VLD4LNd16Pseudo_UPD:
3139 case ARM::VLD4LNd32Pseudo_UPD:
3140 case ARM::VLD4LNq16Pseudo_UPD:
3141 case ARM::VLD4LNq32Pseudo_UPD:
3142 case ARM::VST1LNq8Pseudo:
3143 case ARM::VST1LNq16Pseudo:
3144 case ARM::VST1LNq32Pseudo:
3145 case ARM::VST1LNq8Pseudo_UPD:
3146 case ARM::VST1LNq16Pseudo_UPD:
3147 case ARM::VST1LNq32Pseudo_UPD:
3148 case ARM::VST2LNd8Pseudo:
3149 case ARM::VST2LNd16Pseudo:
3150 case ARM::VST2LNd32Pseudo:
3151 case ARM::VST2LNq16Pseudo:
3152 case ARM::VST2LNq32Pseudo:
3153 case ARM::VST2LNd8Pseudo_UPD:
3154 case ARM::VST2LNd16Pseudo_UPD:
3155 case ARM::VST2LNd32Pseudo_UPD:
3156 case ARM::VST2LNq16Pseudo_UPD:
3157 case ARM::VST2LNq32Pseudo_UPD:
3158 case ARM::VST3LNd8Pseudo:
3159 case ARM::VST3LNd16Pseudo:
3160 case ARM::VST3LNd32Pseudo:
3161 case ARM::VST3LNq16Pseudo:
3162 case ARM::VST3LNq32Pseudo:
3163 case ARM::VST3LNd8Pseudo_UPD:
3164 case ARM::VST3LNd16Pseudo_UPD:
3165 case ARM::VST3LNd32Pseudo_UPD:
3166 case ARM::VST3LNq16Pseudo_UPD:
3167 case ARM::VST3LNq32Pseudo_UPD:
3168 case ARM::VST4LNd8Pseudo:
3169 case ARM::VST4LNd16Pseudo:
3170 case ARM::VST4LNd32Pseudo:
3171 case ARM::VST4LNq16Pseudo:
3172 case ARM::VST4LNq32Pseudo:
3173 case ARM::VST4LNd8Pseudo_UPD:
3174 case ARM::VST4LNd16Pseudo_UPD:
3175 case ARM::VST4LNd32Pseudo_UPD:
3176 case ARM::VST4LNq16Pseudo_UPD:
3177 case ARM::VST4LNq32Pseudo_UPD:
3178 ExpandLaneOp(MBBI);
3179 return true;
3180
3181 case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, Opc: ARM::VTBL3, IsExt: false); return true;
3182 case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, Opc: ARM::VTBL4, IsExt: false); return true;
3183 case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, Opc: ARM::VTBX3, IsExt: true); return true;
3184 case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, Opc: ARM::VTBX4, IsExt: true); return true;
3185
3186 case ARM::MQQPRLoad:
3187 case ARM::MQQPRStore:
3188 case ARM::MQQQQPRLoad:
3189 case ARM::MQQQQPRStore:
3190 ExpandMQQPRLoadStore(MBBI);
3191 return true;
3192
3193 case ARM::tCMP_SWAP_8:
3194 assert(STI->isThumb());
3195 return ExpandCMP_SWAP(MBB, MBBI, LdrexOp: ARM::t2LDREXB, StrexOp: ARM::t2STREXB, UxtOp: ARM::tUXTB,
3196 NextMBBI);
3197 case ARM::tCMP_SWAP_16:
3198 assert(STI->isThumb());
3199 return ExpandCMP_SWAP(MBB, MBBI, LdrexOp: ARM::t2LDREXH, StrexOp: ARM::t2STREXH, UxtOp: ARM::tUXTH,
3200 NextMBBI);
3201 case ARM::tCMP_SWAP_32:
3202 assert(STI->isThumb());
3203 return ExpandCMP_SWAP(MBB, MBBI, LdrexOp: ARM::t2LDREX, StrexOp: ARM::t2STREX, UxtOp: 0, NextMBBI);
3204
3205 case ARM::CMP_SWAP_8:
3206 assert(!STI->isThumb());
3207 return ExpandCMP_SWAP(MBB, MBBI, LdrexOp: ARM::LDREXB, StrexOp: ARM::STREXB, UxtOp: ARM::UXTB,
3208 NextMBBI);
3209 case ARM::CMP_SWAP_16:
3210 assert(!STI->isThumb());
3211 return ExpandCMP_SWAP(MBB, MBBI, LdrexOp: ARM::LDREXH, StrexOp: ARM::STREXH, UxtOp: ARM::UXTH,
3212 NextMBBI);
3213 case ARM::CMP_SWAP_32:
3214 assert(!STI->isThumb());
3215 return ExpandCMP_SWAP(MBB, MBBI, LdrexOp: ARM::LDREX, StrexOp: ARM::STREX, UxtOp: 0, NextMBBI);
3216
3217 case ARM::CMP_SWAP_64:
3218 return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);
3219
3220 case ARM::tBL_PUSHLR:
3221 case ARM::BL_PUSHLR: {
3222 const bool Thumb = Opcode == ARM::tBL_PUSHLR;
3223 Register Reg = MI.getOperand(i: 0).getReg();
3224 assert(Reg == ARM::LR && "expect LR register!");
3225 MachineInstrBuilder MIB;
3226 if (Thumb) {
3227 // push {lr}
3228 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::tPUSH))
3229 .add(MOs: predOps(Pred: ARMCC::AL))
3230 .addReg(RegNo: Reg);
3231
3232 // bl __gnu_mcount_nc
3233 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::tBL));
3234 } else {
3235 // stmdb sp!, {lr}
3236 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::STMDB_UPD))
3237 .addReg(RegNo: ARM::SP, Flags: RegState::Define)
3238 .addReg(RegNo: ARM::SP)
3239 .add(MOs: predOps(Pred: ARMCC::AL))
3240 .addReg(RegNo: Reg);
3241
3242 // bl __gnu_mcount_nc
3243 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::BL));
3244 }
3245 MIB.cloneMemRefs(OtherMI: MI);
3246 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
3247 MIB.add(MO);
3248 MI.eraseFromParent();
3249 return true;
3250 }
3251 case ARM::t2CALL_BTI: {
3252 MachineFunction &MF = *MI.getMF();
3253 MachineInstrBuilder MIB =
3254 BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::tBL));
3255 MIB.cloneMemRefs(OtherMI: MI);
3256 for (unsigned i = 0; i < MI.getNumOperands(); ++i)
3257 MIB.add(MO: MI.getOperand(i));
3258 if (MI.isCandidateForAdditionalCallInfo())
3259 MF.moveAdditionalCallInfo(Old: &MI, New: MIB.getInstr());
3260 MIBundleBuilder Bundler(MBB, MI);
3261 Bundler.append(MI: MIB);
3262 Bundler.append(MI: BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: ARM::t2BTI)));
3263 finalizeBundle(MBB, FirstMI: Bundler.begin(), LastMI: Bundler.end());
3264 MI.eraseFromParent();
3265 return true;
3266 }
3267 case ARM::LOADDUAL:
3268 case ARM::STOREDUAL: {
3269 Register PairReg = MI.getOperand(i: 0).getReg();
3270
3271 MachineInstrBuilder MIB =
3272 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
3273 MCID: TII->get(Opcode: Opcode == ARM::LOADDUAL ? ARM::LDRD : ARM::STRD))
3274 .addReg(RegNo: TRI->getSubReg(Reg: PairReg, Idx: ARM::gsub_0),
3275 Flags: getDefRegState(B: Opcode == ARM::LOADDUAL))
3276 .addReg(RegNo: TRI->getSubReg(Reg: PairReg, Idx: ARM::gsub_1),
3277 Flags: getDefRegState(B: Opcode == ARM::LOADDUAL));
3278 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands()))
3279 MIB.add(MO);
3280 MIB.add(MOs: predOps(Pred: ARMCC::AL));
3281 MIB.cloneMemRefs(OtherMI: MI);
3282 MI.eraseFromParent();
3283 return true;
3284 }
3285 }
3286}
3287
3288bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
3289 bool Modified = false;
3290
3291 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3292 while (MBBI != E) {
3293 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
3294 Modified |= ExpandMI(MBB, MBBI, NextMBBI&: NMBBI);
3295 MBBI = NMBBI;
3296 }
3297
3298 return Modified;
3299}
3300
3301bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
3302 STI = &MF.getSubtarget<ARMSubtarget>();
3303 TII = STI->getInstrInfo();
3304 TRI = STI->getRegisterInfo();
3305 AFI = MF.getInfo<ARMFunctionInfo>();
3306
3307 LLVM_DEBUG(dbgs() << "********** ARM EXPAND PSEUDO INSTRUCTIONS **********\n"
3308 << "********** Function: " << MF.getName() << '\n');
3309
3310 bool Modified = false;
3311 for (MachineBasicBlock &MBB : MF)
3312 Modified |= ExpandMBB(MBB);
3313 if (VerifyARMPseudo)
3314 MF.verify(p: this, Banner: "After expanding ARM pseudo instructions.");
3315
3316 LLVM_DEBUG(dbgs() << "***************************************************\n");
3317 return Modified;
3318}
3319
3320/// createARMExpandPseudoPass - returns an instance of the pseudo instruction
3321/// expansion pass.
3322FunctionPass *llvm::createARMExpandPseudoPass() {
3323 return new ARMExpandPseudo();
3324}
3325