1//===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Pass to verify generated machine code. The following is checked:
10//
11// Operand counts: All explicit operands must be present.
12//
13// Register classes: All physical and virtual register operands must be
14// compatible with the register class required by the instruction descriptor.
15//
16// Register live intervals: Registers must be defined only once, and must be
17// defined before use.
18//
19// The machine code verifier is enabled with the command-line option
20// -verify-machineinstrs.
21//===----------------------------------------------------------------------===//
22
23#include "llvm/CodeGen/MachineVerifier.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/ADT/DenseSet.h"
27#include "llvm/ADT/DepthFirstIterator.h"
28#include "llvm/ADT/PostOrderIterator.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SetOperations.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/StringRef.h"
34#include "llvm/ADT/Twine.h"
35#include "llvm/CodeGen/CodeGenCommonISel.h"
36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
37#include "llvm/CodeGen/LiveInterval.h"
38#include "llvm/CodeGen/LiveIntervals.h"
39#include "llvm/CodeGen/LiveRangeCalc.h"
40#include "llvm/CodeGen/LiveStacks.h"
41#include "llvm/CodeGen/LiveVariables.h"
42#include "llvm/CodeGen/MachineBasicBlock.h"
43#include "llvm/CodeGen/MachineConvergenceVerifier.h"
44#include "llvm/CodeGen/MachineDominators.h"
45#include "llvm/CodeGen/MachineFrameInfo.h"
46#include "llvm/CodeGen/MachineFunction.h"
47#include "llvm/CodeGen/MachineFunctionPass.h"
48#include "llvm/CodeGen/MachineInstr.h"
49#include "llvm/CodeGen/MachineInstrBundle.h"
50#include "llvm/CodeGen/MachineMemOperand.h"
51#include "llvm/CodeGen/MachineOperand.h"
52#include "llvm/CodeGen/MachineRegisterInfo.h"
53#include "llvm/CodeGen/PseudoSourceValue.h"
54#include "llvm/CodeGen/RegisterBank.h"
55#include "llvm/CodeGen/RegisterBankInfo.h"
56#include "llvm/CodeGen/SlotIndexes.h"
57#include "llvm/CodeGen/StackMaps.h"
58#include "llvm/CodeGen/TargetInstrInfo.h"
59#include "llvm/CodeGen/TargetLowering.h"
60#include "llvm/CodeGen/TargetOpcodes.h"
61#include "llvm/CodeGen/TargetRegisterInfo.h"
62#include "llvm/CodeGen/TargetSubtargetInfo.h"
63#include "llvm/CodeGenTypes/LowLevelType.h"
64#include "llvm/IR/BasicBlock.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/EHPersonalities.h"
67#include "llvm/IR/Function.h"
68#include "llvm/IR/InlineAsm.h"
69#include "llvm/IR/Instructions.h"
70#include "llvm/InitializePasses.h"
71#include "llvm/MC/LaneBitmask.h"
72#include "llvm/MC/MCAsmInfo.h"
73#include "llvm/MC/MCDwarf.h"
74#include "llvm/MC/MCInstrDesc.h"
75#include "llvm/MC/MCRegisterInfo.h"
76#include "llvm/MC/MCTargetOptions.h"
77#include "llvm/Pass.h"
78#include "llvm/Support/Casting.h"
79#include "llvm/Support/ErrorHandling.h"
80#include "llvm/Support/ManagedStatic.h"
81#include "llvm/Support/MathExtras.h"
82#include "llvm/Support/ModRef.h"
83#include "llvm/Support/Mutex.h"
84#include "llvm/Support/raw_ostream.h"
85#include "llvm/Target/TargetMachine.h"
86#include <algorithm>
87#include <cassert>
88#include <cstddef>
89#include <cstdint>
90#include <iterator>
91#include <string>
92#include <utility>
93
94using namespace llvm;
95
96namespace {
97
98/// Used the by the ReportedErrors class to guarantee only one error is reported
99/// at one time.
100static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock;
101
102static bool hasPhysRegClassForType(const TargetRegisterInfo &TRI,
103 MCRegister Reg, LLT Ty) {
104 assert(Reg.isPhysical() && "reg must be a physical register");
105 assert(Ty.isValid() && "expected a valid type");
106
107 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
108 if (TRI.isTypeLegalForClass(RC: *RC, T: Ty))
109 return true;
110
111 return llvm::any_of(Range: TRI.regclasses(), P: [&](const TargetRegisterClass &RC) {
112 return RC.contains(Reg) && TRI.isTypeLegalForClass(RC, T: Ty);
113 });
114}
115
116struct MachineVerifier {
117 MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b,
118 raw_ostream *OS, bool AbortOnError = true)
119 : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b),
120 ReportedErrs(AbortOnError) {}
121
122 MachineVerifier(Pass *pass, const char *b, raw_ostream *OS,
123 bool AbortOnError = true)
124 : PASS(pass), OS(OS ? *OS : nulls()), Banner(b),
125 ReportedErrs(AbortOnError) {}
126
127 MachineVerifier(const char *b, LiveVariables *LiveVars,
128 LiveIntervals *LiveInts, LiveStacks *LiveStks,
129 SlotIndexes *Indexes, raw_ostream *OS,
130 bool AbortOnError = true)
131 : OS(OS ? *OS : nulls()), Banner(b), LiveVars(LiveVars),
132 LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes),
133 ReportedErrs(AbortOnError) {}
134
135 /// \returns true if no problems were found.
136 bool verify(const MachineFunction &MF);
137
138 MachineFunctionAnalysisManager *MFAM = nullptr;
139 Pass *const PASS = nullptr;
140 raw_ostream &OS;
141 const char *Banner;
142 const MachineFunction *MF = nullptr;
143 const TargetMachine *TM = nullptr;
144 const TargetInstrInfo *TII = nullptr;
145 const TargetRegisterInfo *TRI = nullptr;
146 const MachineRegisterInfo *MRI = nullptr;
147 const RegisterBankInfo *RBI = nullptr;
148
149 // Avoid querying the MachineFunctionProperties for each operand.
150 bool isFunctionRegBankSelected = false;
151 bool isFunctionSelected = false;
152 bool isFunctionTracksDebugUserValues = false;
153
154 using RegVector = SmallVector<Register, 16>;
155 using RegMaskVector = SmallVector<const uint32_t *, 4>;
156 using RegSet = DenseSet<Register>;
157 using RegMap = DenseMap<Register, const MachineInstr *>;
158 using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;
159
160 const MachineInstr *FirstNonPHI = nullptr;
161 const MachineInstr *FirstTerminator = nullptr;
162 BlockSet FunctionBlocks;
163
164 BitVector regsReserved;
165 RegSet regsLive;
166 RegVector regsDefined, regsDead, regsKilled;
167 RegMaskVector regMasks;
168
169 SlotIndex lastIndex;
170
171 // Add Reg and any sub-registers to RV
172 void addRegWithSubRegs(RegVector &RV, Register Reg) {
173 RV.push_back(Elt: Reg);
174 if (Reg.isPhysical())
175 append_range(C&: RV, R: TRI->subregs(Reg: Reg.asMCReg()));
176 }
177
178 struct BBInfo {
179 // Is this MBB reachable from the MF entry point?
180 bool reachable = false;
181
182 // Vregs that must be live in because they are used without being
183 // defined. Map value is the user. vregsLiveIn doesn't include regs
184 // that only are used by PHI nodes.
185 RegMap vregsLiveIn;
186
187 // Regs killed in MBB. They may be defined again, and will then be in both
188 // regsKilled and regsLiveOut.
189 RegSet regsKilled;
190
191 // Regs defined in MBB and live out. Note that vregs passing through may
192 // be live out without being mentioned here.
193 RegSet regsLiveOut;
194
195 // Vregs that pass through MBB untouched. This set is disjoint from
196 // regsKilled and regsLiveOut.
197 RegSet vregsPassed;
198
199 // Vregs that must pass through MBB because they are needed by a successor
200 // block. This set is disjoint from regsLiveOut.
201 RegSet vregsRequired;
202
203 // Set versions of block's predecessor and successor lists.
204 BlockSet Preds, Succs;
205
206 BBInfo() = default;
207
208 // Add register to vregsRequired if it belongs there. Return true if
209 // anything changed.
210 bool addRequired(Register Reg) {
211 if (!Reg.isVirtual())
212 return false;
213 if (regsLiveOut.count(V: Reg))
214 return false;
215 return vregsRequired.insert(V: Reg).second;
216 }
217
218 // Same for a full set.
219 bool addRequired(const RegSet &RS) {
220 bool Changed = false;
221 for (Register Reg : RS)
222 Changed |= addRequired(Reg);
223 return Changed;
224 }
225
226 // Same for a full map.
227 bool addRequired(const RegMap &RM) {
228 bool Changed = false;
229 for (const auto &I : RM)
230 Changed |= addRequired(Reg: I.first);
231 return Changed;
232 }
233
234 // Live-out registers are either in regsLiveOut or vregsPassed.
235 bool isLiveOut(Register Reg) const {
236 return regsLiveOut.count(V: Reg) || vregsPassed.count(V: Reg);
237 }
238 };
239
240 // Extra register info per MBB.
241 DenseMap<const MachineBasicBlock *, BBInfo> MBBInfoMap;
242
243 bool isReserved(Register Reg) {
244 return Reg.id() < regsReserved.size() && regsReserved.test(Idx: Reg.id());
245 }
246
247 bool isAllocatable(Register Reg) const {
248 return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(RegNo: Reg) &&
249 !regsReserved.test(Idx: Reg.id());
250 }
251
252 // Analysis information if available
253 LiveVariables *LiveVars = nullptr;
254 LiveIntervals *LiveInts = nullptr;
255 LiveStacks *LiveStks = nullptr;
256 SlotIndexes *Indexes = nullptr;
257
258 /// A class to track the number of reported error and to guarantee that only
259 /// one error is reported at one time.
260 class ReportedErrors {
261 unsigned NumReported = 0;
262 bool AbortOnError;
263
264 public:
265 /// \param AbortOnError -- If set, abort after printing the first error.
266 ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {}
267
268 ~ReportedErrors() {
269 if (!hasError())
270 return;
271 if (AbortOnError)
272 report_fatal_error(reason: "Found " + Twine(NumReported) +
273 " machine code errors.");
274 // Since we haven't aborted, release the lock to allow other threads to
275 // report errors.
276 ReportedErrorsLock->unlock();
277 }
278
279 /// Increment the number of reported errors.
280 /// \returns true if this is the first reported error.
281 bool increment() {
282 // If this is the first error this thread has encountered, grab the lock
283 // to prevent other threads from reporting errors at the same time.
284 // Otherwise we assume we already have the lock.
285 if (!hasError())
286 ReportedErrorsLock->lock();
287 ++NumReported;
288 return NumReported == 1;
289 }
290
291 /// \returns true if an error was reported.
292 bool hasError() { return NumReported; }
293 };
294 ReportedErrors ReportedErrs;
295
296 // This is calculated only when trying to verify convergence control tokens.
297 // Similar to the LLVM IR verifier, we calculate this locally instead of
298 // relying on the pass manager.
299 MachineDominatorTree DT;
300
301 void visitMachineFunctionBefore();
302 void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
303 void visitMachineBundleBefore(const MachineInstr *MI);
304
305 /// Verify that all of \p MI's virtual register operands are scalars.
306 /// \returns True if all virtual register operands are scalar. False
307 /// otherwise.
308 bool verifyAllRegOpsScalar(const MachineInstr &MI,
309 const MachineRegisterInfo &MRI);
310 bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);
311
312 bool verifyGIntrinsicSideEffects(const MachineInstr *MI);
313 bool verifyGIntrinsicConvergence(const MachineInstr *MI);
314 void verifyPreISelGenericInstruction(const MachineInstr *MI);
315
316 void visitMachineInstrBefore(const MachineInstr *MI);
317 void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
318 void visitMachineBundleAfter(const MachineInstr *MI);
319 void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);
320 void visitMachineFunctionAfter();
321
322 void report(const char *msg, const MachineFunction *MF);
323 void report(const char *msg, const MachineBasicBlock *MBB);
324 void report(const char *msg, const MachineInstr *MI);
325 void report(const char *msg, const MachineOperand *MO, unsigned MONum,
326 LLT MOVRegType = LLT{});
327 void report(const Twine &Msg, const MachineInstr *MI);
328
329 void report_context(const LiveInterval &LI) const;
330 void report_context(const LiveRange &LR, VirtRegOrUnit VRegOrUnit,
331 LaneBitmask LaneMask) const;
332 void report_context(const LiveRange::Segment &S) const;
333 void report_context(const VNInfo &VNI) const;
334 void report_context(SlotIndex Pos) const;
335 void report_context(MCPhysReg PhysReg) const;
336 void report_context_liverange(const LiveRange &LR) const;
337 void report_context_lanemask(LaneBitmask LaneMask) const;
338 void report_context_vreg(Register VReg) const;
339 void report_context_vreg_regunit(VirtRegOrUnit VRegOrUnit) const;
340
341 void verifyInlineAsm(const MachineInstr *MI);
342
343 void checkLiveness(const MachineOperand *MO, unsigned MONum);
344 void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
345 SlotIndex UseIdx, const LiveRange &LR,
346 VirtRegOrUnit VRegOrUnit,
347 LaneBitmask LaneMask = LaneBitmask::getNone());
348 void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
349 SlotIndex DefIdx, const LiveRange &LR,
350 VirtRegOrUnit VRegOrUnit, bool SubRangeCheck = false,
351 LaneBitmask LaneMask = LaneBitmask::getNone());
352
353 void markReachable(const MachineBasicBlock *MBB);
354 void calcRegsPassed();
355 void checkPHIOps(const MachineBasicBlock &MBB);
356
357 void calcRegsRequired();
358 void verifyLiveVariables();
359 void verifyLiveIntervals();
360 void verifyLiveInterval(const LiveInterval &);
361 void verifyLiveRangeValue(const LiveRange &, const VNInfo *, VirtRegOrUnit,
362 LaneBitmask);
363 void verifyLiveRangeSegment(const LiveRange &,
364 const LiveRange::const_iterator I, VirtRegOrUnit,
365 LaneBitmask);
366 void verifyLiveRange(const LiveRange &, VirtRegOrUnit,
367 LaneBitmask LaneMask = LaneBitmask::getNone());
368
369 void verifyStackFrame();
370 /// Check that the stack protector is the top-most object in the stack.
371 void verifyStackProtector();
372
373 void verifySlotIndexes() const;
374 void verifyProperties(const MachineFunction &MF);
375};
376
377struct MachineVerifierLegacyPass : public MachineFunctionPass {
378 static char ID; // Pass ID, replacement for typeid
379
380 const std::string Banner;
381
382 MachineVerifierLegacyPass(std::string banner = std::string())
383 : MachineFunctionPass(ID), Banner(std::move(banner)) {}
384
385 void getAnalysisUsage(AnalysisUsage &AU) const override {
386 AU.addUsedIfAvailable<LiveStacksWrapperLegacy>();
387 AU.addUsedIfAvailable<LiveVariablesWrapperPass>();
388 AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
389 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
390 AU.setPreservesAll();
391 MachineFunctionPass::getAnalysisUsage(AU);
392 }
393
394 bool runOnMachineFunction(MachineFunction &MF) override {
395 // Skip functions that have known verification problems.
396 // FIXME: Remove this mechanism when all problematic passes have been
397 // fixed.
398 if (MF.getProperties().hasFailsVerification())
399 return false;
400
401 MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);
402 return false;
403 }
404};
405
406} // end anonymous namespace
407
408PreservedAnalyses
409MachineVerifierPass::run(MachineFunction &MF,
410 MachineFunctionAnalysisManager &MFAM) {
411 // Skip functions that have known verification problems.
412 // FIXME: Remove this mechanism when all problematic passes have been
413 // fixed.
414 if (MF.getProperties().hasFailsVerification())
415 return PreservedAnalyses::all();
416 MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);
417 return PreservedAnalyses::all();
418}
419
420char MachineVerifierLegacyPass::ID = 0;
421
422INITIALIZE_PASS(MachineVerifierLegacyPass, "machineverifier",
423 "Verify generated machine code", false, false)
424
425FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {
426 return new MachineVerifierLegacyPass(Banner);
427}
428
429void llvm::verifyMachineFunction(const std::string &Banner,
430 const MachineFunction &MF) {
431 // TODO: Use MFAM after porting below analyses.
432 // LiveVariables *LiveVars;
433 // LiveIntervals *LiveInts;
434 // LiveStacks *LiveStks;
435 // SlotIndexes *Indexes;
436 MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);
437}
438
439bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,
440 bool AbortOnError) const {
441 return MachineVerifier(p, Banner, OS, AbortOnError).verify(MF: *this);
442}
443
444bool MachineFunction::verify(MachineFunctionAnalysisManager &MFAM,
445 const char *Banner, raw_ostream *OS,
446 bool AbortOnError) const {
447 return MachineVerifier(MFAM, Banner, OS, AbortOnError).verify(MF: *this);
448}
449
450bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
451 const char *Banner, raw_ostream *OS,
452 bool AbortOnError) const {
453 return MachineVerifier(Banner, /*LiveVars=*/nullptr, LiveInts,
454 /*LiveStks=*/nullptr, Indexes, OS, AbortOnError)
455 .verify(MF: *this);
456}
457
458void MachineVerifier::verifySlotIndexes() const {
459 if (Indexes == nullptr)
460 return;
461
462 // Ensure the IdxMBB list is sorted by slot indexes.
463 SlotIndex Last;
464 for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),
465 E = Indexes->MBBIndexEnd(); I != E; ++I) {
466 assert(!Last.isValid() || I->first > Last);
467 Last = I->first;
468 }
469}
470
471void MachineVerifier::verifyProperties(const MachineFunction &MF) {
472 // If a pass has introduced virtual registers without clearing the
473 // NoVRegs property (or set it without allocating the vregs)
474 // then report an error.
475 if (MF.getProperties().hasNoVRegs() && MRI->getNumVirtRegs())
476 report(msg: "Function has NoVRegs property but there are VReg operands", MF: &MF);
477}
478
479bool MachineVerifier::verify(const MachineFunction &MF) {
480 this->MF = &MF;
481 TM = &MF.getTarget();
482 TII = MF.getSubtarget().getInstrInfo();
483 TRI = MF.getSubtarget().getRegisterInfo();
484 RBI = MF.getSubtarget().getRegBankInfo();
485 MRI = &MF.getRegInfo();
486
487 const MachineFunctionProperties &Props = MF.getProperties();
488 const bool isFunctionFailedISel = Props.hasFailedISel();
489
490 // If we're mid-GlobalISel and we already triggered the fallback path then
491 // it's expected that the MIR is somewhat broken but that's ok since we'll
492 // reset it and clear the FailedISel attribute in ResetMachineFunctions.
493 if (isFunctionFailedISel)
494 return true;
495
496 isFunctionRegBankSelected = Props.hasRegBankSelected();
497 isFunctionSelected = Props.hasSelected();
498 isFunctionTracksDebugUserValues = Props.hasTracksDebugUserValues();
499
500 if (PASS) {
501 auto *LISWrapper = PASS->getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
502 LiveInts = LISWrapper ? &LISWrapper->getLIS() : nullptr;
503 // We don't want to verify LiveVariables if LiveIntervals is available.
504 auto *LVWrapper = PASS->getAnalysisIfAvailable<LiveVariablesWrapperPass>();
505 if (!LiveInts)
506 LiveVars = LVWrapper ? &LVWrapper->getLV() : nullptr;
507 auto *LSWrapper = PASS->getAnalysisIfAvailable<LiveStacksWrapperLegacy>();
508 LiveStks = LSWrapper ? &LSWrapper->getLS() : nullptr;
509 auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>();
510 Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
511 }
512 if (MFAM) {
513 MachineFunction &Func = const_cast<MachineFunction &>(MF);
514 LiveInts = MFAM->getCachedResult<LiveIntervalsAnalysis>(IR&: Func);
515 if (!LiveInts)
516 LiveVars = MFAM->getCachedResult<LiveVariablesAnalysis>(IR&: Func);
517 // TODO: LiveStks = MFAM->getCachedResult<LiveStacksAnalysis>(Func);
518 Indexes = MFAM->getCachedResult<SlotIndexesAnalysis>(IR&: Func);
519 }
520
521 verifySlotIndexes();
522
523 verifyProperties(MF);
524
525 visitMachineFunctionBefore();
526 for (const MachineBasicBlock &MBB : MF) {
527 visitMachineBasicBlockBefore(MBB: &MBB);
528 // Keep track of the current bundle header.
529 const MachineInstr *CurBundle = nullptr;
530 // Do we expect the next instruction to be part of the same bundle?
531 bool InBundle = false;
532
533 for (const MachineInstr &MI : MBB.instrs()) {
534 if (MI.getParent() != &MBB) {
535 report(msg: "Bad instruction parent pointer", MBB: &MBB);
536 OS << "Instruction: " << MI;
537 continue;
538 }
539
540 // Check for consistent bundle flags.
541 if (InBundle && !MI.isBundledWithPred())
542 report(msg: "Missing BundledPred flag, "
543 "BundledSucc was set on predecessor",
544 MI: &MI);
545 if (!InBundle && MI.isBundledWithPred())
546 report(msg: "BundledPred flag is set, "
547 "but BundledSucc not set on predecessor",
548 MI: &MI);
549
550 // Is this a bundle header?
551 if (!MI.isInsideBundle()) {
552 if (CurBundle)
553 visitMachineBundleAfter(MI: CurBundle);
554 CurBundle = &MI;
555 visitMachineBundleBefore(MI: CurBundle);
556 } else if (!CurBundle)
557 report(msg: "No bundle header", MI: &MI);
558 visitMachineInstrBefore(MI: &MI);
559 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
560 const MachineOperand &Op = MI.getOperand(i: I);
561 if (Op.getParent() != &MI) {
562 // Make sure to use correct addOperand / removeOperand / ChangeTo
563 // functions when replacing operands of a MachineInstr.
564 report(msg: "Instruction has operand with wrong parent set", MI: &MI);
565 }
566
567 visitMachineOperand(MO: &Op, MONum: I);
568 }
569
570 // Was this the last bundled instruction?
571 InBundle = MI.isBundledWithSucc();
572 }
573 if (CurBundle)
574 visitMachineBundleAfter(MI: CurBundle);
575 if (InBundle)
576 report(msg: "BundledSucc flag set on last instruction in block", MI: &MBB.back());
577 visitMachineBasicBlockAfter(MBB: &MBB);
578 }
579 visitMachineFunctionAfter();
580
581 // Clean up.
582 regsLive.clear();
583 regsDefined.clear();
584 regsDead.clear();
585 regsKilled.clear();
586 regMasks.clear();
587 MBBInfoMap.clear();
588
589 return !ReportedErrs.hasError();
590}
591
592void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
593 assert(MF);
594 OS << '\n';
595 if (ReportedErrs.increment()) {
596 if (Banner)
597 OS << "# " << Banner << '\n';
598
599 if (LiveInts != nullptr)
600 LiveInts->print(O&: OS);
601 else
602 MF->print(OS, Indexes);
603 }
604
605 OS << "*** Bad machine code: " << msg << " ***\n"
606 << "- function: " << MF->getName() << '\n';
607}
608
609void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
610 assert(MBB);
611 report(msg, MF: MBB->getParent());
612 OS << "- basic block: " << printMBBReference(MBB: *MBB) << ' ' << MBB->getName()
613 << " (" << (const void *)MBB << ')';
614 if (Indexes)
615 OS << " [" << Indexes->getMBBStartIdx(mbb: MBB) << ';'
616 << Indexes->getMBBEndIdx(mbb: MBB) << ')';
617 OS << '\n';
618}
619
620void MachineVerifier::report(const char *msg, const MachineInstr *MI) {
621 assert(MI);
622 report(msg, MBB: MI->getParent());
623 OS << "- instruction: ";
624 if (Indexes && Indexes->hasIndex(instr: *MI))
625 OS << Indexes->getInstructionIndex(MI: *MI) << '\t';
626 MI->print(OS, /*IsStandalone=*/true);
627}
628
629void MachineVerifier::report(const char *msg, const MachineOperand *MO,
630 unsigned MONum, LLT MOVRegType) {
631 assert(MO);
632 report(msg, MI: MO->getParent());
633 OS << "- operand " << MONum << ": ";
634 MO->print(os&: OS, TypeToPrint: MOVRegType, TRI);
635 OS << '\n';
636}
637
638void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) {
639 report(msg: Msg.str().c_str(), MI);
640}
641
642void MachineVerifier::report_context(SlotIndex Pos) const {
643 OS << "- at: " << Pos << '\n';
644}
645
646void MachineVerifier::report_context(const LiveInterval &LI) const {
647 OS << "- interval: " << LI << '\n';
648}
649
650void MachineVerifier::report_context(const LiveRange &LR,
651 VirtRegOrUnit VRegOrUnit,
652 LaneBitmask LaneMask) const {
653 report_context_liverange(LR);
654 report_context_vreg_regunit(VRegOrUnit);
655 if (LaneMask.any())
656 report_context_lanemask(LaneMask);
657}
658
659void MachineVerifier::report_context(const LiveRange::Segment &S) const {
660 OS << "- segment: " << S << '\n';
661}
662
663void MachineVerifier::report_context(const VNInfo &VNI) const {
664 OS << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n";
665}
666
667void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
668 OS << "- liverange: " << LR << '\n';
669}
670
671void MachineVerifier::report_context(MCPhysReg PReg) const {
672 OS << "- p. register: " << printReg(Reg: PReg, TRI) << '\n';
673}
674
675void MachineVerifier::report_context_vreg(Register VReg) const {
676 OS << "- v. register: " << printReg(Reg: VReg, TRI) << '\n';
677}
678
679void MachineVerifier::report_context_vreg_regunit(
680 VirtRegOrUnit VRegOrUnit) const {
681 if (VRegOrUnit.isVirtualReg()) {
682 report_context_vreg(VReg: VRegOrUnit.asVirtualReg());
683 } else {
684 OS << "- regunit: " << printRegUnit(Unit: VRegOrUnit.asMCRegUnit(), TRI)
685 << '\n';
686 }
687}
688
689void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {
690 OS << "- lanemask: " << PrintLaneMask(LaneMask) << '\n';
691}
692
693void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
694 BBInfo &MInfo = MBBInfoMap[MBB];
695 if (!MInfo.reachable) {
696 MInfo.reachable = true;
697 for (const MachineBasicBlock *Succ : MBB->successors())
698 markReachable(MBB: Succ);
699 }
700}
701
702void MachineVerifier::visitMachineFunctionBefore() {
703 lastIndex = SlotIndex();
704 regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()
705 : TRI->getReservedRegs(MF: *MF);
706
707 if (!MF->empty())
708 markReachable(MBB: &MF->front());
709
710 // Build a set of the basic blocks in the function.
711 FunctionBlocks.clear();
712 for (const auto &MBB : *MF) {
713 FunctionBlocks.insert(Ptr: &MBB);
714 BBInfo &MInfo = MBBInfoMap[&MBB];
715
716 MInfo.Preds.insert_range(R: MBB.predecessors());
717 if (MInfo.Preds.size() != MBB.pred_size())
718 report(msg: "MBB has duplicate entries in its predecessor list.", MBB: &MBB);
719
720 MInfo.Succs.insert_range(R: MBB.successors());
721 if (MInfo.Succs.size() != MBB.succ_size())
722 report(msg: "MBB has duplicate entries in its successor list.", MBB: &MBB);
723 }
724
725 // Check that the register use lists are sane.
726 MRI->verifyUseLists();
727
728 if (!MF->empty()) {
729 verifyStackFrame();
730 verifyStackProtector();
731 }
732}
733
734static bool hasPHIs(const MachineFunction &MF) {
735 return !MF.getProperties().hasNoPHIs() &&
736 any_of(Range: MF, P: [](const MachineBasicBlock &MBB) {
737 return !MBB.phis().empty();
738 });
739}
740
741void
742MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
743 FirstTerminator = nullptr;
744 FirstNonPHI = nullptr;
745
746 if (MRI->tracksLiveness() && hasPHIs(MF: *MF)) {
747 // If this block has allocatable physical registers live-in, check that
748 // it is an entry block or landing pad.
749 for (const auto &LI : MBB->liveins()) {
750 if (isAllocatable(Reg: LI.PhysReg) && !MBB->isEHPad() &&
751 MBB->getIterator() != MBB->getParent()->begin() &&
752 !MBB->isInlineAsmBrIndirectTarget()) {
753 report(msg: "MBB has allocatable live-in, but isn't entry, landing-pad, or "
754 "inlineasm-br-indirect-target.",
755 MBB);
756 report_context(PReg: LI.PhysReg);
757 }
758 }
759 }
760
761 if (MBB->isIRBlockAddressTaken()) {
762 if (!MBB->getAddressTakenIRBlock()->hasAddressTaken())
763 report(msg: "ir-block-address-taken is associated with basic block not used by "
764 "a blockaddress.",
765 MBB);
766 }
767
768 // Count the number of landing pad successors.
769 SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs;
770 for (const auto *succ : MBB->successors()) {
771 if (succ->isEHPad())
772 LandingPadSuccs.insert(Ptr: succ);
773 if (!FunctionBlocks.count(Ptr: succ))
774 report(msg: "MBB has successor that isn't part of the function.", MBB);
775 if (!MBBInfoMap[succ].Preds.count(Ptr: MBB)) {
776 report(msg: "Inconsistent CFG", MBB);
777 OS << "MBB is not in the predecessor list of the successor "
778 << printMBBReference(MBB: *succ) << ".\n";
779 }
780 }
781
782 // Check the predecessor list.
783 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
784 if (!FunctionBlocks.count(Ptr: Pred))
785 report(msg: "MBB has predecessor that isn't part of the function.", MBB);
786 if (!MBBInfoMap[Pred].Succs.count(Ptr: MBB)) {
787 report(msg: "Inconsistent CFG", MBB);
788 OS << "MBB is not in the successor list of the predecessor "
789 << printMBBReference(MBB: *Pred) << ".\n";
790 }
791 }
792
793 const MCAsmInfo &AsmInfo = TM->getMCAsmInfo();
794 const BasicBlock *BB = MBB->getBasicBlock();
795 const Function &F = MF->getFunction();
796 if (LandingPadSuccs.size() > 1 &&
797 !(AsmInfo.getExceptionHandlingType() == ExceptionHandling::SjLj && BB &&
798 isa<SwitchInst>(Val: BB->getTerminator())) &&
799 !isScopedEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
800 report(msg: "MBB has more than one landing pad successor", MBB);
801
802 // Call analyzeBranch. If it succeeds, there several more conditions to check.
803 const MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
804 SmallVector<MachineOperand, 4> Cond;
805 if (!TII->analyzeBranch(MBB: *MBB, TBB, FBB, Cond)) {
806 // Ok, analyzeBranch thinks it knows what's going on with this block. Let's
807 // check whether its answers match up with reality.
808 if (!TBB && !FBB) {
809 // Block falls through to its successor.
810 if (!MBB->empty() && MBB->back().isBarrier() &&
811 !TII->isPredicated(MI: MBB->back())) {
812 report(msg: "MBB exits via unconditional fall-through but ends with a "
813 "barrier instruction!", MBB);
814 }
815 if (!Cond.empty()) {
816 report(msg: "MBB exits via unconditional fall-through but has a condition!",
817 MBB);
818 }
819 } else if (TBB && !FBB && Cond.empty()) {
820 // Block unconditionally branches somewhere.
821 if (MBB->empty()) {
822 report(msg: "MBB exits via unconditional branch but doesn't contain "
823 "any instructions!", MBB);
824 } else if (!MBB->back().isBarrier()) {
825 report(msg: "MBB exits via unconditional branch but doesn't end with a "
826 "barrier instruction!", MBB);
827 } else if (!MBB->back().isTerminator()) {
828 report(msg: "MBB exits via unconditional branch but the branch isn't a "
829 "terminator instruction!", MBB);
830 }
831 } else if (TBB && !FBB && !Cond.empty()) {
832 // Block conditionally branches somewhere, otherwise falls through.
833 if (MBB->empty()) {
834 report(msg: "MBB exits via conditional branch/fall-through but doesn't "
835 "contain any instructions!", MBB);
836 } else if (MBB->back().isBarrier()) {
837 report(msg: "MBB exits via conditional branch/fall-through but ends with a "
838 "barrier instruction!", MBB);
839 } else if (!MBB->back().isTerminator()) {
840 report(msg: "MBB exits via conditional branch/fall-through but the branch "
841 "isn't a terminator instruction!", MBB);
842 }
843 } else if (TBB && FBB) {
844 // Block conditionally branches somewhere, otherwise branches
845 // somewhere else.
846 if (MBB->empty()) {
847 report(msg: "MBB exits via conditional branch/branch but doesn't "
848 "contain any instructions!", MBB);
849 } else if (!MBB->back().isBarrier()) {
850 report(msg: "MBB exits via conditional branch/branch but doesn't end with a "
851 "barrier instruction!", MBB);
852 } else if (!MBB->back().isTerminator()) {
853 report(msg: "MBB exits via conditional branch/branch but the branch "
854 "isn't a terminator instruction!", MBB);
855 }
856 if (Cond.empty()) {
857 report(msg: "MBB exits via conditional branch/branch but there's no "
858 "condition!", MBB);
859 }
860 } else {
861 report(msg: "analyzeBranch returned invalid data!", MBB);
862 }
863
864 // Now check that the successors match up with the answers reported by
865 // analyzeBranch.
866 if (TBB && !MBB->isSuccessor(MBB: TBB))
867 report(msg: "MBB exits via jump or conditional branch, but its target isn't a "
868 "CFG successor!",
869 MBB);
870 if (FBB && !MBB->isSuccessor(MBB: FBB))
871 report(msg: "MBB exits via conditional branch, but its target isn't a CFG "
872 "successor!",
873 MBB);
874
875 // There might be a fallthrough to the next block if there's either no
876 // unconditional true branch, or if there's a condition, and one of the
877 // branches is missing.
878 bool Fallthrough = !TBB || (!Cond.empty() && !FBB);
879
880 // A conditional fallthrough must be an actual CFG successor, not
881 // unreachable. (Conversely, an unconditional fallthrough might not really
882 // be a successor, because the block might end in unreachable.)
883 if (!Cond.empty() && !FBB) {
884 MachineFunction::const_iterator MBBI = std::next(x: MBB->getIterator());
885 if (MBBI == MF->end()) {
886 report(msg: "MBB conditionally falls through out of function!", MBB);
887 } else if (!MBB->isSuccessor(MBB: &*MBBI))
888 report(msg: "MBB exits via conditional branch/fall-through but the CFG "
889 "successors don't match the actual successors!",
890 MBB);
891 }
892
893 // Verify that there aren't any extra un-accounted-for successors.
894 for (const MachineBasicBlock *SuccMBB : MBB->successors()) {
895 // If this successor is one of the branch targets, it's okay.
896 if (SuccMBB == TBB || SuccMBB == FBB)
897 continue;
898 // If we might have a fallthrough, and the successor is the fallthrough
899 // block, that's also ok.
900 if (Fallthrough && SuccMBB == MBB->getNextNode())
901 continue;
902 // Also accept successors which are for exception-handling or might be
903 // inlineasm_br targets.
904 if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
905 continue;
906 report(msg: "MBB has unexpected successors which are not branch targets, "
907 "fallthrough, EHPads, or inlineasm_br targets.",
908 MBB);
909 }
910 }
911
912 regsLive.clear();
913 if (MRI->tracksLiveness()) {
914 for (const auto &LI : MBB->liveins()) {
915 if (!LI.PhysReg.isPhysical()) {
916 report(msg: "MBB live-in list contains non-physical register", MBB);
917 continue;
918 }
919 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: LI.PhysReg));
920 }
921 }
922
923 const MachineFrameInfo &MFI = MF->getFrameInfo();
924 BitVector PR = MFI.getPristineRegs(MF: *MF);
925 for (unsigned I : PR.set_bits())
926 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: I));
927
928 regsKilled.clear();
929 regsDefined.clear();
930
931 if (Indexes)
932 lastIndex = Indexes->getMBBStartIdx(mbb: MBB);
933}
934
935// This function gets called for all bundle headers, including normal
936// stand-alone unbundled instructions.
937void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
938 if (Indexes && Indexes->hasIndex(instr: *MI)) {
939 SlotIndex idx = Indexes->getInstructionIndex(MI: *MI);
940 if (!(idx > lastIndex)) {
941 report(msg: "Instruction index out of order", MI);
942 OS << "Last instruction was at " << lastIndex << '\n';
943 }
944 lastIndex = idx;
945 }
946
947 // Ensure non-terminators don't follow terminators.
948 if (MI->isTerminator()) {
949 if (!FirstTerminator)
950 FirstTerminator = MI;
951 } else if (FirstTerminator) {
952 // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to
953 // precede non-terminators.
954 if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) {
955 report(msg: "Non-terminator instruction after the first terminator", MI);
956 OS << "First terminator was:\t" << *FirstTerminator;
957 }
958 }
959}
960
961// The operands on an INLINEASM instruction must follow a template.
962// Verify that the flag operands make sense.
963void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
964 // The first two operands on INLINEASM are the asm string and global flags.
965 if (MI->getNumOperands() < 2) {
966 report(msg: "Too few operands on inline asm", MI);
967 return;
968 }
969 if (!MI->getOperand(i: 0).isSymbol())
970 report(msg: "Asm string must be an external symbol", MI);
971 if (!MI->getOperand(i: 1).isImm())
972 report(msg: "Asm flags must be an immediate", MI);
973 // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
974 // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,
975 // and Extra_IsConvergent = 32, Extra_MayUnwind = 64.
976 if (!isUInt<7>(x: MI->getOperand(i: 1).getImm()))
977 report(msg: "Unknown asm flags", MO: &MI->getOperand(i: 1), MONum: 1);
978
979 static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");
980
981 unsigned OpNo = InlineAsm::MIOp_FirstOperand;
982 unsigned NumOps;
983 for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
984 const MachineOperand &MO = MI->getOperand(i: OpNo);
985 // There may be implicit ops after the fixed operands.
986 if (!MO.isImm())
987 break;
988 const InlineAsm::Flag F(MO.getImm());
989 NumOps = 1 + F.getNumOperandRegisters();
990 }
991
992 if (OpNo > MI->getNumOperands())
993 report(msg: "Missing operands in last group", MI);
994
995 // An optional MDNode follows the groups.
996 if (OpNo < MI->getNumOperands() && MI->getOperand(i: OpNo).isMetadata())
997 ++OpNo;
998
999 // All trailing operands must be implicit registers.
1000 for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
1001 const MachineOperand &MO = MI->getOperand(i: OpNo);
1002 if (!MO.isReg() || !MO.isImplicit())
1003 report(msg: "Expected implicit register after groups", MO: &MO, MONum: OpNo);
1004 }
1005
1006 if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) {
1007 const MachineBasicBlock *MBB = MI->getParent();
1008
1009 for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands();
1010 i != e; ++i) {
1011 const MachineOperand &MO = MI->getOperand(i);
1012
1013 if (!MO.isMBB())
1014 continue;
1015
1016 // Check the successor & predecessor lists look ok, assume they are
1017 // not. Find the indirect target without going through the successors.
1018 const MachineBasicBlock *IndirectTargetMBB = MO.getMBB();
1019 if (!IndirectTargetMBB) {
1020 report(msg: "INLINEASM_BR indirect target does not exist", MO: &MO, MONum: i);
1021 break;
1022 }
1023
1024 if (!MBB->isSuccessor(MBB: IndirectTargetMBB))
1025 report(msg: "INLINEASM_BR indirect target missing from successor list", MO: &MO,
1026 MONum: i);
1027
1028 if (!IndirectTargetMBB->isPredecessor(MBB))
1029 report(msg: "INLINEASM_BR indirect target predecessor list missing parent",
1030 MO: &MO, MONum: i);
1031 }
1032 }
1033}
1034
1035bool MachineVerifier::verifyAllRegOpsScalar(const MachineInstr &MI,
1036 const MachineRegisterInfo &MRI) {
1037 if (none_of(Range: MI.explicit_operands(), P: [&MRI](const MachineOperand &Op) {
1038 if (!Op.isReg())
1039 return false;
1040 const auto Reg = Op.getReg();
1041 if (Reg.isPhysical())
1042 return false;
1043 return !MRI.getType(Reg).isScalar();
1044 }))
1045 return true;
1046 report(msg: "All register operands must have scalar types", MI: &MI);
1047 return false;
1048}
1049
1050/// Check that types are consistent when two operands need to have the same
1051/// number of vector elements.
1052/// \return true if the types are valid.
1053bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,
1054 const MachineInstr *MI) {
1055 if (Ty0.isVector() != Ty1.isVector()) {
1056 report(msg: "operand types must be all-vector or all-scalar", MI);
1057 // Generally we try to report as many issues as possible at once, but in
1058 // this case it's not clear what should we be comparing the size of the
1059 // scalar with: the size of the whole vector or its lane. Instead of
1060 // making an arbitrary choice and emitting not so helpful message, let's
1061 // avoid the extra noise and stop here.
1062 return false;
1063 }
1064
1065 if (Ty0.isVector() && Ty0.getElementCount() != Ty1.getElementCount()) {
1066 report(msg: "operand types must preserve number of vector elements", MI);
1067 return false;
1068 }
1069
1070 return true;
1071}
1072
1073bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
1074 auto Opcode = MI->getOpcode();
1075 bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
1076 Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
1077 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1078 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1079 AttributeSet Attrs = Intrinsic::getFnAttributes(
1080 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1081 bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
1082 if (NoSideEffects && DeclHasSideEffects) {
1083 report(Msg: Twine(TII->getName(Opcode),
1084 " used with intrinsic that accesses memory"),
1085 MI);
1086 return false;
1087 }
1088 if (!NoSideEffects && !DeclHasSideEffects) {
1089 report(Msg: Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI);
1090 return false;
1091 }
1092 }
1093
1094 return true;
1095}
1096
1097bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
1098 auto Opcode = MI->getOpcode();
1099 bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
1100 Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
1101 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1102 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1103 AttributeSet Attrs = Intrinsic::getFnAttributes(
1104 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1105 bool DeclIsConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
1106 if (NotConvergent && DeclIsConvergent) {
1107 report(Msg: Twine(TII->getName(Opcode), " used with a convergent intrinsic"),
1108 MI);
1109 return false;
1110 }
1111 if (!NotConvergent && !DeclIsConvergent) {
1112 report(
1113 Msg: Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"),
1114 MI);
1115 return false;
1116 }
1117 }
1118
1119 return true;
1120}
1121
1122void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
1123 if (isFunctionSelected)
1124 report(msg: "Unexpected generic instruction in a Selected function", MI);
1125
1126 const MCInstrDesc &MCID = MI->getDesc();
1127 unsigned NumOps = MI->getNumOperands();
1128
1129 // Branches must reference a basic block if they are not indirect
1130 if (MI->isBranch() && !MI->isIndirectBranch()) {
1131 bool HasMBB = false;
1132 for (const MachineOperand &Op : MI->operands()) {
1133 if (Op.isMBB()) {
1134 HasMBB = true;
1135 break;
1136 }
1137 }
1138
1139 if (!HasMBB) {
1140 report(msg: "Branch instruction is missing a basic block operand or "
1141 "isIndirectBranch property",
1142 MI);
1143 }
1144 }
1145
1146 // Check types.
1147 SmallVector<LLT, 4> Types;
1148 for (unsigned I = 0, E = std::min(a: MCID.getNumOperands(), b: NumOps);
1149 I != E; ++I) {
1150 if (!MCID.operands()[I].isGenericType())
1151 continue;
1152 // Generic instructions specify type equality constraints between some of
1153 // their operands. Make sure these are consistent.
1154 size_t TypeIdx = MCID.operands()[I].getGenericTypeIndex();
1155 Types.resize(N: std::max(a: TypeIdx + 1, b: Types.size()));
1156
1157 const MachineOperand *MO = &MI->getOperand(i: I);
1158 if (!MO->isReg()) {
1159 report(msg: "generic instruction must use register operands", MI);
1160 continue;
1161 }
1162
1163 LLT OpTy = MRI->getType(Reg: MO->getReg());
1164 // Don't report a type mismatch if there is no actual mismatch, only a
1165 // type missing, to reduce noise:
1166 if (OpTy.isValid()) {
1167 // Only the first valid type for a type index will be printed: don't
1168 // overwrite it later so it's always clear which type was expected:
1169 if (!Types[TypeIdx].isValid())
1170 Types[TypeIdx] = OpTy;
1171 else if (Types[TypeIdx] != OpTy)
1172 report(msg: "Type mismatch in generic instruction", MO, MONum: I, MOVRegType: OpTy);
1173 } else {
1174 // Generic instructions must have types attached to their operands.
1175 report(msg: "Generic instruction is missing a virtual register type", MO, MONum: I);
1176 }
1177 }
1178
1179 // Generic opcodes must not have physical register operands.
1180 for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
1181 const MachineOperand *MO = &MI->getOperand(i: I);
1182 if (MO->isReg() && MO->getReg().isPhysical())
1183 report(msg: "Generic instruction cannot have physical register", MO, MONum: I);
1184 }
1185
1186 // Avoid out of bounds in checks below. This was already reported earlier.
1187 if (MI->getNumOperands() < MCID.getNumOperands())
1188 return;
1189
1190 StringRef ErrorInfo;
1191 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
1192 report(msg: ErrorInfo.data(), MI);
1193
1194 // Verify properties of various specific instruction types
1195 unsigned Opc = MI->getOpcode();
1196 switch (Opc) {
1197 case TargetOpcode::G_ASSERT_SEXT:
1198 case TargetOpcode::G_ASSERT_ZEXT: {
1199 std::string OpcName =
1200 Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT";
1201 if (!MI->getOperand(i: 2).isImm()) {
1202 report(Msg: Twine(OpcName, " expects an immediate operand #2"), MI);
1203 break;
1204 }
1205
1206 Register Dst = MI->getOperand(i: 0).getReg();
1207 Register Src = MI->getOperand(i: 1).getReg();
1208 LLT SrcTy = MRI->getType(Reg: Src);
1209 int64_t Imm = MI->getOperand(i: 2).getImm();
1210 if (Imm <= 0) {
1211 report(Msg: Twine(OpcName, " size must be >= 1"), MI);
1212 break;
1213 }
1214
1215 if (Imm >= SrcTy.getScalarSizeInBits()) {
1216 report(Msg: Twine(OpcName, " size must be less than source bit width"), MI);
1217 break;
1218 }
1219
1220 const RegisterBank *SrcRB = RBI->getRegBank(Reg: Src, MRI: *MRI, TRI: *TRI);
1221 const RegisterBank *DstRB = RBI->getRegBank(Reg: Dst, MRI: *MRI, TRI: *TRI);
1222
1223 // Allow only the source bank to be set.
1224 if ((SrcRB && DstRB && SrcRB != DstRB) || (DstRB && !SrcRB)) {
1225 report(Msg: Twine(OpcName, " cannot change register bank"), MI);
1226 break;
1227 }
1228
1229 // Don't allow a class change. Do allow member class->regbank.
1230 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: Dst);
1231 if (DstRC && DstRC != MRI->getRegClassOrNull(Reg: Src)) {
1232 report(
1233 Msg: Twine(OpcName, " source and destination register classes must match"),
1234 MI);
1235 break;
1236 }
1237
1238 break;
1239 }
1240
1241 case TargetOpcode::G_CONSTANT:
1242 case TargetOpcode::G_FCONSTANT: {
1243 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1244 if (DstTy.isVector())
1245 report(msg: "Instruction cannot use a vector result type", MI);
1246
1247 if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {
1248 if (!MI->getOperand(i: 1).isCImm()) {
1249 report(msg: "G_CONSTANT operand must be cimm", MI);
1250 break;
1251 }
1252
1253 const ConstantInt *CI = MI->getOperand(i: 1).getCImm();
1254 if (CI->getBitWidth() != DstTy.getSizeInBits())
1255 report(msg: "inconsistent constant size", MI);
1256 } else {
1257 if (!MI->getOperand(i: 1).isFPImm()) {
1258 report(msg: "G_FCONSTANT operand must be fpimm", MI);
1259 break;
1260 }
1261 const ConstantFP *CF = MI->getOperand(i: 1).getFPImm();
1262
1263 if (APFloat::getSizeInBits(Sem: CF->getValueAPF().getSemantics()) !=
1264 DstTy.getSizeInBits()) {
1265 report(msg: "inconsistent constant size", MI);
1266 }
1267 }
1268
1269 break;
1270 }
1271 case TargetOpcode::G_LOAD:
1272 case TargetOpcode::G_STORE:
1273 case TargetOpcode::G_ZEXTLOAD:
1274 case TargetOpcode::G_SEXTLOAD:
1275 case TargetOpcode::G_FPEXTLOAD:
1276 case TargetOpcode::G_FPTRUNCSTORE: {
1277 LLT ValTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1278 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1279 if (!PtrTy.isPointer())
1280 report(msg: "Generic memory instruction must access a pointer", MI);
1281
1282 // Generic loads and stores must have a single MachineMemOperand
1283 // describing that access.
1284 if (!MI->hasOneMemOperand()) {
1285 report(msg: "Generic instruction accessing memory must have one mem operand",
1286 MI);
1287 } else {
1288 const MachineMemOperand &MMO = **MI->memoperands_begin();
1289 if (isa<GExtLoad>(Val: *MI)) {
1290 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1291 RHS: ValTy.getSizeInBits()))
1292 report(msg: "Generic extload must have a narrower memory type", MI);
1293 } else if (isa<GFPTruncStore>(Val: *MI)) {
1294 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1295 RHS: ValTy.getSizeInBits()))
1296 report(msg: "Generic truncstore must have a narrower memory type", MI);
1297 } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
1298 if (TypeSize::isKnownGT(LHS: MMO.getSize().getValue(),
1299 RHS: ValTy.getSizeInBytes()))
1300 report(msg: "load memory size cannot exceed result size", MI);
1301
1302 if (MMO.getRanges()) {
1303 ConstantInt *i =
1304 mdconst::extract<ConstantInt>(MD: MMO.getRanges()->getOperand(I: 0));
1305 const LLT RangeTy = LLT::scalar(SizeInBits: i->getIntegerType()->getBitWidth());
1306 const LLT MemTy = MMO.getMemoryType();
1307 if (MemTy.getScalarType() != RangeTy ||
1308 ValTy.isScalar() != MemTy.isScalar() ||
1309 (ValTy.isVector() &&
1310 ValTy.getNumElements() != MemTy.getNumElements())) {
1311 report(msg: "range is incompatible with the result type", MI);
1312 }
1313 }
1314 } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
1315 if (TypeSize::isKnownLT(LHS: ValTy.getSizeInBytes(),
1316 RHS: MMO.getSize().getValue()))
1317 report(msg: "store memory size cannot exceed value size", MI);
1318 }
1319
1320 const AtomicOrdering Order = MMO.getSuccessOrdering();
1321 if (isa<GAnyStore>(Val: *MI)) {
1322 if (Order == AtomicOrdering::Acquire ||
1323 Order == AtomicOrdering::AcquireRelease)
1324 report(msg: "atomic store cannot use acquire ordering", MI);
1325
1326 } else {
1327 if (Order == AtomicOrdering::Release ||
1328 Order == AtomicOrdering::AcquireRelease)
1329 report(msg: "atomic load cannot use release ordering", MI);
1330 }
1331 }
1332
1333 break;
1334 }
1335 case TargetOpcode::G_PHI: {
1336 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1337 if (!DstTy.isValid() || !all_of(Range: drop_begin(RangeOrContainer: MI->operands()),
1338 P: [this, &DstTy](const MachineOperand &MO) {
1339 if (!MO.isReg())
1340 return true;
1341 LLT Ty = MRI->getType(Reg: MO.getReg());
1342 if (!Ty.isValid() || (Ty != DstTy))
1343 return false;
1344 return true;
1345 }))
1346 report(msg: "Generic Instruction G_PHI has operands with incompatible/missing "
1347 "types",
1348 MI);
1349 break;
1350 }
1351 case TargetOpcode::G_BITCAST: {
1352 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1353 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1354 if (!DstTy.isValid() || !SrcTy.isValid())
1355 break;
1356
1357 if (SrcTy.isPointer() != DstTy.isPointer())
1358 report(msg: "bitcast cannot convert between pointers and other types", MI);
1359
1360 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1361 report(msg: "bitcast sizes must match", MI);
1362
1363 bool SameType = SrcTy.getKind() == DstTy.getKind();
1364 if (SameType && SrcTy.isPointerOrPointerVector())
1365 SameType &= SrcTy.getAddressSpace() == DstTy.getAddressSpace();
1366
1367 SameType &= SrcTy.getScalarSizeInBits() == DstTy.getScalarSizeInBits();
1368
1369 if (SameType && SrcTy.isVector())
1370 SameType &= SrcTy.getElementCount() == DstTy.getElementCount();
1371 if (SameType && SrcTy.isFloatOrFloatVector())
1372 SameType &= SrcTy.getFpSemantics() == DstTy.getFpSemantics();
1373
1374 if (SameType)
1375 report(msg: "bitcast must change the type", MI);
1376
1377 break;
1378 }
1379 case TargetOpcode::G_INTTOPTR:
1380 case TargetOpcode::G_PTRTOINT:
1381 case TargetOpcode::G_ADDRSPACE_CAST: {
1382 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1383 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1384 if (!DstTy.isValid() || !SrcTy.isValid())
1385 break;
1386
1387 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1388
1389 DstTy = DstTy.getScalarType();
1390 SrcTy = SrcTy.getScalarType();
1391
1392 if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {
1393 if (!DstTy.isPointer())
1394 report(msg: "inttoptr result type must be a pointer", MI);
1395 if (SrcTy.isPointer())
1396 report(msg: "inttoptr source type must not be a pointer", MI);
1397 } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {
1398 if (!SrcTy.isPointer())
1399 report(msg: "ptrtoint source type must be a pointer", MI);
1400 if (DstTy.isPointer())
1401 report(msg: "ptrtoint result type must not be a pointer", MI);
1402 } else {
1403 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
1404 if (!SrcTy.isPointer() || !DstTy.isPointer())
1405 report(msg: "addrspacecast types must be pointers", MI);
1406 else {
1407 if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())
1408 report(msg: "addrspacecast must convert different address spaces", MI);
1409 }
1410 }
1411
1412 break;
1413 }
1414 case TargetOpcode::G_PTR_ADD: {
1415 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1416 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1417 LLT OffsetTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1418 if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())
1419 break;
1420
1421 if (!PtrTy.isPointerOrPointerVector())
1422 report(msg: "gep first operand must be a pointer", MI);
1423
1424 if (OffsetTy.isPointerOrPointerVector())
1425 report(msg: "gep offset operand must not be a pointer", MI);
1426
1427 if (PtrTy.isPointerOrPointerVector()) {
1428 const DataLayout &DL = MF->getDataLayout();
1429 unsigned AS = PtrTy.getAddressSpace();
1430 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
1431 if (OffsetTy.getScalarSizeInBits() != IndexSizeInBits) {
1432 report(msg: "gep offset operand must match index size for address space",
1433 MI);
1434 }
1435 }
1436
1437 // TODO: Is the offset allowed to be a scalar with a vector?
1438 break;
1439 }
1440 case TargetOpcode::G_PTRMASK: {
1441 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1442 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1443 LLT MaskTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1444 if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid())
1445 break;
1446
1447 if (!DstTy.isPointerOrPointerVector())
1448 report(msg: "ptrmask result type must be a pointer", MI);
1449
1450 if (!MaskTy.getScalarType().isScalar())
1451 report(msg: "ptrmask mask type must be an integer", MI);
1452
1453 verifyVectorElementMatch(Ty0: DstTy, Ty1: MaskTy, MI);
1454 break;
1455 }
1456 case TargetOpcode::G_SEXT:
1457 case TargetOpcode::G_ZEXT:
1458 case TargetOpcode::G_ANYEXT:
1459 case TargetOpcode::G_TRUNC:
1460 case TargetOpcode::G_TRUNC_SSAT_S:
1461 case TargetOpcode::G_TRUNC_SSAT_U:
1462 case TargetOpcode::G_TRUNC_USAT_U:
1463 case TargetOpcode::G_FPEXT:
1464 case TargetOpcode::G_FPTRUNC: {
1465 // Number of operands and presense of types is already checked (and
1466 // reported in case of any issues), so no need to report them again. As
1467 // we're trying to report as many issues as possible at once, however, the
1468 // instructions aren't guaranteed to have the right number of operands or
1469 // types attached to them at this point
1470 assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
1471 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1472 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1473 if (!DstTy.isValid() || !SrcTy.isValid())
1474 break;
1475
1476 if (DstTy.isPointerOrPointerVector() || SrcTy.isPointerOrPointerVector())
1477 report(msg: "Generic extend/truncate can not operate on pointers", MI);
1478
1479 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1480
1481 unsigned DstSize = DstTy.getScalarSizeInBits();
1482 unsigned SrcSize = SrcTy.getScalarSizeInBits();
1483 switch (MI->getOpcode()) {
1484 default:
1485 if (DstSize <= SrcSize)
1486 report(msg: "Generic extend has destination type no larger than source", MI);
1487 break;
1488 case TargetOpcode::G_TRUNC:
1489 case TargetOpcode::G_TRUNC_SSAT_S:
1490 case TargetOpcode::G_TRUNC_SSAT_U:
1491 case TargetOpcode::G_TRUNC_USAT_U:
1492 case TargetOpcode::G_FPTRUNC:
1493 if (DstSize >= SrcSize)
1494 report(msg: "Generic truncate has destination type no smaller than source",
1495 MI);
1496 break;
1497 }
1498 break;
1499 }
1500 case TargetOpcode::G_SELECT: {
1501 LLT SelTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1502 LLT CondTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1503 if (!SelTy.isValid() || !CondTy.isValid())
1504 break;
1505
1506 // Scalar condition select on a vector is valid.
1507 if (CondTy.isVector())
1508 verifyVectorElementMatch(Ty0: SelTy, Ty1: CondTy, MI);
1509 break;
1510 }
1511 case TargetOpcode::G_MERGE_VALUES: {
1512 // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,
1513 // e.g. s2N = MERGE sN, sN
1514 // Merging multiple scalars into a vector is not allowed, should use
1515 // G_BUILD_VECTOR for that.
1516 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1517 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1518 if (DstTy.isVector() || SrcTy.isVector())
1519 report(msg: "G_MERGE_VALUES cannot operate on vectors", MI);
1520
1521 const unsigned NumOps = MI->getNumOperands();
1522 if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1))
1523 report(msg: "G_MERGE_VALUES result size is inconsistent", MI);
1524
1525 for (unsigned I = 2; I != NumOps; ++I) {
1526 if (MRI->getType(Reg: MI->getOperand(i: I).getReg()) != SrcTy)
1527 report(msg: "G_MERGE_VALUES source types do not match", MI);
1528 }
1529
1530 break;
1531 }
1532 case TargetOpcode::G_UNMERGE_VALUES: {
1533 unsigned NumDsts = MI->getNumOperands() - 1;
1534 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1535 for (unsigned i = 1; i < NumDsts; ++i) {
1536 if (MRI->getType(Reg: MI->getOperand(i).getReg()) != DstTy) {
1537 report(msg: "G_UNMERGE_VALUES destination types do not match", MI);
1538 break;
1539 }
1540 }
1541
1542 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: NumDsts).getReg());
1543 if (DstTy.isVector()) {
1544 // This case is the converse of G_CONCAT_VECTORS.
1545 if (!SrcTy.isVector() ||
1546 (SrcTy.getScalarType() != DstTy.getScalarType() &&
1547 !SrcTy.isPointerVector()) ||
1548 SrcTy.isScalableVector() != DstTy.isScalableVector() ||
1549 SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1550 report(msg: "G_UNMERGE_VALUES source operand does not match vector "
1551 "destination operands",
1552 MI);
1553 } else if (SrcTy.isVector()) {
1554 // This case is the converse of G_BUILD_VECTOR, but relaxed to allow
1555 // mismatched types as long as the total size matches:
1556 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<4 x s32>)
1557 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1558 report(msg: "G_UNMERGE_VALUES vector source operand does not match scalar "
1559 "destination operands",
1560 MI);
1561 } else {
1562 // This case is the converse of G_MERGE_VALUES.
1563 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits()) {
1564 report(msg: "G_UNMERGE_VALUES scalar source operand does not match scalar "
1565 "destination operands",
1566 MI);
1567 }
1568 }
1569 break;
1570 }
1571 case TargetOpcode::G_BUILD_VECTOR: {
1572 // Source types must be scalars, dest type a vector. Total size of scalars
1573 // must match the dest vector size.
1574 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1575 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1576 if (!DstTy.isVector() || SrcEltTy.isVector()) {
1577 report(msg: "G_BUILD_VECTOR must produce a vector from scalar operands", MI);
1578 break;
1579 }
1580
1581 if (DstTy.getElementType() != SrcEltTy)
1582 report(msg: "G_BUILD_VECTOR result element type must match source type", MI);
1583
1584 if (DstTy.getNumElements() != MI->getNumOperands() - 1)
1585 report(msg: "G_BUILD_VECTOR must have an operand for each element", MI);
1586
1587 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1588 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1589 report(msg: "G_BUILD_VECTOR source operand types are not homogeneous", MI);
1590
1591 break;
1592 }
1593 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1594 // Source types must be scalars, dest type a vector. Scalar types must be
1595 // larger than the dest vector elt type, as this is a truncating operation.
1596 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1597 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1598 if (!DstTy.isVector() || SrcEltTy.isVector())
1599 report(msg: "G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands",
1600 MI);
1601 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1602 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1603 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not homogeneous",
1604 MI);
1605 if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits())
1606 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not larger than "
1607 "dest elt type",
1608 MI);
1609 break;
1610 }
1611 case TargetOpcode::G_CONCAT_VECTORS: {
1612 // Source types should be vectors, and total size should match the dest
1613 // vector size.
1614 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1615 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1616 if (!DstTy.isVector() || !SrcTy.isVector())
1617 report(msg: "G_CONCAT_VECTOR requires vector source and destination operands",
1618 MI);
1619
1620 if (MI->getNumOperands() < 3)
1621 report(msg: "G_CONCAT_VECTOR requires at least 2 source operands", MI);
1622
1623 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1624 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1625 report(msg: "G_CONCAT_VECTOR source operand types are not homogeneous", MI);
1626 if (DstTy.getElementCount() !=
1627 SrcTy.getElementCount() * (MI->getNumOperands() - 1))
1628 report(msg: "G_CONCAT_VECTOR num dest and source elements should match", MI);
1629 break;
1630 }
1631 case TargetOpcode::G_ICMP:
1632 case TargetOpcode::G_FCMP: {
1633 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1634 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1635
1636 if ((DstTy.isVector() != SrcTy.isVector()) ||
1637 (DstTy.isVector() &&
1638 DstTy.getElementCount() != SrcTy.getElementCount()))
1639 report(msg: "Generic vector icmp/fcmp must preserve number of lanes", MI);
1640
1641 break;
1642 }
1643 case TargetOpcode::G_SCMP:
1644 case TargetOpcode::G_UCMP: {
1645 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1646 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1647
1648 if (SrcTy.isPointerOrPointerVector()) {
1649 report(msg: "Generic scmp/ucmp does not support pointers as operands", MI);
1650 break;
1651 }
1652
1653 if (DstTy.isPointerOrPointerVector()) {
1654 report(msg: "Generic scmp/ucmp does not support pointers as a result", MI);
1655 break;
1656 }
1657
1658 if (DstTy.getScalarSizeInBits() < 2) {
1659 report(msg: "Result type must be at least 2 bits wide", MI);
1660 break;
1661 }
1662
1663 if ((DstTy.isVector() != SrcTy.isVector()) ||
1664 (DstTy.isVector() &&
1665 DstTy.getElementCount() != SrcTy.getElementCount())) {
1666 report(msg: "Generic vector scmp/ucmp must preserve number of lanes", MI);
1667 break;
1668 }
1669
1670 break;
1671 }
1672 case TargetOpcode::G_EXTRACT: {
1673 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1674 if (!SrcOp.isReg()) {
1675 report(msg: "extract source must be a register", MI);
1676 break;
1677 }
1678
1679 const MachineOperand &OffsetOp = MI->getOperand(i: 2);
1680 if (!OffsetOp.isImm()) {
1681 report(msg: "extract offset must be a constant", MI);
1682 break;
1683 }
1684
1685 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1686 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1687 if (SrcSize == DstSize)
1688 report(msg: "extract source must be larger than result", MI);
1689
1690 if (DstSize + OffsetOp.getImm() > SrcSize)
1691 report(msg: "extract reads past end of register", MI);
1692 break;
1693 }
1694 case TargetOpcode::G_INSERT: {
1695 const MachineOperand &SrcOp = MI->getOperand(i: 2);
1696 if (!SrcOp.isReg()) {
1697 report(msg: "insert source must be a register", MI);
1698 break;
1699 }
1700
1701 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
1702 if (!OffsetOp.isImm()) {
1703 report(msg: "insert offset must be a constant", MI);
1704 break;
1705 }
1706
1707 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1708 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1709
1710 if (DstSize <= SrcSize)
1711 report(msg: "inserted size must be smaller than total register", MI);
1712
1713 if (SrcSize + OffsetOp.getImm() > DstSize)
1714 report(msg: "insert writes past end of register", MI);
1715
1716 break;
1717 }
1718 case TargetOpcode::G_JUMP_TABLE: {
1719 if (!MI->getOperand(i: 1).isJTI())
1720 report(msg: "G_JUMP_TABLE source operand must be a jump table index", MI);
1721 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1722 if (!DstTy.isPointer())
1723 report(msg: "G_JUMP_TABLE dest operand must have a pointer type", MI);
1724 break;
1725 }
1726 case TargetOpcode::G_BRJT: {
1727 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
1728 report(msg: "G_BRJT src operand 0 must be a pointer type", MI);
1729
1730 if (!MI->getOperand(i: 1).isJTI())
1731 report(msg: "G_BRJT src operand 1 must be a jump table index", MI);
1732
1733 const auto &IdxOp = MI->getOperand(i: 2);
1734 if (!IdxOp.isReg() || MRI->getType(Reg: IdxOp.getReg()).isPointer())
1735 report(msg: "G_BRJT src operand 2 must be a scalar reg type", MI);
1736 break;
1737 }
1738 case TargetOpcode::G_INTRINSIC:
1739 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1740 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1741 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
1742 // TODO: Should verify number of def and use operands, but the current
1743 // interface requires passing in IR types for mangling.
1744 const MachineOperand &IntrIDOp = MI->getOperand(i: MI->getNumExplicitDefs());
1745 if (!IntrIDOp.isIntrinsicID()) {
1746 report(msg: "G_INTRINSIC first src operand must be an intrinsic ID", MI);
1747 break;
1748 }
1749
1750 if (!verifyGIntrinsicSideEffects(MI))
1751 break;
1752 if (!verifyGIntrinsicConvergence(MI))
1753 break;
1754
1755 break;
1756 }
1757 case TargetOpcode::G_SEXT_INREG: {
1758 if (!MI->getOperand(i: 2).isImm()) {
1759 report(msg: "G_SEXT_INREG expects an immediate operand #2", MI);
1760 break;
1761 }
1762
1763 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1764 int64_t Imm = MI->getOperand(i: 2).getImm();
1765 if (Imm <= 0)
1766 report(msg: "G_SEXT_INREG size must be >= 1", MI);
1767 if (Imm >= SrcTy.getScalarSizeInBits())
1768 report(msg: "G_SEXT_INREG size must be less than source bit width", MI);
1769 break;
1770 }
1771 case TargetOpcode::G_BSWAP: {
1772 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1773 if (DstTy.getScalarSizeInBits() % 16 != 0)
1774 report(msg: "G_BSWAP size must be a multiple of 16 bits", MI);
1775 break;
1776 }
1777 case TargetOpcode::G_VSCALE: {
1778 if (!MI->getOperand(i: 1).isCImm()) {
1779 report(msg: "G_VSCALE operand must be cimm", MI);
1780 break;
1781 }
1782 if (MI->getOperand(i: 1).getCImm()->isZero()) {
1783 report(msg: "G_VSCALE immediate cannot be zero", MI);
1784 break;
1785 }
1786 break;
1787 }
1788 case TargetOpcode::G_STEP_VECTOR: {
1789 if (!MI->getOperand(i: 1).isCImm()) {
1790 report(msg: "operand must be cimm", MI);
1791 break;
1792 }
1793
1794 if (!MI->getOperand(i: 1).getCImm()->getValue().isStrictlyPositive()) {
1795 report(msg: "step must be > 0", MI);
1796 break;
1797 }
1798
1799 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1800 if (!DstTy.isScalableVector()) {
1801 report(msg: "Destination type must be a scalable vector", MI);
1802 break;
1803 }
1804
1805 // <vscale x 2 x p0>
1806 if (!DstTy.getElementType().isScalar()) {
1807 report(msg: "Destination element type must be scalar", MI);
1808 break;
1809 }
1810
1811 if (MI->getOperand(i: 1).getCImm()->getBitWidth() !=
1812 DstTy.getElementType().getScalarSizeInBits()) {
1813 report(msg: "step bitwidth differs from result type element bitwidth", MI);
1814 break;
1815 }
1816 break;
1817 }
1818 case TargetOpcode::G_INSERT_SUBVECTOR: {
1819 const MachineOperand &Src0Op = MI->getOperand(i: 1);
1820 if (!Src0Op.isReg()) {
1821 report(msg: "G_INSERT_SUBVECTOR first source must be a register", MI);
1822 break;
1823 }
1824
1825 const MachineOperand &Src1Op = MI->getOperand(i: 2);
1826 if (!Src1Op.isReg()) {
1827 report(msg: "G_INSERT_SUBVECTOR second source must be a register", MI);
1828 break;
1829 }
1830
1831 const MachineOperand &IndexOp = MI->getOperand(i: 3);
1832 if (!IndexOp.isImm()) {
1833 report(msg: "G_INSERT_SUBVECTOR index must be an immediate", MI);
1834 break;
1835 }
1836
1837 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1838 LLT Src1Ty = MRI->getType(Reg: Src1Op.getReg());
1839
1840 if (!DstTy.isVector()) {
1841 report(msg: "Destination type must be a vector", MI);
1842 break;
1843 }
1844
1845 if (!Src1Ty.isVector()) {
1846 report(msg: "Second source must be a vector", MI);
1847 break;
1848 }
1849
1850 if (DstTy.getElementType() != Src1Ty.getElementType()) {
1851 report(msg: "Element type of vectors must be the same", MI);
1852 break;
1853 }
1854
1855 if (!DstTy.isScalable() && Src1Ty.isScalable()) {
1856 report(msg: "Cannot insert a scalable vector into a fixed length vector", MI);
1857 break;
1858 }
1859
1860 bool IsMixedFixedIntoScalable =
1861 DstTy.isScalableVector() && Src1Ty.isFixedVector();
1862
1863 if (!IsMixedFixedIntoScalable &&
1864 ElementCount::isKnownGT(LHS: Src1Ty.getElementCount(),
1865 RHS: DstTy.getElementCount())) {
1866 report(msg: "Second source must be smaller than destination vector", MI);
1867 break;
1868 }
1869
1870 uint64_t Idx = IndexOp.getImm();
1871 uint64_t Src1MinLen = Src1Ty.getElementCount().getKnownMinValue();
1872 if (IndexOp.getImm() % Src1MinLen != 0) {
1873 report(msg: "Index must be a multiple of the second source vector's "
1874 "minimum vector length",
1875 MI);
1876 break;
1877 }
1878
1879 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1880 if (Idx >= DstMinLen ||
1881 (!IsMixedFixedIntoScalable && Idx + Src1MinLen > DstMinLen)) {
1882 report(msg: "Subvector type and index must not cause insert to overrun the "
1883 "vector being inserted into",
1884 MI);
1885 break;
1886 }
1887
1888 break;
1889 }
1890 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1891 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1892 if (!SrcOp.isReg()) {
1893 report(msg: "G_EXTRACT_SUBVECTOR first source must be a register", MI);
1894 break;
1895 }
1896
1897 const MachineOperand &IndexOp = MI->getOperand(i: 2);
1898 if (!IndexOp.isImm()) {
1899 report(msg: "G_EXTRACT_SUBVECTOR index must be an immediate", MI);
1900 break;
1901 }
1902
1903 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1904 LLT SrcTy = MRI->getType(Reg: SrcOp.getReg());
1905
1906 if (!DstTy.isVector()) {
1907 report(msg: "Destination type must be a vector", MI);
1908 break;
1909 }
1910
1911 if (!SrcTy.isVector()) {
1912 report(msg: "Source must be a vector", MI);
1913 break;
1914 }
1915
1916 if (DstTy.getElementType() != SrcTy.getElementType()) {
1917 report(msg: "Element type of vectors must be the same", MI);
1918 break;
1919 }
1920
1921 if (DstTy.isScalable() && !SrcTy.isScalable()) {
1922 report(msg: "Cannot extract a scalable vector from a fixed length vector", MI);
1923 break;
1924 }
1925
1926 if (ElementCount::isKnownGT(LHS: DstTy.getElementCount(),
1927 RHS: SrcTy.getElementCount())) {
1928 report(msg: "Destination vector must be smaller than source vector", MI);
1929 break;
1930 }
1931
1932 uint64_t Idx = IndexOp.getImm();
1933 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1934 if (Idx % DstMinLen != 0) {
1935 report(msg: "Index must be a multiple of the destination vector's minimum "
1936 "vector length",
1937 MI);
1938 break;
1939 }
1940
1941 bool IsMixedFixedFromScalable =
1942 DstTy.isFixedVector() && SrcTy.isScalableVector();
1943 uint64_t SrcMinLen = SrcTy.getElementCount().getKnownMinValue();
1944 if (Idx >= SrcMinLen ||
1945 (!IsMixedFixedFromScalable && Idx + DstMinLen > SrcMinLen)) {
1946 report(msg: "Destination type and index must not cause extract to overrun the "
1947 "source vector",
1948 MI);
1949 break;
1950 }
1951
1952 break;
1953 }
1954 case TargetOpcode::G_SHUFFLE_VECTOR: {
1955 const MachineOperand &MaskOp = MI->getOperand(i: 3);
1956 if (!MaskOp.isShuffleMask()) {
1957 report(msg: "Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);
1958 break;
1959 }
1960
1961 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1962 LLT Src0Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1963 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1964
1965 if (Src0Ty != Src1Ty)
1966 report(msg: "Source operands must be the same type", MI);
1967
1968 if (Src0Ty.getScalarType() != DstTy.getScalarType()) {
1969 report(msg: "G_SHUFFLE_VECTOR cannot change element type", MI);
1970 break;
1971 }
1972 if (!Src0Ty.isVector()) {
1973 report(msg: "G_SHUFFLE_VECTOR must have vector src", MI);
1974 break;
1975 }
1976 if (!DstTy.isVector()) {
1977 report(msg: "G_SHUFFLE_VECTOR must have vector dst", MI);
1978 break;
1979 }
1980
1981 // Don't check that all operands are vector because scalars are used in
1982 // place of 1 element vectors.
1983 int SrcNumElts = Src0Ty.getNumElements();
1984 int DstNumElts = DstTy.getNumElements();
1985
1986 ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();
1987
1988 if (static_cast<int>(MaskIdxes.size()) != DstNumElts)
1989 report(msg: "Wrong result type for shufflemask", MI);
1990
1991 for (int Idx : MaskIdxes) {
1992 if (Idx < 0)
1993 continue;
1994
1995 if (Idx >= 2 * SrcNumElts)
1996 report(msg: "Out of bounds shuffle index", MI);
1997 }
1998
1999 break;
2000 }
2001
2002 case TargetOpcode::G_SPLAT_VECTOR: {
2003 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2004 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2005
2006 if (!DstTy.isScalableVector()) {
2007 report(msg: "Destination type must be a scalable vector", MI);
2008 break;
2009 }
2010
2011 if (!SrcTy.isScalar() && !SrcTy.isPointer()) {
2012 report(msg: "Source type must be a scalar or pointer", MI);
2013 break;
2014 }
2015
2016 if (TypeSize::isKnownGT(LHS: DstTy.getElementType().getSizeInBits(),
2017 RHS: SrcTy.getSizeInBits())) {
2018 report(msg: "Element type of the destination must be the same size or smaller "
2019 "than the source type",
2020 MI);
2021 break;
2022 }
2023
2024 break;
2025 }
2026 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2027 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2028 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2029 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2030
2031 if (!DstTy.isScalar() && !DstTy.isPointer()) {
2032 report(msg: "Destination type must be a scalar or pointer", MI);
2033 break;
2034 }
2035
2036 if (!SrcTy.isVector()) {
2037 report(msg: "First source must be a vector", MI);
2038 break;
2039 }
2040
2041 auto TLI = MF->getSubtarget().getTargetLowering();
2042 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2043 report(msg: "Index type must match VectorIdxTy", MI);
2044 break;
2045 }
2046
2047 break;
2048 }
2049 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2050 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2051 LLT VecTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2052 LLT ScaTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2053 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 3).getReg());
2054
2055 if (!DstTy.isVector()) {
2056 report(msg: "Destination type must be a vector", MI);
2057 break;
2058 }
2059
2060 if (VecTy != DstTy) {
2061 report(msg: "Destination type and vector type must match", MI);
2062 break;
2063 }
2064
2065 if (!ScaTy.isScalar() && !ScaTy.isPointer()) {
2066 report(msg: "Inserted element must be a scalar or pointer", MI);
2067 break;
2068 }
2069
2070 auto TLI = MF->getSubtarget().getTargetLowering();
2071 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2072 report(msg: "Index type must match VectorIdxTy", MI);
2073 break;
2074 }
2075
2076 break;
2077 }
2078 case TargetOpcode::G_DYN_STACKALLOC: {
2079 const MachineOperand &DstOp = MI->getOperand(i: 0);
2080 const MachineOperand &AllocOp = MI->getOperand(i: 1);
2081 const MachineOperand &AlignOp = MI->getOperand(i: 2);
2082
2083 if (!DstOp.isReg() || !MRI->getType(Reg: DstOp.getReg()).isPointer()) {
2084 report(msg: "dst operand 0 must be a pointer type", MI);
2085 break;
2086 }
2087
2088 if (!AllocOp.isReg() || !MRI->getType(Reg: AllocOp.getReg()).isScalar()) {
2089 report(msg: "src operand 1 must be a scalar reg type", MI);
2090 break;
2091 }
2092
2093 if (!AlignOp.isImm()) {
2094 report(msg: "src operand 2 must be an immediate type", MI);
2095 break;
2096 }
2097 break;
2098 }
2099 case TargetOpcode::G_MEMCPY_INLINE:
2100 case TargetOpcode::G_MEMCPY:
2101 case TargetOpcode::G_MEMMOVE: {
2102 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2103 if (MMOs.size() != 2) {
2104 report(msg: "memcpy/memmove must have 2 memory operands", MI);
2105 break;
2106 }
2107
2108 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) ||
2109 (MMOs[1]->isStore() || !MMOs[1]->isLoad())) {
2110 report(msg: "wrong memory operand types", MI);
2111 break;
2112 }
2113
2114 if (MMOs[0]->getSize() != MMOs[1]->getSize())
2115 report(msg: "inconsistent memory operand sizes", MI);
2116
2117 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2118 LLT SrcPtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2119
2120 if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) {
2121 report(msg: "memory instruction operand must be a pointer", MI);
2122 break;
2123 }
2124
2125 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2126 report(msg: "inconsistent store address space", MI);
2127 if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())
2128 report(msg: "inconsistent load address space", MI);
2129
2130 if (Opc != TargetOpcode::G_MEMCPY_INLINE)
2131 if (!MI->getOperand(i: 3).isImm() || (MI->getOperand(i: 3).getImm() & ~1LL))
2132 report(msg: "'tail' flag (operand 3) must be an immediate 0 or 1", MI);
2133
2134 break;
2135 }
2136 case TargetOpcode::G_BZERO:
2137 case TargetOpcode::G_MEMSET:
2138 case TargetOpcode::G_MEMSET_INLINE: {
2139 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2140 std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset"
2141 : Opc == TargetOpcode::G_MEMSET_INLINE ? "memset_inline"
2142 : "bzero";
2143 if (MMOs.size() != 1) {
2144 report(Msg: Twine(Name, " must have 1 memory operand"), MI);
2145 break;
2146 }
2147
2148 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
2149 report(Msg: Twine(Name, " memory operand must be a store"), MI);
2150 break;
2151 }
2152
2153 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2154 if (!DstPtrTy.isPointer()) {
2155 report(Msg: Twine(Name, " operand must be a pointer"), MI);
2156 break;
2157 }
2158
2159 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2160 report(Msg: "inconsistent " + Twine(Name, " address space"), MI);
2161
2162 if (Opc != TargetOpcode::G_MEMSET_INLINE) {
2163 if (!MI->getOperand(i: MI->getNumOperands() - 1).isImm() ||
2164 (MI->getOperand(i: MI->getNumOperands() - 1).getImm() & ~1LL))
2165 report(msg: "'tail' flag (last operand) must be an immediate 0 or 1", MI);
2166 }
2167
2168 break;
2169 }
2170 case TargetOpcode::G_UBSANTRAP: {
2171 const MachineOperand &KindOp = MI->getOperand(i: 0);
2172 if (!MI->getOperand(i: 0).isImm()) {
2173 report(msg: "Crash kind must be an immediate", MO: &KindOp, MONum: 0);
2174 break;
2175 }
2176 int64_t Kind = MI->getOperand(i: 0).getImm();
2177 if (!isInt<8>(x: Kind))
2178 report(msg: "Crash kind must be 8 bit wide", MO: &KindOp, MONum: 0);
2179 break;
2180 }
2181 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
2182 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
2183 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2184 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2185 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2186 if (!DstTy.isScalar())
2187 report(msg: "Vector reduction requires a scalar destination type", MI);
2188 if (!Src1Ty.isScalar())
2189 report(msg: "Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI);
2190 if (!Src2Ty.isVector())
2191 report(msg: "Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);
2192 break;
2193 }
2194 case TargetOpcode::G_VECREDUCE_FADD:
2195 case TargetOpcode::G_VECREDUCE_FMUL:
2196 case TargetOpcode::G_VECREDUCE_FMAX:
2197 case TargetOpcode::G_VECREDUCE_FMIN:
2198 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
2199 case TargetOpcode::G_VECREDUCE_FMINIMUM:
2200 case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
2201 case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
2202 case TargetOpcode::G_VECREDUCE_ADD:
2203 case TargetOpcode::G_VECREDUCE_MUL:
2204 case TargetOpcode::G_VECREDUCE_AND:
2205 case TargetOpcode::G_VECREDUCE_OR:
2206 case TargetOpcode::G_VECREDUCE_XOR:
2207 case TargetOpcode::G_VECREDUCE_SMAX:
2208 case TargetOpcode::G_VECREDUCE_SMIN:
2209 case TargetOpcode::G_VECREDUCE_UMAX:
2210 case TargetOpcode::G_VECREDUCE_UMIN: {
2211 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2212 if (!DstTy.isScalar())
2213 report(msg: "Vector reduction requires a scalar destination type", MI);
2214 break;
2215 }
2216
2217 case TargetOpcode::G_SBFX:
2218 case TargetOpcode::G_UBFX: {
2219 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2220 if (DstTy.isVector()) {
2221 report(msg: "Bitfield extraction is not supported on vectors", MI);
2222 break;
2223 }
2224 break;
2225 }
2226 case TargetOpcode::G_SHL:
2227 case TargetOpcode::G_LSHR:
2228 case TargetOpcode::G_ASHR:
2229 case TargetOpcode::G_ROTR:
2230 case TargetOpcode::G_ROTL: {
2231 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2232 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2233 if (Src1Ty.isVector() != Src2Ty.isVector()) {
2234 report(msg: "Shifts and rotates require operands to be either all scalars or "
2235 "all vectors",
2236 MI);
2237 break;
2238 }
2239 break;
2240 }
2241 case TargetOpcode::G_LLROUND:
2242 case TargetOpcode::G_LROUND: {
2243 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2244 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2245 if (!DstTy.isValid() || !SrcTy.isValid())
2246 break;
2247 if (SrcTy.isPointer() || DstTy.isPointer()) {
2248 StringRef Op = SrcTy.isPointer() ? "Source" : "Destination";
2249 report(Msg: Twine(Op, " operand must not be a pointer type"), MI);
2250 } else if (SrcTy.isScalar()) {
2251 verifyAllRegOpsScalar(MI: *MI, MRI: *MRI);
2252 break;
2253 } else if (SrcTy.isVector()) {
2254 verifyVectorElementMatch(Ty0: SrcTy, Ty1: DstTy, MI);
2255 break;
2256 }
2257 break;
2258 }
2259 case TargetOpcode::G_IS_FPCLASS: {
2260 LLT DestTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2261 LLT DestEltTy = DestTy.getScalarType();
2262 if (!DestEltTy.isScalar()) {
2263 report(msg: "Destination must be a scalar or vector of scalars", MI);
2264 break;
2265 }
2266 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2267 LLT SrcEltTy = SrcTy.getScalarType();
2268 if (!SrcEltTy.isScalar()) {
2269 report(msg: "Source must be a scalar or vector of scalars", MI);
2270 break;
2271 }
2272 if (!verifyVectorElementMatch(Ty0: DestTy, Ty1: SrcTy, MI))
2273 break;
2274 const MachineOperand &TestMO = MI->getOperand(i: 2);
2275 if (!TestMO.isImm()) {
2276 report(msg: "floating-point class set (operand 2) must be an immediate", MI);
2277 break;
2278 }
2279 int64_t Test = TestMO.getImm();
2280 if (Test < 0 || Test > fcAllFlags) {
2281 report(msg: "Incorrect floating-point class set (operand 2)", MI);
2282 break;
2283 }
2284 break;
2285 }
2286 case TargetOpcode::G_PREFETCH: {
2287 const MachineOperand &AddrOp = MI->getOperand(i: 0);
2288 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer()) {
2289 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 0);
2290 break;
2291 }
2292 const MachineOperand &RWOp = MI->getOperand(i: 1);
2293 if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {
2294 report(msg: "rw operand must be an immediate 0-1", MO: &RWOp, MONum: 1);
2295 break;
2296 }
2297 const MachineOperand &LocalityOp = MI->getOperand(i: 2);
2298 if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {
2299 report(msg: "locality operand must be an immediate 0-3", MO: &LocalityOp, MONum: 2);
2300 break;
2301 }
2302 const MachineOperand &CacheTypeOp = MI->getOperand(i: 3);
2303 if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {
2304 report(msg: "cache type operand must be an immediate 0-1", MO: &CacheTypeOp, MONum: 3);
2305 break;
2306 }
2307 break;
2308 }
2309 case TargetOpcode::G_ASSERT_ALIGN: {
2310 if (MI->getOperand(i: 2).getImm() < 1)
2311 report(msg: "alignment immediate must be >= 1", MI);
2312 break;
2313 }
2314 case TargetOpcode::G_CONSTANT_POOL: {
2315 if (!MI->getOperand(i: 1).isCPI())
2316 report(msg: "Src operand 1 must be a constant pool index", MI);
2317 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
2318 report(msg: "Dst operand 0 must be a pointer", MI);
2319 break;
2320 }
2321 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {
2322 const MachineOperand &AddrOp = MI->getOperand(i: 1);
2323 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer())
2324 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 1);
2325 break;
2326 }
2327 case TargetOpcode::G_SMIN:
2328 case TargetOpcode::G_SMAX:
2329 case TargetOpcode::G_UMIN:
2330 case TargetOpcode::G_UMAX: {
2331 const LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2332 if (DstTy.isPointerOrPointerVector())
2333 report(msg: "Generic smin/smax/umin/umax does not support pointer operands",
2334 MI);
2335 break;
2336 }
2337 default:
2338 break;
2339 }
2340}
2341
2342void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
2343 const MCInstrDesc &MCID = MI->getDesc();
2344 if (MI->getNumOperands() < MCID.getNumOperands()) {
2345 report(msg: "Too few operands", MI);
2346 OS << MCID.getNumOperands() << " operands expected, but "
2347 << MI->getNumOperands() << " given.\n";
2348 }
2349
2350 if (MI->getFlag(Flag: MachineInstr::NoConvergent) && !MCID.isConvergent())
2351 report(msg: "NoConvergent flag expected only on convergent instructions.", MI);
2352
2353 if (MI->isPHI()) {
2354 if (MF->getProperties().hasNoPHIs())
2355 report(msg: "Found PHI instruction with NoPHIs property set", MI);
2356
2357 if (FirstNonPHI)
2358 report(msg: "Found PHI instruction after non-PHI", MI);
2359 } else if (FirstNonPHI == nullptr)
2360 FirstNonPHI = MI;
2361
2362 // Check the tied operands.
2363 if (MI->isInlineAsm())
2364 verifyInlineAsm(MI);
2365
2366 // Check that unspillable terminators define a reg and have at most one use.
2367 if (TII->isUnspillableTerminator(MI)) {
2368 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
2369 report(msg: "Unspillable Terminator does not define a reg", MI);
2370 Register Def = MI->getOperand(i: 0).getReg();
2371 if (Def.isVirtual() && hasPHIs(MF: *MF) &&
2372 std::distance(first: MRI->use_nodbg_begin(RegNo: Def), last: MRI->use_nodbg_end()) > 1)
2373 report(msg: "Unspillable Terminator expected to have at most one use!", MI);
2374 }
2375
2376 // A fully-formed DBG_VALUE must have a location. Ignore partially formed
2377 // DBG_VALUEs: these are convenient to use in tests, but should never get
2378 // generated.
2379 if (MI->isDebugValue() && MI->getNumOperands() == 4)
2380 if (!MI->getDebugLoc())
2381 report(msg: "Missing DebugLoc for debug instruction", MI);
2382
2383 // Meta instructions should never be the subject of debug value tracking,
2384 // they don't create a value in the output program at all.
2385 if (MI->isMetaInstruction() && MI->peekDebugInstrNum())
2386 report(msg: "Metadata instruction should not have a value tracking number", MI);
2387
2388 // Check the MachineMemOperands for basic consistency.
2389 for (MachineMemOperand *Op : MI->memoperands()) {
2390 if (Op->isLoad() && !MI->mayLoad())
2391 report(msg: "Missing mayLoad flag", MI);
2392 if (Op->isStore() && !MI->mayStore())
2393 report(msg: "Missing mayStore flag", MI);
2394 }
2395
2396 // Debug values must not have a slot index.
2397 // Other instructions must have one, unless they are inside a bundle.
2398 if (LiveInts) {
2399 bool mapped = !LiveInts->isNotInMIMap(Instr: *MI);
2400 if (MI->isDebugOrPseudoInstr()) {
2401 if (mapped)
2402 report(msg: "Debug instruction has a slot index", MI);
2403 } else if (MI->isInsideBundle()) {
2404 if (mapped)
2405 report(msg: "Instruction inside bundle has a slot index", MI);
2406 } else {
2407 if (!mapped)
2408 report(msg: "Missing slot index", MI);
2409 }
2410 }
2411
2412 unsigned Opc = MCID.getOpcode();
2413 if (isPreISelGenericOpcode(Opcode: Opc) || isPreISelGenericOptimizationHint(Opcode: Opc)) {
2414 verifyPreISelGenericInstruction(MI);
2415 return;
2416 }
2417
2418 StringRef ErrorInfo;
2419 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
2420 report(msg: ErrorInfo.data(), MI);
2421
2422 // Verify properties of various specific instruction types
2423 switch (MI->getOpcode()) {
2424 case TargetOpcode::COPY: {
2425 const MachineOperand &DstOp = MI->getOperand(i: 0);
2426 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2427 const Register SrcReg = SrcOp.getReg();
2428 const Register DstReg = DstOp.getReg();
2429
2430 LLT DstTy = MRI->getType(Reg: DstReg);
2431 LLT SrcTy = MRI->getType(Reg: SrcReg);
2432 if (SrcTy.isValid() && DstTy.isValid()) {
2433 // If both types are valid, check that the types are the same.
2434 if (SrcTy != DstTy) {
2435 report(msg: "Copy Instruction is illegal with mismatching types", MI);
2436 OS << "Def = " << DstTy << ", Src = " << SrcTy << '\n';
2437 }
2438
2439 break;
2440 }
2441
2442 if (!SrcTy.isValid() && !DstTy.isValid())
2443 break;
2444
2445 // If we have only one valid type, this is likely a copy between a virtual
2446 // and physical register.
2447 TypeSize SrcSize = TypeSize::getZero();
2448 TypeSize DstSize = TypeSize::getZero();
2449 if (SrcReg.isPhysical() && DstTy.isValid()) {
2450 if (!hasPhysRegClassForType(TRI: *TRI, Reg: SrcReg, Ty: DstTy))
2451 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2452 } else {
2453 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2454 }
2455
2456 if (DstReg.isPhysical() && SrcTy.isValid()) {
2457 if (!hasPhysRegClassForType(TRI: *TRI, Reg: DstReg, Ty: SrcTy))
2458 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2459 } else {
2460 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2461 }
2462
2463 // The next two checks allow COPY between physical and virtual registers,
2464 // when the virtual register has a scalable size and the physical register
2465 // has a fixed size. These checks allow COPY between *potentially*
2466 // mismatched sizes. However, once RegisterBankSelection occurs,
2467 // MachineVerifier should be able to resolve a fixed size for the scalable
2468 // vector, and at that point this function will know for sure whether the
2469 // sizes are mismatched and correctly report a size mismatch.
2470 if (SrcReg.isPhysical() && DstReg.isVirtual() && DstSize.isScalable() &&
2471 !SrcSize.isScalable())
2472 break;
2473 if (SrcReg.isVirtual() && DstReg.isPhysical() && SrcSize.isScalable() &&
2474 !DstSize.isScalable())
2475 break;
2476
2477 if (SrcSize.isNonZero() && DstSize.isNonZero() && SrcSize != DstSize) {
2478 if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {
2479 report(msg: "Copy Instruction is illegal with mismatching sizes", MI);
2480 OS << "Def Size = " << DstSize << ", Src Size = " << SrcSize << '\n';
2481 }
2482 }
2483 break;
2484 }
2485 case TargetOpcode::COPY_LANEMASK: {
2486 const MachineOperand &DstOp = MI->getOperand(i: 0);
2487 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2488 const MachineOperand &LaneMaskOp = MI->getOperand(i: 2);
2489 const Register SrcReg = SrcOp.getReg();
2490 const LaneBitmask LaneMask = LaneMaskOp.getLaneMask();
2491 LaneBitmask SrcMaxLaneMask = LaneBitmask::getAll();
2492
2493 if (DstOp.getSubReg())
2494 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &DstOp, MONum: 0);
2495
2496 if (SrcOp.getSubReg())
2497 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &SrcOp, MONum: 1);
2498
2499 if (LaneMask.none())
2500 report(msg: "COPY_LANEMASK must read at least one lane", MI);
2501
2502 if (SrcReg.isPhysical()) {
2503 const TargetRegisterClass *SrcRC = TRI->getMinimalPhysRegClass(Reg: SrcReg);
2504 if (SrcRC)
2505 SrcMaxLaneMask = SrcRC->getLaneMask();
2506 } else {
2507 SrcMaxLaneMask = MRI->getMaxLaneMaskForVReg(Reg: SrcReg);
2508 }
2509
2510 // COPY_LANEMASK should be used only for partial copy. For full
2511 // copy, one should strictly use the COPY instruction.
2512 if (SrcMaxLaneMask == LaneMask)
2513 report(msg: "COPY_LANEMASK cannot be used to do full copy", MI);
2514
2515 // If LaneMask is greater than the SrcMaxLaneMask, it implies
2516 // COPY_LANEMASK is attempting to read from the lanes that
2517 // don't exists in the source register.
2518 if (SrcMaxLaneMask < LaneMask)
2519 report(msg: "COPY_LANEMASK attempts to read from the lanes that "
2520 "don't exist in the source register",
2521 MI);
2522
2523 break;
2524 }
2525 case TargetOpcode::STATEPOINT: {
2526 StatepointOpers SO(MI);
2527 if (!MI->getOperand(i: SO.getIDPos()).isImm() ||
2528 !MI->getOperand(i: SO.getNBytesPos()).isImm() ||
2529 !MI->getOperand(i: SO.getNCallArgsPos()).isImm()) {
2530 report(msg: "meta operands to STATEPOINT not constant!", MI);
2531 break;
2532 }
2533
2534 auto VerifyStackMapConstant = [&](unsigned Offset) {
2535 if (Offset >= MI->getNumOperands()) {
2536 report(msg: "stack map constant to STATEPOINT is out of range!", MI);
2537 return;
2538 }
2539 if (!MI->getOperand(i: Offset - 1).isImm() ||
2540 MI->getOperand(i: Offset - 1).getImm() != StackMaps::ConstantOp ||
2541 !MI->getOperand(i: Offset).isImm())
2542 report(msg: "stack map constant to STATEPOINT not well formed!", MI);
2543 };
2544 VerifyStackMapConstant(SO.getCCIdx());
2545 VerifyStackMapConstant(SO.getFlagsIdx());
2546 VerifyStackMapConstant(SO.getNumDeoptArgsIdx());
2547 VerifyStackMapConstant(SO.getNumGCPtrIdx());
2548 VerifyStackMapConstant(SO.getNumAllocaIdx());
2549 VerifyStackMapConstant(SO.getNumGcMapEntriesIdx());
2550
2551 // Verify that all explicit statepoint defs are tied to gc operands as
2552 // they are expected to be a relocation of gc operands.
2553 unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx();
2554 unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2;
2555 for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) {
2556 unsigned UseOpIdx;
2557 if (!MI->isRegTiedToUseOperand(DefOpIdx: Idx, UseOpIdx: &UseOpIdx)) {
2558 report(msg: "STATEPOINT defs expected to be tied", MI);
2559 break;
2560 }
2561 if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) {
2562 report(msg: "STATEPOINT def tied to non-gc operand", MI);
2563 break;
2564 }
2565 }
2566
2567 // TODO: verify we have properly encoded deopt arguments
2568 } break;
2569 case TargetOpcode::INSERT_SUBREG: {
2570 unsigned InsertedSize;
2571 if (unsigned SubIdx = MI->getOperand(i: 2).getSubReg())
2572 InsertedSize = TRI->getSubRegIdxSize(Idx: SubIdx);
2573 else
2574 InsertedSize = TRI->getRegSizeInBits(Reg: MI->getOperand(i: 2).getReg(), MRI: *MRI);
2575 unsigned SubRegSize = TRI->getSubRegIdxSize(Idx: MI->getOperand(i: 3).getImm());
2576 if (SubRegSize < InsertedSize) {
2577 report(msg: "INSERT_SUBREG expected inserted value to have equal or lesser "
2578 "size than the subreg it was inserted into", MI);
2579 break;
2580 }
2581 } break;
2582 case TargetOpcode::REG_SEQUENCE: {
2583 unsigned NumOps = MI->getNumOperands();
2584 if (!(NumOps & 1)) {
2585 report(msg: "Invalid number of operands for REG_SEQUENCE", MI);
2586 break;
2587 }
2588
2589 for (unsigned I = 1; I != NumOps; I += 2) {
2590 const MachineOperand &RegOp = MI->getOperand(i: I);
2591 const MachineOperand &SubRegOp = MI->getOperand(i: I + 1);
2592
2593 if (!RegOp.isReg())
2594 report(msg: "Invalid register operand for REG_SEQUENCE", MO: &RegOp, MONum: I);
2595
2596 if (!SubRegOp.isImm() || SubRegOp.getImm() == 0 ||
2597 SubRegOp.getImm() >= TRI->getNumSubRegIndices()) {
2598 report(msg: "Invalid subregister index operand for REG_SEQUENCE",
2599 MO: &SubRegOp, MONum: I + 1);
2600 }
2601 }
2602
2603 Register DstReg = MI->getOperand(i: 0).getReg();
2604 if (DstReg.isPhysical())
2605 report(msg: "REG_SEQUENCE does not support physical register results", MI);
2606
2607 if (MI->getOperand(i: 0).getSubReg())
2608 report(msg: "Invalid subreg result for REG_SEQUENCE", MI);
2609
2610 break;
2611 }
2612 }
2613}
2614
2615void
2616MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
2617 const MachineInstr *MI = MO->getParent();
2618 const MCInstrDesc &MCID = MI->getDesc();
2619 unsigned NumDefs = MCID.getNumDefs();
2620 if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)
2621 NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;
2622
2623 // The first MCID.NumDefs operands must be explicit register defines
2624 if (MONum < NumDefs) {
2625 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2626 if (!MO->isReg())
2627 report(msg: "Explicit definition must be a register", MO, MONum);
2628 else if (!MO->isDef() && !MCOI.isOptionalDef())
2629 report(msg: "Explicit definition marked as use", MO, MONum);
2630 else if (MO->isImplicit())
2631 report(msg: "Explicit definition marked as implicit", MO, MONum);
2632 } else if (MONum < MCID.getNumOperands()) {
2633 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2634 // Don't check if it's the last operand in a variadic instruction. See,
2635 // e.g., LDM_RET in the arm back end. Check non-variadic operands only.
2636 bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1;
2637 if (!IsOptional) {
2638 if (MO->isReg()) {
2639 if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs())
2640 report(msg: "Explicit operand marked as def", MO, MONum);
2641 if (MO->isImplicit())
2642 report(msg: "Explicit operand marked as implicit", MO, MONum);
2643 }
2644
2645 // Check that an instruction has register operands only as expected.
2646 if (MCOI.OperandType == MCOI::OPERAND_REGISTER &&
2647 !MO->isReg() && !MO->isFI())
2648 report(msg: "Expected a register operand.", MO, MONum);
2649 if (MO->isReg()) {
2650 if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE ||
2651 (MCOI.OperandType == MCOI::OPERAND_PCREL &&
2652 !TII->isPCRelRegisterOperandLegal(MO: *MO)))
2653 report(msg: "Expected a non-register operand.", MO, MONum);
2654 }
2655 }
2656
2657 int TiedTo = MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::TIED_TO);
2658 if (TiedTo != -1) {
2659 if (!MO->isReg())
2660 report(msg: "Tied use must be a register", MO, MONum);
2661 else if (!MO->isTied())
2662 report(msg: "Operand should be tied", MO, MONum);
2663 else if (unsigned(TiedTo) != MI->findTiedOperandIdx(OpIdx: MONum))
2664 report(msg: "Tied def doesn't match MCInstrDesc", MO, MONum);
2665 else if (MO->getReg().isPhysical()) {
2666 const MachineOperand &MOTied = MI->getOperand(i: TiedTo);
2667 if (!MOTied.isReg())
2668 report(msg: "Tied counterpart must be a register", MO: &MOTied, MONum: TiedTo);
2669 else if (MOTied.getReg().isPhysical() &&
2670 MO->getReg() != MOTied.getReg())
2671 report(msg: "Tied physical registers must match.", MO: &MOTied, MONum: TiedTo);
2672 }
2673 } else if (MO->isReg() && MO->isTied())
2674 report(msg: "Explicit operand should not be tied", MO, MONum);
2675 } else if (!MI->isVariadic()) {
2676 // ARM adds %reg0 operands to indicate predicates. We'll allow that.
2677 if (!MO->isValidExcessOperand())
2678 report(msg: "Extra explicit operand on non-variadic instruction", MO, MONum);
2679 }
2680
2681 // Verify earlyClobber def operand
2682 if (MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::EARLY_CLOBBER) != -1) {
2683 if (!MO->isReg())
2684 report(msg: "Early clobber must be a register", MI);
2685 if (!MO->isEarlyClobber())
2686 report(msg: "Missing earlyClobber flag", MI);
2687 }
2688
2689 switch (MO->getType()) {
2690 case MachineOperand::MO_Register: {
2691 // Verify debug flag on debug instructions. Check this first because reg0
2692 // indicates an undefined debug value.
2693 if (MI->isDebugInstr() && MO->isUse()) {
2694 if (!MO->isDebug())
2695 report(msg: "Register operand must be marked debug", MO, MONum);
2696 } else if (MO->isDebug()) {
2697 report(msg: "Register operand must not be marked debug", MO, MONum);
2698 }
2699
2700 const Register Reg = MO->getReg();
2701 if (!Reg)
2702 return;
2703 if (MRI->tracksLiveness() && !MI->isDebugInstr())
2704 checkLiveness(MO, MONum);
2705
2706 if (MO->isDef() && MO->isUndef() && !MO->getSubReg() &&
2707 MO->getReg().isVirtual()) // TODO: Apply to physregs too
2708 report(msg: "Undef virtual register def operands require a subregister", MO, MONum);
2709
2710 // Verify the consistency of tied operands.
2711 if (MO->isTied()) {
2712 unsigned OtherIdx = MI->findTiedOperandIdx(OpIdx: MONum);
2713 const MachineOperand &OtherMO = MI->getOperand(i: OtherIdx);
2714 if (!OtherMO.isReg())
2715 report(msg: "Must be tied to a register", MO, MONum);
2716 if (!OtherMO.isTied())
2717 report(msg: "Missing tie flags on tied operand", MO, MONum);
2718 if (MI->findTiedOperandIdx(OpIdx: OtherIdx) != MONum)
2719 report(msg: "Inconsistent tie links", MO, MONum);
2720 if (MONum < MCID.getNumDefs()) {
2721 if (OtherIdx < MCID.getNumOperands()) {
2722 if (-1 == MCID.getOperandConstraint(OpNum: OtherIdx, Constraint: MCOI::TIED_TO))
2723 report(msg: "Explicit def tied to explicit use without tie constraint",
2724 MO, MONum);
2725 } else {
2726 if (!OtherMO.isImplicit())
2727 report(msg: "Explicit def should be tied to implicit use", MO, MONum);
2728 }
2729 }
2730 }
2731
2732 // Verify two-address constraints after the twoaddressinstruction pass.
2733 // Both twoaddressinstruction pass and phi-node-elimination pass call
2734 // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification
2735 // after twoaddressinstruction pass not after phi-node-elimination pass. So
2736 // we shouldn't use the IsSSA as the condition, we should based on
2737 // TiedOpsRewritten property to verify two-address constraints, this
2738 // property will be set in twoaddressinstruction pass.
2739 unsigned DefIdx;
2740 if (MF->getProperties().hasTiedOpsRewritten() && MO->isUse() &&
2741 MI->isRegTiedToDefOperand(UseOpIdx: MONum, DefOpIdx: &DefIdx) &&
2742 Reg != MI->getOperand(i: DefIdx).getReg())
2743 report(msg: "Two-address instruction operands must be identical", MO, MONum);
2744
2745 // Check register classes.
2746 unsigned SubIdx = MO->getSubReg();
2747
2748 if (Reg.isPhysical()) {
2749 if (SubIdx) {
2750 report(msg: "Illegal subregister index for physical register", MO, MONum);
2751 return;
2752 }
2753 if (MONum < MCID.getNumOperands()) {
2754 if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum)) {
2755 if (!DRC->contains(Reg)) {
2756 report(msg: "Illegal physical register for instruction", MO, MONum);
2757 OS << printReg(Reg, TRI) << " is not a "
2758 << TRI->getRegClassName(Class: DRC) << " register.\n";
2759 }
2760 }
2761 }
2762 if (MO->isRenamable()) {
2763 if (MRI->isReserved(PhysReg: Reg)) {
2764 report(msg: "isRenamable set on reserved register", MO, MONum);
2765 return;
2766 }
2767 }
2768 } else {
2769 // Virtual register.
2770 const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);
2771 if (!RC) {
2772 // This is a generic virtual register.
2773
2774 // Do not allow undef uses for generic virtual registers. This ensures
2775 // getVRegDef can never fail and return null on a generic register.
2776 //
2777 // FIXME: This restriction should probably be broadened to all SSA
2778 // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still
2779 // run on the SSA function just before phi elimination.
2780 if (MO->isUndef())
2781 report(msg: "Generic virtual register use cannot be undef", MO, MONum);
2782
2783 // Debug value instruction is permitted to use undefined vregs.
2784 // This is a performance measure to skip the overhead of immediately
2785 // pruning unused debug operands. The final undef substitution occurs
2786 // when debug values are allocated in LDVImpl::handleDebugValue, so
2787 // these verifications always apply after this pass.
2788 if (isFunctionTracksDebugUserValues || !MO->isUse() ||
2789 !MI->isDebugValue() || !MRI->def_empty(RegNo: Reg)) {
2790 // If we're post-Select, we can't have gvregs anymore.
2791 if (isFunctionSelected) {
2792 report(msg: "Generic virtual register invalid in a Selected function",
2793 MO, MONum);
2794 return;
2795 }
2796
2797 // The gvreg must have a type and it must not have a SubIdx.
2798 LLT Ty = MRI->getType(Reg);
2799 if (!Ty.isValid()) {
2800 report(msg: "Generic virtual register must have a valid type", MO,
2801 MONum);
2802 return;
2803 }
2804
2805 const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
2806 const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo();
2807
2808 // If we're post-RegBankSelect, the gvreg must have a bank.
2809 if (!RegBank && isFunctionRegBankSelected) {
2810 report(msg: "Generic virtual register must have a bank in a "
2811 "RegBankSelected function",
2812 MO, MONum);
2813 return;
2814 }
2815
2816 // Make sure the register fits into its register bank if any.
2817 if (RegBank && Ty.isValid() && !Ty.isScalableVector() &&
2818 RBI->getMaximumSize(RegBankID: RegBank->getID()) < Ty.getSizeInBits()) {
2819 report(msg: "Register bank is too small for virtual register", MO,
2820 MONum);
2821 OS << "Register bank " << RegBank->getName() << " too small("
2822 << RBI->getMaximumSize(RegBankID: RegBank->getID()) << ") to fit "
2823 << Ty.getSizeInBits() << "-bits\n";
2824 return;
2825 }
2826 }
2827
2828 if (SubIdx) {
2829 report(msg: "Generic virtual register does not allow subregister index", MO,
2830 MONum);
2831 return;
2832 }
2833
2834 // If this is a target specific instruction and this operand
2835 // has register class constraint, the virtual register must
2836 // comply to it.
2837 if (!isPreISelGenericOpcode(Opcode: MCID.getOpcode()) &&
2838 MONum < MCID.getNumOperands() && TII->getRegClass(MCID, OpNum: MONum)) {
2839 report(msg: "Virtual register does not match instruction constraint", MO,
2840 MONum);
2841 OS << "Expect register class "
2842 << TRI->getRegClassName(Class: TII->getRegClass(MCID, OpNum: MONum))
2843 << " but got nothing\n";
2844 return;
2845 }
2846
2847 break;
2848 }
2849 // Validate that SubIdx can be applied to the virtual register.
2850 if (!TRI->isSubRegValidForRegClass(RC, Idx: SubIdx)) {
2851 report(msg: "Invalid subregister index for virtual register", MO, MONum);
2852 OS << "Register class " << TRI->getRegClassName(Class: RC)
2853 << " does not support subreg index "
2854 << TRI->getSubRegIndexName(SubIdx) << '\n';
2855 return;
2856 }
2857 if (MONum >= MCID.getNumOperands())
2858 break;
2859 const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum);
2860 if (!DRC)
2861 break;
2862
2863 // If SubIdx is used, verify that RC with SubIdx can be used for an
2864 // operand of class DRC. This is valid if for every register in RC, the
2865 // register obtained by applying SubIdx to it is in DRC.
2866 if (SubIdx && TRI->getMatchingSuperRegClass(A: RC, B: DRC, Idx: SubIdx) != RC) {
2867 report(msg: "Illegal virtual register for instruction", MO, MONum);
2868 OS << TRI->getRegClassName(Class: RC) << "." << TRI->getSubRegIndexName(SubIdx)
2869 << " cannot be used for " << TRI->getRegClassName(Class: DRC)
2870 << " operands.";
2871 }
2872
2873 // If no SubIdx is used, verify that RC is a sub-class of DRC.
2874 if (!SubIdx && !RC->hasSuperClassEq(RC: DRC)) {
2875 report(msg: "Illegal virtual register for instruction", MO, MONum);
2876 OS << "Expected a " << TRI->getRegClassName(Class: DRC)
2877 << " register, but got a " << TRI->getRegClassName(Class: RC)
2878 << " register\n";
2879 }
2880 }
2881 break;
2882 }
2883
2884 case MachineOperand::MO_RegisterMask:
2885 regMasks.push_back(Elt: MO->getRegMask());
2886 break;
2887
2888 case MachineOperand::MO_MachineBasicBlock:
2889 if (MI->isPHI() && !MO->getMBB()->isSuccessor(MBB: MI->getParent()))
2890 report(msg: "PHI operand is not in the CFG", MO, MONum);
2891 break;
2892
2893 case MachineOperand::MO_FrameIndex:
2894 if (LiveStks && LiveStks->hasInterval(Slot: MO->getIndex()) &&
2895 LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
2896 int FI = MO->getIndex();
2897 LiveInterval &LI = LiveStks->getInterval(Slot: FI);
2898 SlotIndex Idx = LiveInts->getInstructionIndex(Instr: *MI);
2899
2900 bool MayStore = MI->mayStore();
2901 bool MayLoad = MI->mayLoad();
2902 // For a memory-to-memory move, we need to check if the frame
2903 // index is used for storing or loading, by inspecting the
2904 // memory operands.
2905 if (MayStore && MayLoad) {
2906 for (const MachineMemOperand *MMO : MI->memoperands()) {
2907 const auto *Value = dyn_cast_if_present<FixedStackPseudoSourceValue>(
2908 Val: MMO->getPseudoValue());
2909 if (!Value || Value->getFrameIndex() != FI)
2910 continue;
2911
2912 if (MMO->isStore())
2913 MayLoad = false;
2914 else
2915 MayStore = false;
2916 break;
2917 }
2918 if (MayLoad == MayStore)
2919 report(msg: "Missing fixed stack memoperand.", MI);
2920 }
2921 if (MayLoad && !LI.liveAt(index: Idx.getRegSlot(EC: true))) {
2922 report(msg: "Instruction loads from dead spill slot", MO, MONum);
2923 OS << "Live stack: " << LI << '\n';
2924 }
2925 if (MayStore && !LI.liveAt(index: Idx.getRegSlot())) {
2926 report(msg: "Instruction stores to dead spill slot", MO, MONum);
2927 OS << "Live stack: " << LI << '\n';
2928 }
2929 }
2930 break;
2931
2932 case MachineOperand::MO_CFIIndex:
2933 if (MO->getCFIIndex() >= MF->getFrameInstructions().size())
2934 report(msg: "CFI instruction has invalid index", MO, MONum);
2935 break;
2936
2937 default:
2938 break;
2939 }
2940}
2941
2942void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
2943 unsigned MONum, SlotIndex UseIdx,
2944 const LiveRange &LR,
2945 VirtRegOrUnit VRegOrUnit,
2946 LaneBitmask LaneMask) {
2947 const MachineInstr *MI = MO->getParent();
2948
2949 if (!LR.verify()) {
2950 report(msg: "invalid live range", MO, MONum);
2951 report_context_liverange(LR);
2952 report_context_vreg_regunit(VRegOrUnit);
2953 report_context(Pos: UseIdx);
2954 return;
2955 }
2956
2957 LiveQueryResult LRQ = LR.Query(Idx: UseIdx);
2958 bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut());
2959 // Check if we have a segment at the use, note however that we only need one
2960 // live subregister range, the others may be dead.
2961 if (!HasValue && LaneMask.none()) {
2962 report(msg: "No live segment at use", MO, MONum);
2963 report_context_liverange(LR);
2964 report_context_vreg_regunit(VRegOrUnit);
2965 report_context(Pos: UseIdx);
2966 }
2967 if (MO->isKill() && !LRQ.isKill()) {
2968 report(msg: "Live range continues after kill flag", MO, MONum);
2969 report_context_liverange(LR);
2970 report_context_vreg_regunit(VRegOrUnit);
2971 if (LaneMask.any())
2972 report_context_lanemask(LaneMask);
2973 report_context(Pos: UseIdx);
2974 }
2975}
2976
2977void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
2978 unsigned MONum, SlotIndex DefIdx,
2979 const LiveRange &LR,
2980 VirtRegOrUnit VRegOrUnit,
2981 bool SubRangeCheck,
2982 LaneBitmask LaneMask) {
2983 if (!LR.verify()) {
2984 report(msg: "invalid live range", MO, MONum);
2985 report_context_liverange(LR);
2986 report_context_vreg_regunit(VRegOrUnit);
2987 if (LaneMask.any())
2988 report_context_lanemask(LaneMask);
2989 report_context(Pos: DefIdx);
2990 }
2991
2992 if (const VNInfo *VNI = LR.getVNInfoAt(Idx: DefIdx)) {
2993 // The LR can correspond to the whole reg and its def slot is not obliged
2994 // to be the same as the MO' def slot. E.g. when we check here "normal"
2995 // subreg MO but there is other EC subreg MO in the same instruction so the
2996 // whole reg has EC def slot and differs from the currently checked MO' def
2997 // slot. For example:
2998 // %0 [16e,32r:0) 0@16e L..3 [16e,32r:0) 0@16e L..C [16r,32r:0) 0@16r
2999 // Check that there is an early-clobber def of the same superregister
3000 // somewhere is performed in visitMachineFunctionAfter()
3001 if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) ||
3002 !SlotIndex::isSameInstr(A: VNI->def, B: DefIdx) ||
3003 (VNI->def != DefIdx &&
3004 (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) {
3005 report(msg: "Inconsistent valno->def", MO, MONum);
3006 report_context_liverange(LR);
3007 report_context_vreg_regunit(VRegOrUnit);
3008 if (LaneMask.any())
3009 report_context_lanemask(LaneMask);
3010 report_context(VNI: *VNI);
3011 report_context(Pos: DefIdx);
3012 }
3013 } else {
3014 report(msg: "No live segment at def", MO, MONum);
3015 report_context_liverange(LR);
3016 report_context_vreg_regunit(VRegOrUnit);
3017 if (LaneMask.any())
3018 report_context_lanemask(LaneMask);
3019 report_context(Pos: DefIdx);
3020 }
3021 // Check that, if the dead def flag is present, LiveInts agree.
3022 if (MO->isDead()) {
3023 LiveQueryResult LRQ = LR.Query(Idx: DefIdx);
3024 if (!LRQ.isDeadDef()) {
3025 assert(VRegOrUnit.isVirtualReg() && "Expecting a virtual register.");
3026 // A dead subreg def only tells us that the specific subreg is dead. There
3027 // could be other non-dead defs of other subregs, or we could have other
3028 // parts of the register being live through the instruction. So unless we
3029 // are checking liveness for a subrange it is ok for the live range to
3030 // continue, given that we have a dead def of a subregister.
3031 if (SubRangeCheck || MO->getSubReg() == 0) {
3032 report(msg: "Live range continues after dead def flag", MO, MONum);
3033 report_context_liverange(LR);
3034 report_context_vreg_regunit(VRegOrUnit);
3035 if (LaneMask.any())
3036 report_context_lanemask(LaneMask);
3037 }
3038 }
3039 }
3040}
3041
3042void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
3043 const MachineInstr *MI = MO->getParent();
3044 const Register Reg = MO->getReg();
3045 const unsigned SubRegIdx = MO->getSubReg();
3046
3047 const LiveInterval *LI = nullptr;
3048 if (LiveInts && Reg.isVirtual()) {
3049 if (LiveInts->hasInterval(Reg)) {
3050 LI = &LiveInts->getInterval(Reg);
3051 if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&
3052 !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(VReg: Reg))
3053 report(msg: "Live interval for subreg operand has no subranges", MO, MONum);
3054 } else {
3055 report(msg: "Virtual register has no live interval", MO, MONum);
3056 }
3057 }
3058
3059 // Both use and def operands can read a register.
3060 if (MO->readsReg()) {
3061 if (MO->isKill())
3062 addRegWithSubRegs(RV&: regsKilled, Reg);
3063
3064 // Check that LiveVars knows this kill (unless we are inside a bundle, in
3065 // which case we have already checked that LiveVars knows any kills on the
3066 // bundle header instead).
3067 if (LiveVars && Reg.isVirtual() && MO->isKill() &&
3068 !MI->isBundledWithPred()) {
3069 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
3070 if (!is_contained(Range&: VI.Kills, Element: MI))
3071 report(msg: "Kill missing from LiveVariables", MO, MONum);
3072 }
3073
3074 // Check LiveInts liveness and kill.
3075 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3076 SlotIndex UseIdx;
3077 if (MI->isPHI()) {
3078 // PHI use occurs on the edge, so check for live out here instead.
3079 UseIdx = LiveInts->getMBBEndIdx(
3080 mbb: MI->getOperand(i: MONum + 1).getMBB()).getPrevSlot();
3081 } else {
3082 UseIdx = LiveInts->getInstructionIndex(Instr: *MI);
3083 }
3084 // Check the cached regunit intervals.
3085 if (Reg.isPhysical() && !isReserved(Reg)) {
3086 for (MCRegUnit Unit : TRI->regunits(Reg: Reg.asMCReg())) {
3087 if (MRI->isReservedRegUnit(Unit))
3088 continue;
3089 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3090 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LR, VRegOrUnit: VirtRegOrUnit(Unit));
3091 }
3092 }
3093
3094 if (Reg.isVirtual()) {
3095 // This is a virtual register interval.
3096 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3097
3098 if (LI->hasSubRanges() && !MO->isDef()) {
3099 LaneBitmask MOMask = SubRegIdx != 0
3100 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3101 : MRI->getMaxLaneMaskForVReg(Reg);
3102 LaneBitmask LiveInMask;
3103 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3104 if ((MOMask & SR.LaneMask).none())
3105 continue;
3106 checkLivenessAtUse(MO, MONum, UseIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg),
3107 LaneMask: SR.LaneMask);
3108 LiveQueryResult LRQ = SR.Query(Idx: UseIdx);
3109 if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))
3110 LiveInMask |= SR.LaneMask;
3111 }
3112 // At least parts of the register has to be live at the use.
3113 if ((LiveInMask & MOMask).none()) {
3114 report(msg: "No live subrange at use", MO, MONum);
3115 report_context(LI: *LI);
3116 report_context(Pos: UseIdx);
3117 }
3118 // For PHIs all lanes should be live
3119 if (MI->isPHI() && LiveInMask != MOMask) {
3120 report(msg: "Not all lanes of PHI source live at use", MO, MONum);
3121 report_context(LI: *LI);
3122 report_context(Pos: UseIdx);
3123 }
3124 }
3125 }
3126 }
3127
3128 // Use of a dead register.
3129 if (!regsLive.count(V: Reg)) {
3130 if (Reg.isPhysical()) {
3131 // Reserved registers may be used even when 'dead'.
3132 bool Bad = !isReserved(Reg);
3133 // We are fine if just any subregister has a defined value.
3134 if (Bad) {
3135
3136 for (const MCPhysReg &SubReg : TRI->subregs(Reg)) {
3137 if (regsLive.count(V: SubReg)) {
3138 Bad = false;
3139 break;
3140 }
3141 }
3142 }
3143 // If there is an additional implicit-use of a super register we stop
3144 // here. By definition we are fine if the super register is not
3145 // (completely) dead, if the complete super register is dead we will
3146 // get a report for its operand.
3147 if (Bad) {
3148 for (const MachineOperand &MOP : MI->uses()) {
3149 if (!MOP.isReg() || !MOP.isImplicit())
3150 continue;
3151
3152 if (!MOP.getReg().isPhysical())
3153 continue;
3154
3155 if (MOP.getReg() != Reg &&
3156 all_of(Range: TRI->regunits(Reg), P: [&](const MCRegUnit RegUnit) {
3157 return llvm::is_contained(Range: TRI->regunits(Reg: MOP.getReg()),
3158 Element: RegUnit);
3159 }))
3160 Bad = false;
3161 }
3162 }
3163 if (Bad)
3164 report(msg: "Using an undefined physical register", MO, MONum);
3165 } else if (MRI->def_empty(RegNo: Reg)) {
3166 report(msg: "Reading virtual register without a def", MO, MONum);
3167 } else {
3168 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3169 // We don't know which virtual registers are live in, so only complain
3170 // if vreg was killed in this MBB. Otherwise keep track of vregs that
3171 // must be live in. PHI instructions are handled separately.
3172 if (MInfo.regsKilled.count(V: Reg))
3173 report(msg: "Using a killed virtual register", MO, MONum);
3174 else if (!MI->isPHI())
3175 MInfo.vregsLiveIn.insert(KV: std::make_pair(x: Reg, y&: MI));
3176 }
3177 }
3178 }
3179
3180 if (MO->isDef()) {
3181 // Register defined.
3182 // TODO: verify that earlyclobber ops are not used.
3183 if (MO->isDead())
3184 addRegWithSubRegs(RV&: regsDead, Reg);
3185 else
3186 addRegWithSubRegs(RV&: regsDefined, Reg);
3187
3188 // Verify SSA form.
3189 if (MRI->isSSA() && Reg.isVirtual()) {
3190 if (!MRI->hasOneDef(RegNo: Reg))
3191 report(msg: "Multiple virtual register defs in SSA form", MO, MONum);
3192 if (MO->getSubReg())
3193 report(msg: "Subreg def in SSA form", MO, MONum);
3194 }
3195
3196 // Check LiveInts for a live segment, but only for virtual registers.
3197 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3198 SlotIndex DefIdx = LiveInts->getInstructionIndex(Instr: *MI);
3199 DefIdx = DefIdx.getRegSlot(EC: MO->isEarlyClobber());
3200
3201 if (Reg.isVirtual()) {
3202 checkLivenessAtDef(MO, MONum, DefIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3203
3204 if (LI->hasSubRanges()) {
3205 LaneBitmask MOMask = SubRegIdx != 0
3206 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3207 : MRI->getMaxLaneMaskForVReg(Reg);
3208 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3209 if ((SR.LaneMask & MOMask).none())
3210 continue;
3211 checkLivenessAtDef(MO, MONum, DefIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg), SubRangeCheck: true,
3212 LaneMask: SR.LaneMask);
3213 }
3214 }
3215 }
3216 }
3217 }
3218}
3219
3220// This function gets called after visiting all instructions in a bundle. The
3221// argument points to the bundle header.
3222// Normal stand-alone instructions are also considered 'bundles', and this
3223// function is called for all of them.
3224void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {
3225 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3226 set_union(S1&: MInfo.regsKilled, S2: regsKilled);
3227 set_subtract(S1&: regsLive, S2: regsKilled); regsKilled.clear();
3228 // Kill any masked registers.
3229 while (!regMasks.empty()) {
3230 const uint32_t *Mask = regMasks.pop_back_val();
3231 for (Register Reg : regsLive)
3232 if (Reg.isPhysical() &&
3233 MachineOperand::clobbersPhysReg(RegMask: Mask, PhysReg: Reg.asMCReg()))
3234 regsDead.push_back(Elt: Reg);
3235 }
3236 set_subtract(S1&: regsLive, S2: regsDead); regsDead.clear();
3237 set_union(S1&: regsLive, S2: regsDefined); regsDefined.clear();
3238}
3239
3240void
3241MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {
3242 MBBInfoMap[MBB].regsLiveOut = regsLive;
3243 regsLive.clear();
3244
3245 if (Indexes) {
3246 SlotIndex stop = Indexes->getMBBEndIdx(mbb: MBB);
3247 if (!(stop > lastIndex)) {
3248 report(msg: "Block ends before last instruction index", MBB);
3249 OS << "Block ends at " << stop << " last instruction was at " << lastIndex
3250 << '\n';
3251 }
3252 lastIndex = stop;
3253 }
3254}
3255
3256namespace {
3257// This implements a set of registers that serves as a filter: can filter other
3258// sets by passing through elements not in the filter and blocking those that
3259// are. Any filter implicitly includes the full set of physical registers upon
3260// creation, thus filtering them all out. The filter itself as a set only grows,
3261// and needs to be as efficient as possible.
3262struct VRegFilter {
3263 // Add elements to the filter itself. \pre Input set \p FromRegSet must have
3264 // no duplicates. Both virtual and physical registers are fine.
3265 template <typename RegSetT> void add(const RegSetT &FromRegSet) {
3266 SmallVector<Register, 0> VRegsBuffer;
3267 filterAndAdd(FromRegSet, VRegsBuffer);
3268 }
3269 // Filter \p FromRegSet through the filter and append passed elements into \p
3270 // ToVRegs. All elements appended are then added to the filter itself.
3271 // \returns true if anything changed.
3272 template <typename RegSetT>
3273 bool filterAndAdd(const RegSetT &FromRegSet,
3274 SmallVectorImpl<Register> &ToVRegs) {
3275 unsigned SparseUniverse = Sparse.size();
3276 unsigned NewSparseUniverse = SparseUniverse;
3277 unsigned NewDenseSize = Dense.size();
3278 size_t Begin = ToVRegs.size();
3279 for (Register Reg : FromRegSet) {
3280 if (!Reg.isVirtual())
3281 continue;
3282 unsigned Index = Reg.virtRegIndex();
3283 if (Index < SparseUniverseMax) {
3284 if (Index < SparseUniverse && Sparse.test(Idx: Index))
3285 continue;
3286 NewSparseUniverse = std::max(a: NewSparseUniverse, b: Index + 1);
3287 } else {
3288 if (Dense.count(V: Reg))
3289 continue;
3290 ++NewDenseSize;
3291 }
3292 ToVRegs.push_back(Elt: Reg);
3293 }
3294 size_t End = ToVRegs.size();
3295 if (Begin == End)
3296 return false;
3297 // Reserving space in sets once performs better than doing so continuously
3298 // and pays easily for double look-ups (even in Dense with SparseUniverseMax
3299 // tuned all the way down) and double iteration (the second one is over a
3300 // SmallVector, which is a lot cheaper compared to DenseSet or BitVector).
3301 Sparse.resize(N: NewSparseUniverse);
3302 Dense.reserve(Size: NewDenseSize);
3303 for (unsigned I = Begin; I < End; ++I) {
3304 Register Reg = ToVRegs[I];
3305 unsigned Index = Reg.virtRegIndex();
3306 if (Index < SparseUniverseMax)
3307 Sparse.set(Index);
3308 else
3309 Dense.insert(V: Reg);
3310 }
3311 return true;
3312 }
3313
3314private:
3315 static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8;
3316 // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyond
3317 // are tracked by Dense. The only purpose of the threshold and the Dense set
3318 // is to have a reasonably growing memory usage in pathological cases (large
3319 // number of very sparse VRegFilter instances live at the same time). In
3320 // practice even in the worst-by-execution time cases having all elements
3321 // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more
3322 // space efficient than if tracked by Dense. The threshold is set to keep the
3323 // worst-case memory usage within 2x of figures determined empirically for
3324 // "all Dense" scenario in such worst-by-execution-time cases.
3325 BitVector Sparse;
3326 DenseSet<Register> Dense;
3327};
3328
3329// Implements both a transfer function and a (binary, in-place) join operator
3330// for a dataflow over register sets with set union join and filtering transfer
3331// (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.
3332// Maintains out_b as its state, allowing for O(n) iteration over it at any
3333// time, where n is the size of the set (as opposed to O(U) where U is the
3334// universe). filter_b implicitly contains all physical registers at all times.
3335class FilteringVRegSet {
3336 VRegFilter Filter;
3337 SmallVector<Register, 0> VRegs;
3338
3339public:
3340 // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.
3341 // Both virtual and physical registers are fine.
3342 template <typename RegSetT> void addToFilter(const RegSetT &RS) {
3343 Filter.add(RS);
3344 }
3345 // Passes \p RS through the filter_b (transfer function) and adds what's left
3346 // to itself (out_b).
3347 template <typename RegSetT> bool add(const RegSetT &RS) {
3348 // Double-duty the Filter: to maintain VRegs a set (and the join operation
3349 // a set union) just add everything being added here to the Filter as well.
3350 return Filter.filterAndAdd(RS, VRegs);
3351 }
3352 using const_iterator = decltype(VRegs)::const_iterator;
3353 const_iterator begin() const { return VRegs.begin(); }
3354 const_iterator end() const { return VRegs.end(); }
3355 size_t size() const { return VRegs.size(); }
3356};
3357} // namespace
3358
3359// Calculate the largest possible vregsPassed sets. These are the registers that
3360// can pass through an MBB live, but may not be live every time. It is assumed
3361// that all vregsPassed sets are empty before the call.
3362void MachineVerifier::calcRegsPassed() {
3363 if (MF->empty())
3364 // ReversePostOrderTraversal doesn't handle empty functions.
3365 return;
3366
3367 for (const MachineBasicBlock *MB :
3368 ReversePostOrderTraversal<const MachineFunction *>(MF)) {
3369 FilteringVRegSet VRegs;
3370 BBInfo &Info = MBBInfoMap[MB];
3371 assert(Info.reachable);
3372
3373 VRegs.addToFilter(RS: Info.regsKilled);
3374 VRegs.addToFilter(RS: Info.regsLiveOut);
3375 for (const MachineBasicBlock *Pred : MB->predecessors()) {
3376 const BBInfo &PredInfo = MBBInfoMap[Pred];
3377 if (!PredInfo.reachable)
3378 continue;
3379
3380 VRegs.add(RS: PredInfo.regsLiveOut);
3381 VRegs.add(RS: PredInfo.vregsPassed);
3382 }
3383 Info.vregsPassed.reserve(Size: VRegs.size());
3384 Info.vregsPassed.insert_range(R&: VRegs);
3385 }
3386}
3387
3388// Calculate the set of virtual registers that must be passed through each basic
3389// block in order to satisfy the requirements of successor blocks. This is very
3390// similar to calcRegsPassed, only backwards.
3391void MachineVerifier::calcRegsRequired() {
3392 // First push live-in regs to predecessors' vregsRequired.
3393 SmallPtrSet<const MachineBasicBlock*, 8> todo;
3394 for (const auto &MBB : *MF) {
3395 BBInfo &MInfo = MBBInfoMap[&MBB];
3396 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3397 BBInfo &PInfo = MBBInfoMap[Pred];
3398 if (PInfo.addRequired(RM: MInfo.vregsLiveIn))
3399 todo.insert(Ptr: Pred);
3400 }
3401
3402 // Handle the PHI node.
3403 for (const MachineInstr &MI : MBB.phis()) {
3404 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3405 // Skip those Operands which are undef regs or not regs.
3406 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())
3407 continue;
3408
3409 // Get register and predecessor for one PHI edge.
3410 Register Reg = MI.getOperand(i).getReg();
3411 const MachineBasicBlock *Pred = MI.getOperand(i: i + 1).getMBB();
3412
3413 BBInfo &PInfo = MBBInfoMap[Pred];
3414 if (PInfo.addRequired(Reg))
3415 todo.insert(Ptr: Pred);
3416 }
3417 }
3418 }
3419
3420 // Iteratively push vregsRequired to predecessors. This will converge to the
3421 // same final state regardless of DenseSet iteration order.
3422 while (!todo.empty()) {
3423 const MachineBasicBlock *MBB = *todo.begin();
3424 todo.erase(Ptr: MBB);
3425 BBInfo &MInfo = MBBInfoMap[MBB];
3426 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
3427 if (Pred == MBB)
3428 continue;
3429 BBInfo &SInfo = MBBInfoMap[Pred];
3430 if (SInfo.addRequired(RS: MInfo.vregsRequired))
3431 todo.insert(Ptr: Pred);
3432 }
3433 }
3434}
3435
3436// Check PHI instructions at the beginning of MBB. It is assumed that
3437// calcRegsPassed has been run so BBInfo::isLiveOut is valid.
3438void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {
3439 BBInfo &MInfo = MBBInfoMap[&MBB];
3440
3441 SmallPtrSet<const MachineBasicBlock*, 8> seen;
3442 for (const MachineInstr &Phi : MBB) {
3443 if (!Phi.isPHI())
3444 break;
3445 seen.clear();
3446
3447 const MachineOperand &MODef = Phi.getOperand(i: 0);
3448 if (!MODef.isReg() || !MODef.isDef()) {
3449 report(msg: "Expected first PHI operand to be a register def", MO: &MODef, MONum: 0);
3450 continue;
3451 }
3452 if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||
3453 MODef.isEarlyClobber() || MODef.isDebug())
3454 report(msg: "Unexpected flag on PHI operand", MO: &MODef, MONum: 0);
3455 Register DefReg = MODef.getReg();
3456 if (!DefReg.isVirtual())
3457 report(msg: "Expected first PHI operand to be a virtual register", MO: &MODef, MONum: 0);
3458
3459 for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {
3460 const MachineOperand &MO0 = Phi.getOperand(i: I);
3461 if (!MO0.isReg()) {
3462 report(msg: "Expected PHI operand to be a register", MO: &MO0, MONum: I);
3463 continue;
3464 }
3465 if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||
3466 MO0.isDebug() || MO0.isTied())
3467 report(msg: "Unexpected flag on PHI operand", MO: &MO0, MONum: I);
3468
3469 const MachineOperand &MO1 = Phi.getOperand(i: I + 1);
3470 if (!MO1.isMBB()) {
3471 report(msg: "Expected PHI operand to be a basic block", MO: &MO1, MONum: I + 1);
3472 continue;
3473 }
3474
3475 const MachineBasicBlock &Pre = *MO1.getMBB();
3476 if (!Pre.isSuccessor(MBB: &MBB)) {
3477 report(msg: "PHI input is not a predecessor block", MO: &MO1, MONum: I + 1);
3478 continue;
3479 }
3480
3481 if (MInfo.reachable) {
3482 seen.insert(Ptr: &Pre);
3483 BBInfo &PrInfo = MBBInfoMap[&Pre];
3484 if (!MO0.isUndef() && PrInfo.reachable &&
3485 !PrInfo.isLiveOut(Reg: MO0.getReg()))
3486 report(msg: "PHI operand is not live-out from predecessor", MO: &MO0, MONum: I);
3487 }
3488 }
3489
3490 // Did we see all predecessors?
3491 if (MInfo.reachable) {
3492 for (MachineBasicBlock *Pred : MBB.predecessors()) {
3493 if (!seen.count(Ptr: Pred)) {
3494 report(msg: "Missing PHI operand", MI: &Phi);
3495 OS << printMBBReference(MBB: *Pred)
3496 << " is a predecessor according to the CFG.\n";
3497 }
3498 }
3499 }
3500 }
3501}
3502
3503static void
3504verifyConvergenceControl(const MachineFunction &MF, MachineDominatorTree &DT,
3505 std::function<void(const Twine &Message)> FailureCB,
3506 raw_ostream &OS) {
3507 MachineConvergenceVerifier CV;
3508 CV.initialize(OS: &OS, FailureCB, F: MF);
3509
3510 for (const auto &MBB : MF) {
3511 CV.visit(BB: MBB);
3512 for (const auto &MI : MBB.instrs())
3513 CV.visit(I: MI);
3514 }
3515
3516 if (CV.sawTokens()) {
3517 DT.recalculate(Func&: const_cast<MachineFunction &>(MF));
3518 CV.verify(DT);
3519 }
3520}
3521
3522void MachineVerifier::visitMachineFunctionAfter() {
3523 auto FailureCB = [this](const Twine &Message) {
3524 report(msg: Message.str().c_str(), MF);
3525 };
3526 verifyConvergenceControl(MF: *MF, DT, FailureCB, OS);
3527
3528 calcRegsPassed();
3529
3530 for (const MachineBasicBlock &MBB : *MF)
3531 checkPHIOps(MBB);
3532
3533 // Now check liveness info if available
3534 calcRegsRequired();
3535
3536 // Check for killed virtual registers that should be live out.
3537 for (const auto &MBB : *MF) {
3538 BBInfo &MInfo = MBBInfoMap[&MBB];
3539 for (Register VReg : MInfo.vregsRequired)
3540 if (MInfo.regsKilled.count(V: VReg)) {
3541 report(msg: "Virtual register killed in block, but needed live out.", MBB: &MBB);
3542 OS << "Virtual register " << printReg(Reg: VReg)
3543 << " is used after the block.\n";
3544 }
3545 }
3546
3547 if (!MF->empty()) {
3548 BBInfo &MInfo = MBBInfoMap[&MF->front()];
3549 for (Register VReg : MInfo.vregsRequired) {
3550 report(msg: "Virtual register defs don't dominate all uses.", MF);
3551 report_context_vreg(VReg);
3552 }
3553 }
3554
3555 if (LiveVars)
3556 verifyLiveVariables();
3557 if (LiveInts)
3558 verifyLiveIntervals();
3559
3560 // Check live-in list of each MBB. If a register is live into MBB, check
3561 // that the register is in regsLiveOut of each predecessor block. Since
3562 // this must come from a definition in the predecessor or its live-in
3563 // list, this will catch a live-through case where the predecessor does not
3564 // have the register in its live-in list. This currently only checks
3565 // registers that have no aliases, are not allocatable and are not
3566 // reserved, which could mean a condition code register for instance.
3567 if (MRI->tracksLiveness())
3568 for (const auto &MBB : *MF)
3569 for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) {
3570 MCRegister LiveInReg = P.PhysReg;
3571 bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid();
3572 if (hasAliases || isAllocatable(Reg: LiveInReg) || isReserved(Reg: LiveInReg))
3573 continue;
3574 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3575 BBInfo &PInfo = MBBInfoMap[Pred];
3576 if (!PInfo.regsLiveOut.count(V: LiveInReg)) {
3577 report(msg: "Live in register not found to be live out from predecessor.",
3578 MBB: &MBB);
3579 OS << TRI->getName(RegNo: LiveInReg) << " not found to be live out from "
3580 << printMBBReference(MBB: *Pred) << '\n';
3581 }
3582 }
3583 }
3584
3585 for (auto CSInfo : MF->getCallSitesInfo())
3586 if (!CSInfo.first->isCall())
3587 report(msg: "Call site info referencing instruction that is not call", MF);
3588
3589 // If there's debug-info, check that we don't have any duplicate value
3590 // tracking numbers.
3591 if (MF->getFunction().getSubprogram()) {
3592 DenseSet<unsigned> SeenNumbers;
3593 for (const auto &MBB : *MF) {
3594 for (const auto &MI : MBB) {
3595 if (auto Num = MI.peekDebugInstrNum()) {
3596 auto Result = SeenNumbers.insert(V: (unsigned)Num);
3597 if (!Result.second)
3598 report(msg: "Instruction has a duplicated value tracking number", MI: &MI);
3599 }
3600 }
3601 }
3602 }
3603}
3604
3605void MachineVerifier::verifyLiveVariables() {
3606 assert(LiveVars && "Don't call verifyLiveVariables without LiveVars");
3607 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3608 Register Reg = Register::index2VirtReg(Index: I);
3609 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
3610 for (const auto &MBB : *MF) {
3611 BBInfo &MInfo = MBBInfoMap[&MBB];
3612
3613 // Our vregsRequired should be identical to LiveVariables' AliveBlocks
3614 if (MInfo.vregsRequired.count(V: Reg)) {
3615 if (!VI.AliveBlocks.test(Idx: MBB.getNumber())) {
3616 report(msg: "LiveVariables: Block missing from AliveBlocks", MBB: &MBB);
3617 OS << "Virtual register " << printReg(Reg)
3618 << " must be live through the block.\n";
3619 }
3620 } else {
3621 if (VI.AliveBlocks.test(Idx: MBB.getNumber())) {
3622 report(msg: "LiveVariables: Block should not be in AliveBlocks", MBB: &MBB);
3623 OS << "Virtual register " << printReg(Reg)
3624 << " is not needed live through the block.\n";
3625 }
3626 }
3627 }
3628 }
3629}
3630
3631void MachineVerifier::verifyLiveIntervals() {
3632 assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
3633 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3634 Register Reg = Register::index2VirtReg(Index: I);
3635
3636 // Spilling and splitting may leave unused registers around. Skip them.
3637 if (MRI->reg_nodbg_empty(RegNo: Reg))
3638 continue;
3639
3640 if (!LiveInts->hasInterval(Reg)) {
3641 report(msg: "Missing live interval for virtual register", MF);
3642 OS << printReg(Reg, TRI) << " still has defs or uses\n";
3643 continue;
3644 }
3645
3646 const LiveInterval &LI = LiveInts->getInterval(Reg);
3647 assert(Reg == LI.reg() && "Invalid reg to interval mapping");
3648 verifyLiveInterval(LI);
3649 }
3650
3651 // Verify all the cached regunit intervals.
3652 for (MCRegUnit Unit : TRI->regunits())
3653 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3654 verifyLiveRange(*LR, VirtRegOrUnit(Unit));
3655}
3656
3657void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
3658 const VNInfo *VNI,
3659 VirtRegOrUnit VRegOrUnit,
3660 LaneBitmask LaneMask) {
3661 if (VNI->isUnused())
3662 return;
3663
3664 const VNInfo *DefVNI = LR.getVNInfoAt(Idx: VNI->def);
3665
3666 if (!DefVNI) {
3667 report(msg: "Value not live at VNInfo def and not marked unused", MF);
3668 report_context(LR, VRegOrUnit, LaneMask);
3669 report_context(VNI: *VNI);
3670 return;
3671 }
3672
3673 if (DefVNI != VNI) {
3674 report(msg: "Live segment at def has different VNInfo", MF);
3675 report_context(LR, VRegOrUnit, LaneMask);
3676 report_context(VNI: *VNI);
3677 return;
3678 }
3679
3680 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: VNI->def);
3681 if (!MBB) {
3682 report(msg: "Invalid VNInfo definition index", MF);
3683 report_context(LR, VRegOrUnit, LaneMask);
3684 report_context(VNI: *VNI);
3685 return;
3686 }
3687
3688 if (VNI->isPHIDef()) {
3689 if (VNI->def != LiveInts->getMBBStartIdx(mbb: MBB)) {
3690 report(msg: "PHIDef VNInfo is not defined at MBB start", MBB);
3691 report_context(LR, VRegOrUnit, LaneMask);
3692 report_context(VNI: *VNI);
3693 }
3694 return;
3695 }
3696
3697 // Non-PHI def.
3698 const MachineInstr *MI = LiveInts->getInstructionFromIndex(index: VNI->def);
3699 if (!MI) {
3700 report(msg: "No instruction at VNInfo def index", MBB);
3701 report_context(LR, VRegOrUnit, LaneMask);
3702 report_context(VNI: *VNI);
3703 return;
3704 }
3705
3706 bool hasDef = false;
3707 bool isEarlyClobber = false;
3708 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3709 if (!MOI->isReg() || !MOI->isDef())
3710 continue;
3711 if (VRegOrUnit.isVirtualReg()) {
3712 if (MOI->getReg() != VRegOrUnit.asVirtualReg())
3713 continue;
3714 } else {
3715 if (!MOI->getReg().isPhysical() ||
3716 !TRI->hasRegUnit(Reg: MOI->getReg(), RegUnit: VRegOrUnit.asMCRegUnit()))
3717 continue;
3718 }
3719 if (LaneMask.any() &&
3720 (TRI->getSubRegIndexLaneMask(SubIdx: MOI->getSubReg()) & LaneMask).none())
3721 continue;
3722 hasDef = true;
3723 if (MOI->isEarlyClobber())
3724 isEarlyClobber = true;
3725 }
3726
3727 if (!hasDef) {
3728 report(msg: "Defining instruction does not modify register", MI);
3729 report_context(LR, VRegOrUnit, LaneMask);
3730 report_context(VNI: *VNI);
3731 }
3732
3733 // Early clobber defs begin at USE slots, but other defs must begin at
3734 // DEF slots.
3735 if (isEarlyClobber) {
3736 if (!VNI->def.isEarlyClobber()) {
3737 report(msg: "Early clobber def must be at an early-clobber slot", MBB);
3738 report_context(LR, VRegOrUnit, LaneMask);
3739 report_context(VNI: *VNI);
3740 }
3741 } else if (!VNI->def.isRegister()) {
3742 report(msg: "Non-PHI, non-early clobber def must be at a register slot", MBB);
3743 report_context(LR, VRegOrUnit, LaneMask);
3744 report_context(VNI: *VNI);
3745 }
3746}
3747
3748void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
3749 const LiveRange::const_iterator I,
3750 VirtRegOrUnit VRegOrUnit,
3751 LaneBitmask LaneMask) {
3752 const LiveRange::Segment &S = *I;
3753 const VNInfo *VNI = S.valno;
3754 assert(VNI && "Live segment has no valno");
3755
3756 if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(ValNo: VNI->id)) {
3757 report(msg: "Foreign valno in live segment", MF);
3758 report_context(LR, VRegOrUnit, LaneMask);
3759 report_context(S);
3760 report_context(VNI: *VNI);
3761 }
3762
3763 if (VNI->isUnused()) {
3764 report(msg: "Live segment valno is marked unused", MF);
3765 report_context(LR, VRegOrUnit, LaneMask);
3766 report_context(S);
3767 }
3768
3769 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: S.start);
3770 if (!MBB) {
3771 report(msg: "Bad start of live segment, no basic block", MF);
3772 report_context(LR, VRegOrUnit, LaneMask);
3773 report_context(S);
3774 return;
3775 }
3776 SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(mbb: MBB);
3777 if (S.start != MBBStartIdx && S.start != VNI->def) {
3778 report(msg: "Live segment must begin at MBB entry or valno def", MBB);
3779 report_context(LR, VRegOrUnit, LaneMask);
3780 report_context(S);
3781 }
3782
3783 const MachineBasicBlock *EndMBB =
3784 LiveInts->getMBBFromIndex(index: S.end.getPrevSlot());
3785 if (!EndMBB) {
3786 report(msg: "Bad end of live segment, no basic block", MF);
3787 report_context(LR, VRegOrUnit, LaneMask);
3788 report_context(S);
3789 return;
3790 }
3791
3792 // Checks for non-live-out segments.
3793 if (S.end != LiveInts->getMBBEndIdx(mbb: EndMBB)) {
3794 // RegUnit intervals are allowed dead phis.
3795 if (!VRegOrUnit.isVirtualReg() && VNI->isPHIDef() && S.start == VNI->def &&
3796 S.end == VNI->def.getDeadSlot())
3797 return;
3798
3799 // The live segment is ending inside EndMBB
3800 const MachineInstr *MI =
3801 LiveInts->getInstructionFromIndex(index: S.end.getPrevSlot());
3802 if (!MI) {
3803 report(msg: "Live segment doesn't end at a valid instruction", MBB: EndMBB);
3804 report_context(LR, VRegOrUnit, LaneMask);
3805 report_context(S);
3806 return;
3807 }
3808
3809 // The block slot must refer to a basic block boundary.
3810 if (S.end.isBlock()) {
3811 report(msg: "Live segment ends at B slot of an instruction", MBB: EndMBB);
3812 report_context(LR, VRegOrUnit, LaneMask);
3813 report_context(S);
3814 }
3815
3816 if (S.end.isDead()) {
3817 // Segment ends on the dead slot.
3818 // That means there must be a dead def.
3819 if (!SlotIndex::isSameInstr(A: S.start, B: S.end)) {
3820 report(msg: "Live segment ending at dead slot spans instructions", MBB: EndMBB);
3821 report_context(LR, VRegOrUnit, LaneMask);
3822 report_context(S);
3823 }
3824 }
3825
3826 // After tied operands are rewritten, a live segment can only end at an
3827 // early-clobber slot if it is being redefined by an early-clobber def.
3828 // TODO: Before tied operands are rewritten, a live segment can only end at
3829 // an early-clobber slot if the last use is tied to an early-clobber def.
3830 if (MF->getProperties().hasTiedOpsRewritten() && S.end.isEarlyClobber()) {
3831 if (I + 1 == LR.end() || (I + 1)->start != S.end) {
3832 report(msg: "Live segment ending at early clobber slot must be "
3833 "redefined by an EC def in the same instruction",
3834 MBB: EndMBB);
3835 report_context(LR, VRegOrUnit, LaneMask);
3836 report_context(S);
3837 }
3838 }
3839
3840 // The following checks only apply to virtual registers. Physreg liveness
3841 // is too weird to check.
3842 if (VRegOrUnit.isVirtualReg()) {
3843 // A live segment can end with either a redefinition, a kill flag on a
3844 // use, or a dead flag on a def.
3845 bool hasRead = false;
3846 bool hasSubRegDef = false;
3847 bool hasDeadDef = false;
3848 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3849 if (!MOI->isReg() || MOI->getReg() != VRegOrUnit.asVirtualReg())
3850 continue;
3851 unsigned Sub = MOI->getSubReg();
3852 LaneBitmask SLM =
3853 Sub != 0 ? TRI->getSubRegIndexLaneMask(SubIdx: Sub) : LaneBitmask::getAll();
3854 if (MOI->isDef()) {
3855 if (Sub != 0) {
3856 hasSubRegDef = true;
3857 // An operand %0:sub0 reads %0:sub1..n. Invert the lane
3858 // mask for subregister defs. Read-undef defs will be handled by
3859 // readsReg below.
3860 SLM = ~SLM;
3861 }
3862 if (MOI->isDead())
3863 hasDeadDef = true;
3864 }
3865 if (LaneMask.any() && (LaneMask & SLM).none())
3866 continue;
3867 if (MOI->readsReg())
3868 hasRead = true;
3869 }
3870 if (S.end.isDead()) {
3871 // Make sure that the corresponding machine operand for a "dead" live
3872 // range has the dead flag. We cannot perform this check for subregister
3873 // liveranges as partially dead values are allowed.
3874 if (LaneMask.none() && !hasDeadDef) {
3875 report(
3876 msg: "Instruction ending live segment on dead slot has no dead flag",
3877 MI);
3878 report_context(LR, VRegOrUnit, LaneMask);
3879 report_context(S);
3880 }
3881 } else {
3882 if (!hasRead) {
3883 // When tracking subregister liveness, the main range must start new
3884 // values on partial register writes, even if there is no read.
3885 if (!MRI->shouldTrackSubRegLiveness(VReg: VRegOrUnit.asVirtualReg()) ||
3886 LaneMask.any() || !hasSubRegDef) {
3887 report(msg: "Instruction ending live segment doesn't read the register",
3888 MI);
3889 report_context(LR, VRegOrUnit, LaneMask);
3890 report_context(S);
3891 }
3892 }
3893 }
3894 }
3895 }
3896
3897 // Now check all the basic blocks in this live segment.
3898 MachineFunction::const_iterator MFI = MBB->getIterator();
3899 // Is this live segment the beginning of a non-PHIDef VN?
3900 if (S.start == VNI->def && !VNI->isPHIDef()) {
3901 // Not live-in to any blocks.
3902 if (MBB == EndMBB)
3903 return;
3904 // Skip this block.
3905 ++MFI;
3906 }
3907
3908 SmallVector<SlotIndex, 4> Undefs;
3909 if (LaneMask.any()) {
3910 LiveInterval &OwnerLI = LiveInts->getInterval(Reg: VRegOrUnit.asVirtualReg());
3911 OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, MRI: *MRI, Indexes: *Indexes);
3912 }
3913
3914 while (true) {
3915 assert(LiveInts->isLiveInToMBB(LR, &*MFI));
3916 // We don't know how to track physregs into a landing pad.
3917 if (!VRegOrUnit.isVirtualReg() && MFI->isEHPad()) {
3918 if (&*MFI == EndMBB)
3919 break;
3920 ++MFI;
3921 continue;
3922 }
3923
3924 // Is VNI a PHI-def in the current block?
3925 bool IsPHI = VNI->isPHIDef() &&
3926 VNI->def == LiveInts->getMBBStartIdx(mbb: &*MFI);
3927
3928 // Check that VNI is live-out of all predecessors.
3929 for (const MachineBasicBlock *Pred : MFI->predecessors()) {
3930 SlotIndex PEnd = LiveInts->getMBBEndIdx(mbb: Pred);
3931 // Predecessor of landing pad live-out on last call.
3932 if (MFI->isEHPad()) {
3933 for (const MachineInstr &MI : llvm::reverse(C: *Pred)) {
3934 if (MI.isCall()) {
3935 PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex();
3936 break;
3937 }
3938 }
3939 }
3940 const VNInfo *PVNI = LR.getVNInfoBefore(Idx: PEnd);
3941
3942 // All predecessors must have a live-out value. However for a phi
3943 // instruction with subregister intervals
3944 // only one of the subregisters (not necessarily the current one) needs to
3945 // be defined.
3946 if (!PVNI && (LaneMask.none() || !IsPHI)) {
3947 if (LiveRangeCalc::isJointlyDominated(MBB: Pred, Defs: Undefs, Indexes: *Indexes))
3948 continue;
3949 report(msg: "Register not marked live out of predecessor", MBB: Pred);
3950 report_context(LR, VRegOrUnit, LaneMask);
3951 report_context(VNI: *VNI);
3952 OS << " live into " << printMBBReference(MBB: *MFI) << '@'
3953 << LiveInts->getMBBStartIdx(mbb: &*MFI) << ", not live before " << PEnd
3954 << '\n';
3955 continue;
3956 }
3957
3958 // Only PHI-defs can take different predecessor values.
3959 if (!IsPHI && PVNI != VNI) {
3960 report(msg: "Different value live out of predecessor", MBB: Pred);
3961 report_context(LR, VRegOrUnit, LaneMask);
3962 OS << "Valno #" << PVNI->id << " live out of "
3963 << printMBBReference(MBB: *Pred) << '@' << PEnd << "\nValno #" << VNI->id
3964 << " live into " << printMBBReference(MBB: *MFI) << '@'
3965 << LiveInts->getMBBStartIdx(mbb: &*MFI) << '\n';
3966 }
3967 }
3968 if (&*MFI == EndMBB)
3969 break;
3970 ++MFI;
3971 }
3972}
3973
3974void MachineVerifier::verifyLiveRange(const LiveRange &LR,
3975 VirtRegOrUnit VRegOrUnit,
3976 LaneBitmask LaneMask) {
3977 for (const VNInfo *VNI : LR.valnos)
3978 verifyLiveRangeValue(LR, VNI, VRegOrUnit, LaneMask);
3979
3980 for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
3981 verifyLiveRangeSegment(LR, I, VRegOrUnit, LaneMask);
3982}
3983
3984void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
3985 Register Reg = LI.reg();
3986 assert(Reg.isVirtual());
3987 verifyLiveRange(LR: LI, VRegOrUnit: VirtRegOrUnit(Reg));
3988
3989 if (LI.hasSubRanges()) {
3990 LaneBitmask Mask;
3991 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
3992 for (const LiveInterval::SubRange &SR : LI.subranges()) {
3993 if ((Mask & SR.LaneMask).any()) {
3994 report(msg: "Lane masks of sub ranges overlap in live interval", MF);
3995 report_context(LI);
3996 }
3997 if ((SR.LaneMask & ~MaxMask).any()) {
3998 report(msg: "Subrange lanemask is invalid", MF);
3999 report_context(LI);
4000 }
4001 if (SR.empty()) {
4002 report(msg: "Subrange must not be empty", MF);
4003 report_context(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
4004 }
4005 Mask |= SR.LaneMask;
4006 verifyLiveRange(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
4007 if (!LI.covers(Other: SR)) {
4008 report(msg: "A Subrange is not covered by the main range", MF);
4009 report_context(LI);
4010 }
4011 }
4012 }
4013
4014 // Check the LI only has one connected component.
4015 ConnectedVNInfoEqClasses ConEQ(*LiveInts);
4016 unsigned NumComp = ConEQ.Classify(LR: LI);
4017 if (NumComp > 1) {
4018 report(msg: "Multiple connected components in live interval", MF);
4019 report_context(LI);
4020 for (unsigned comp = 0; comp != NumComp; ++comp) {
4021 OS << comp << ": valnos";
4022 for (const VNInfo *I : LI.valnos)
4023 if (comp == ConEQ.getEqClass(VNI: I))
4024 OS << ' ' << I->id;
4025 OS << '\n';
4026 }
4027 }
4028}
4029
4030namespace {
4031
4032 // FrameSetup and FrameDestroy can have zero adjustment, so using a single
4033 // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the
4034 // value is zero.
4035 // We use a bool plus an integer to capture the stack state.
4036struct StackStateOfBB {
4037 StackStateOfBB() = default;
4038 StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup)
4039 : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),
4040 ExitIsSetup(ExitSetup) {}
4041
4042 // Can be negative, which means we are setting up a frame.
4043 int EntryValue = 0;
4044 int ExitValue = 0;
4045 bool EntryIsSetup = false;
4046 bool ExitIsSetup = false;
4047};
4048
4049} // end anonymous namespace
4050
4051/// Make sure on every path through the CFG, a FrameSetup <n> is always followed
4052/// by a FrameDestroy <n>, stack adjustments are identical on all
4053/// CFG edges to a merge point, and frame is destroyed at end of a return block.
4054void MachineVerifier::verifyStackFrame() {
4055 unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
4056 unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
4057 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
4058 return;
4059
4060 SmallVector<StackStateOfBB, 8> SPState;
4061 SPState.resize(N: MF->getNumBlockIDs());
4062 df_iterator_default_set<const MachineBasicBlock*> Reachable;
4063
4064 // Visit the MBBs in DFS order.
4065 for (df_ext_iterator<const MachineFunction *,
4066 df_iterator_default_set<const MachineBasicBlock *>>
4067 DFI = df_ext_begin(G: MF, S&: Reachable), DFE = df_ext_end(G: MF, S&: Reachable);
4068 DFI != DFE; ++DFI) {
4069 const MachineBasicBlock *MBB = *DFI;
4070
4071 StackStateOfBB BBState;
4072 // Check the exit state of the DFS stack predecessor.
4073 if (DFI.getPathLength() >= 2) {
4074 const MachineBasicBlock *StackPred = DFI.getPath(n: DFI.getPathLength() - 2);
4075 assert(Reachable.count(StackPred) &&
4076 "DFS stack predecessor is already visited.\n");
4077 BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;
4078 BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;
4079 BBState.ExitValue = BBState.EntryValue;
4080 BBState.ExitIsSetup = BBState.EntryIsSetup;
4081 }
4082
4083 if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) {
4084 report(msg: "Call frame size on entry does not match value computed from "
4085 "predecessor",
4086 MBB);
4087 OS << "Call frame size on entry " << MBB->getCallFrameSize()
4088 << " does not match value computed from predecessor "
4089 << -BBState.EntryValue << '\n';
4090 }
4091
4092 // Update stack state by checking contents of MBB.
4093 for (const auto &I : *MBB) {
4094 if (I.getOpcode() == FrameSetupOpcode) {
4095 if (BBState.ExitIsSetup)
4096 report(msg: "FrameSetup is after another FrameSetup", MI: &I);
4097 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4098 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4099 "instruction.", MI: &I);
4100 BBState.ExitValue -= TII->getFrameTotalSize(I);
4101 BBState.ExitIsSetup = true;
4102 }
4103
4104 if (I.getOpcode() == FrameDestroyOpcode) {
4105 int Size = TII->getFrameTotalSize(I);
4106 if (!BBState.ExitIsSetup)
4107 report(msg: "FrameDestroy is not after a FrameSetup", MI: &I);
4108 int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :
4109 BBState.ExitValue;
4110 if (BBState.ExitIsSetup && AbsSPAdj != Size) {
4111 report(msg: "FrameDestroy <n> is after FrameSetup <m>", MI: &I);
4112 OS << "FrameDestroy <" << Size << "> is after FrameSetup <"
4113 << AbsSPAdj << ">.\n";
4114 }
4115 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4116 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4117 "instruction.", MI: &I);
4118 BBState.ExitValue += Size;
4119 BBState.ExitIsSetup = false;
4120 }
4121 }
4122 SPState[MBB->getNumber()] = BBState;
4123
4124 // Make sure the exit state of any predecessor is consistent with the entry
4125 // state.
4126 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
4127 if (Reachable.count(Ptr: Pred) &&
4128 (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue ||
4129 SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {
4130 report(msg: "The exit stack state of a predecessor is inconsistent.", MBB);
4131 OS << "Predecessor " << printMBBReference(MBB: *Pred) << " has exit state ("
4132 << SPState[Pred->getNumber()].ExitValue << ", "
4133 << SPState[Pred->getNumber()].ExitIsSetup << "), while "
4134 << printMBBReference(MBB: *MBB) << " has entry state ("
4135 << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";
4136 }
4137 }
4138
4139 // Make sure the entry state of any successor is consistent with the exit
4140 // state.
4141 for (const MachineBasicBlock *Succ : MBB->successors()) {
4142 if (Reachable.count(Ptr: Succ) &&
4143 (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue ||
4144 SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {
4145 report(msg: "The entry stack state of a successor is inconsistent.", MBB);
4146 OS << "Successor " << printMBBReference(MBB: *Succ) << " has entry state ("
4147 << SPState[Succ->getNumber()].EntryValue << ", "
4148 << SPState[Succ->getNumber()].EntryIsSetup << "), while "
4149 << printMBBReference(MBB: *MBB) << " has exit state ("
4150 << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";
4151 }
4152 }
4153
4154 // Make sure a basic block with return ends with zero stack adjustment.
4155 if (!MBB->empty() && MBB->back().isReturn()) {
4156 if (BBState.ExitIsSetup)
4157 report(msg: "A return block ends with a FrameSetup.", MBB);
4158 if (BBState.ExitValue)
4159 report(msg: "A return block ends with a nonzero stack adjustment.", MBB);
4160 }
4161 }
4162}
4163
4164void MachineVerifier::verifyStackProtector() {
4165 const MachineFrameInfo &MFI = MF->getFrameInfo();
4166 if (!MFI.hasStackProtectorIndex())
4167 return;
4168 // Only applicable when the offsets of frame objects have been determined,
4169 // which is indicated by a non-zero stack size.
4170 if (!MFI.getStackSize())
4171 return;
4172 const TargetFrameLowering &TFI = *MF->getSubtarget().getFrameLowering();
4173 bool StackGrowsDown =
4174 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
4175 unsigned FI = MFI.getStackProtectorIndex();
4176 int64_t SPStart = MFI.getObjectOffset(ObjectIdx: FI);
4177 int64_t SPEnd = SPStart + MFI.getObjectSize(ObjectIdx: FI);
4178 for (unsigned I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
4179 if (I == FI)
4180 continue;
4181 if (MFI.isDeadObjectIndex(ObjectIdx: I))
4182 continue;
4183 // FIXME: Skip non-default stack objects, as some targets may place them
4184 // above the stack protector. This is a workaround for the fact that
4185 // backends such as AArch64 may place SVE stack objects *above* the stack
4186 // protector.
4187 if (MFI.getStackID(ObjectIdx: I) != TargetStackID::Default)
4188 continue;
4189 // Skip variable-sized objects because they do not have a fixed offset.
4190 if (MFI.isVariableSizedObjectIndex(ObjectIdx: I))
4191 continue;
4192 // FIXME: Skip spill slots which may be allocated above the stack protector.
4193 // Ideally this would only skip callee-saved registers, but we don't have
4194 // that information here. For example, spill-slots used for scavenging are
4195 // not described in CalleeSavedInfo.
4196 if (MFI.isSpillSlotObjectIndex(ObjectIdx: I))
4197 continue;
4198 int64_t ObjStart = MFI.getObjectOffset(ObjectIdx: I);
4199 int64_t ObjEnd = ObjStart + MFI.getObjectSize(ObjectIdx: I);
4200 if (SPStart < ObjEnd && ObjStart < SPEnd) {
4201 report(msg: "Stack protector overlaps with another stack object", MF);
4202 break;
4203 }
4204 if ((StackGrowsDown && SPStart <= ObjStart) ||
4205 (!StackGrowsDown && SPStart >= ObjStart)) {
4206 report(msg: "Stack protector is not the top-most object on the stack", MF);
4207 break;
4208 }
4209 }
4210}
4211