1//===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Pass to verify generated machine code. The following is checked:
10//
11// Operand counts: All explicit operands must be present.
12//
13// Register classes: All physical and virtual register operands must be
14// compatible with the register class required by the instruction descriptor.
15//
16// Register live intervals: Registers must be defined only once, and must be
17// defined before use.
18//
19// The machine code verifier is enabled with the command-line option
20// -verify-machineinstrs.
21//===----------------------------------------------------------------------===//
22
23#include "llvm/CodeGen/MachineVerifier.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/ADT/DenseSet.h"
27#include "llvm/ADT/DepthFirstIterator.h"
28#include "llvm/ADT/PostOrderIterator.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SetOperations.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/StringRef.h"
34#include "llvm/ADT/Twine.h"
35#include "llvm/CodeGen/CodeGenCommonISel.h"
36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
37#include "llvm/CodeGen/LiveInterval.h"
38#include "llvm/CodeGen/LiveIntervals.h"
39#include "llvm/CodeGen/LiveRangeCalc.h"
40#include "llvm/CodeGen/LiveStacks.h"
41#include "llvm/CodeGen/MachineBasicBlock.h"
42#include "llvm/CodeGen/MachineConvergenceVerifier.h"
43#include "llvm/CodeGen/MachineDominators.h"
44#include "llvm/CodeGen/MachineFrameInfo.h"
45#include "llvm/CodeGen/MachineFunction.h"
46#include "llvm/CodeGen/MachineFunctionPass.h"
47#include "llvm/CodeGen/MachineInstr.h"
48#include "llvm/CodeGen/MachineInstrBundle.h"
49#include "llvm/CodeGen/MachineMemOperand.h"
50#include "llvm/CodeGen/MachineOperand.h"
51#include "llvm/CodeGen/MachineRegisterInfo.h"
52#include "llvm/CodeGen/PseudoSourceValue.h"
53#include "llvm/CodeGen/RegisterBank.h"
54#include "llvm/CodeGen/RegisterBankInfo.h"
55#include "llvm/CodeGen/SlotIndexes.h"
56#include "llvm/CodeGen/StackMaps.h"
57#include "llvm/CodeGen/TargetInstrInfo.h"
58#include "llvm/CodeGen/TargetLowering.h"
59#include "llvm/CodeGen/TargetOpcodes.h"
60#include "llvm/CodeGen/TargetRegisterInfo.h"
61#include "llvm/CodeGen/TargetSubtargetInfo.h"
62#include "llvm/CodeGenTypes/LowLevelType.h"
63#include "llvm/IR/BasicBlock.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/EHPersonalities.h"
66#include "llvm/IR/Function.h"
67#include "llvm/IR/InlineAsm.h"
68#include "llvm/IR/Instructions.h"
69#include "llvm/InitializePasses.h"
70#include "llvm/MC/LaneBitmask.h"
71#include "llvm/MC/MCAsmInfo.h"
72#include "llvm/MC/MCDwarf.h"
73#include "llvm/MC/MCInstrDesc.h"
74#include "llvm/MC/MCRegisterInfo.h"
75#include "llvm/MC/MCTargetOptions.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/ErrorHandling.h"
79#include "llvm/Support/ManagedStatic.h"
80#include "llvm/Support/MathExtras.h"
81#include "llvm/Support/ModRef.h"
82#include "llvm/Support/Mutex.h"
83#include "llvm/Support/raw_ostream.h"
84#include "llvm/Target/TargetMachine.h"
85#include <algorithm>
86#include <cassert>
87#include <cstddef>
88#include <cstdint>
89#include <iterator>
90#include <string>
91#include <utility>
92
93using namespace llvm;
94
95namespace {
96
97/// Used the by the ReportedErrors class to guarantee only one error is reported
98/// at one time.
99static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock;
100
101static bool hasPhysRegClassForType(const TargetRegisterInfo &TRI,
102 MCRegister Reg, LLT Ty) {
103 assert(Reg.isPhysical() && "reg must be a physical register");
104 assert(Ty.isValid() && "expected a valid type");
105
106 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
107 if (TRI.isTypeLegalForClass(RC: *RC, T: Ty))
108 return true;
109
110 return llvm::any_of(Range: TRI.regclasses(), P: [&](const TargetRegisterClass &RC) {
111 return RC.contains(Reg) && TRI.isTypeLegalForClass(RC, T: Ty);
112 });
113}
114
115struct MachineVerifier {
116 MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b,
117 raw_ostream *OS, bool AbortOnError = true)
118 : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b),
119 ReportedErrs(AbortOnError) {}
120
121 MachineVerifier(Pass *pass, const char *b, raw_ostream *OS,
122 bool AbortOnError = true)
123 : PASS(pass), OS(OS ? *OS : nulls()), Banner(b),
124 ReportedErrs(AbortOnError) {}
125
126 MachineVerifier(const char *b, LiveIntervals *LiveInts, LiveStacks *LiveStks,
127 SlotIndexes *Indexes, raw_ostream *OS,
128 bool AbortOnError = true)
129 : OS(OS ? *OS : nulls()), Banner(b), LiveInts(LiveInts),
130 LiveStks(LiveStks), Indexes(Indexes), ReportedErrs(AbortOnError) {}
131
132 /// \returns true if no problems were found.
133 bool verify(const MachineFunction &MF);
134
135 MachineFunctionAnalysisManager *MFAM = nullptr;
136 Pass *const PASS = nullptr;
137 raw_ostream &OS;
138 const char *Banner;
139 const MachineFunction *MF = nullptr;
140 const TargetMachine *TM = nullptr;
141 const TargetInstrInfo *TII = nullptr;
142 const TargetRegisterInfo *TRI = nullptr;
143 const MachineRegisterInfo *MRI = nullptr;
144 const RegisterBankInfo *RBI = nullptr;
145
146 // Avoid querying the MachineFunctionProperties for each operand.
147 bool isFunctionRegBankSelected = false;
148 bool isFunctionSelected = false;
149 bool isFunctionTracksDebugUserValues = false;
150
151 using RegVector = SmallVector<Register, 16>;
152 using RegMaskVector = SmallVector<const uint32_t *, 4>;
153 using RegSet = DenseSet<Register>;
154 using RegMap = DenseMap<Register, const MachineInstr *>;
155 using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;
156
157 const MachineInstr *FirstNonPHI = nullptr;
158 const MachineInstr *FirstTerminator = nullptr;
159 BlockSet FunctionBlocks;
160
161 BitVector regsReserved;
162 RegSet regsLive;
163 RegVector regsDefined, regsDead, regsKilled;
164 RegMaskVector regMasks;
165
166 SlotIndex lastIndex;
167
168 // Add Reg and any sub-registers to RV
169 void addRegWithSubRegs(RegVector &RV, Register Reg) {
170 RV.push_back(Elt: Reg);
171 if (Reg.isPhysical())
172 append_range(C&: RV, R: TRI->subregs(Reg: Reg.asMCReg()));
173 }
174
175 struct BBInfo {
176 // Is this MBB reachable from the MF entry point?
177 bool reachable = false;
178
179 // Vregs that must be live in because they are used without being
180 // defined. Map value is the user. vregsLiveIn doesn't include regs
181 // that only are used by PHI nodes.
182 RegMap vregsLiveIn;
183
184 // Regs killed in MBB. They may be defined again, and will then be in both
185 // regsKilled and regsLiveOut.
186 RegSet regsKilled;
187
188 // Regs defined in MBB and live out. Note that vregs passing through may
189 // be live out without being mentioned here.
190 RegSet regsLiveOut;
191
192 // Vregs that pass through MBB untouched. This set is disjoint from
193 // regsKilled and regsLiveOut.
194 RegSet vregsPassed;
195
196 // Vregs that must pass through MBB because they are needed by a successor
197 // block. This set is disjoint from regsLiveOut.
198 RegSet vregsRequired;
199
200 // Set versions of block's predecessor and successor lists.
201 BlockSet Preds, Succs;
202
203 BBInfo() = default;
204
205 // Add register to vregsRequired if it belongs there. Return true if
206 // anything changed.
207 bool addRequired(Register Reg) {
208 if (!Reg.isVirtual())
209 return false;
210 if (regsLiveOut.count(V: Reg))
211 return false;
212 return vregsRequired.insert(V: Reg).second;
213 }
214
215 // Same for a full set.
216 bool addRequired(const RegSet &RS) {
217 bool Changed = false;
218 for (Register Reg : RS)
219 Changed |= addRequired(Reg);
220 return Changed;
221 }
222
223 // Same for a full map.
224 bool addRequired(const RegMap &RM) {
225 bool Changed = false;
226 for (const auto &I : RM)
227 Changed |= addRequired(Reg: I.first);
228 return Changed;
229 }
230
231 // Live-out registers are either in regsLiveOut or vregsPassed.
232 bool isLiveOut(Register Reg) const {
233 return regsLiveOut.count(V: Reg) || vregsPassed.count(V: Reg);
234 }
235 };
236
237 // Extra register info per MBB.
238 DenseMap<const MachineBasicBlock *, BBInfo> MBBInfoMap;
239
240 bool isReserved(Register Reg) {
241 return Reg.id() < regsReserved.size() && regsReserved.test(Idx: Reg.id());
242 }
243
244 bool isAllocatable(Register Reg) const {
245 return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(RegNo: Reg) &&
246 !regsReserved.test(Idx: Reg.id());
247 }
248
249 // Analysis information if available
250 LiveIntervals *LiveInts = nullptr;
251 LiveStacks *LiveStks = nullptr;
252 SlotIndexes *Indexes = nullptr;
253
254 /// A class to track the number of reported error and to guarantee that only
255 /// one error is reported at one time.
256 class ReportedErrors {
257 unsigned NumReported = 0;
258 bool AbortOnError;
259
260 public:
261 /// \param AbortOnError -- If set, abort after printing the first error.
262 ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {}
263
264 ~ReportedErrors() {
265 if (!hasError())
266 return;
267 if (AbortOnError)
268 report_fatal_error(reason: "Found " + Twine(NumReported) +
269 " machine code errors.");
270 // Since we haven't aborted, release the lock to allow other threads to
271 // report errors.
272 ReportedErrorsLock->unlock();
273 }
274
275 /// Increment the number of reported errors.
276 /// \returns true if this is the first reported error.
277 bool increment() {
278 // If this is the first error this thread has encountered, grab the lock
279 // to prevent other threads from reporting errors at the same time.
280 // Otherwise we assume we already have the lock.
281 if (!hasError())
282 ReportedErrorsLock->lock();
283 ++NumReported;
284 return NumReported == 1;
285 }
286
287 /// \returns true if an error was reported.
288 bool hasError() { return NumReported; }
289 };
290 ReportedErrors ReportedErrs;
291
292 // This is calculated only when trying to verify convergence control tokens.
293 // Similar to the LLVM IR verifier, we calculate this locally instead of
294 // relying on the pass manager.
295 MachineDominatorTree DT;
296
297 void visitMachineFunctionBefore();
298 void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
299 void visitMachineBundleBefore(const MachineInstr *MI);
300
301 /// Verify that all of \p MI's virtual register operands are scalars.
302 /// \returns True if all virtual register operands are scalar. False
303 /// otherwise.
304 bool verifyAllRegOpsScalar(const MachineInstr &MI,
305 const MachineRegisterInfo &MRI);
306 bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);
307
308 bool verifyGIntrinsicSideEffects(const MachineInstr *MI);
309 bool verifyGIntrinsicConvergence(const MachineInstr *MI);
310 void verifyPreISelGenericInstruction(const MachineInstr *MI);
311
312 void visitMachineInstrBefore(const MachineInstr *MI);
313 void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
314 void visitMachineBundleAfter(const MachineInstr *MI);
315 void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);
316 void visitMachineFunctionAfter();
317
318 void report(const char *msg, const MachineFunction *MF);
319 void report(const char *msg, const MachineBasicBlock *MBB);
320 void report(const char *msg, const MachineInstr *MI);
321 void report(const char *msg, const MachineOperand *MO, unsigned MONum,
322 LLT MOVRegType = LLT{});
323 void report(const Twine &Msg, const MachineInstr *MI);
324
325 void report_context(const LiveInterval &LI) const;
326 void report_context(const LiveRange &LR, VirtRegOrUnit VRegOrUnit,
327 LaneBitmask LaneMask) const;
328 void report_context(const LiveRange::Segment &S) const;
329 void report_context(const VNInfo &VNI) const;
330 void report_context(SlotIndex Pos) const;
331 void report_context(MCPhysReg PhysReg) const;
332 void report_context_liverange(const LiveRange &LR) const;
333 void report_context_lanemask(LaneBitmask LaneMask) const;
334 void report_context_vreg(Register VReg) const;
335 void report_context_vreg_regunit(VirtRegOrUnit VRegOrUnit) const;
336
337 void verifyInlineAsm(const MachineInstr *MI);
338
339 void checkLiveness(const MachineOperand *MO, unsigned MONum);
340 void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
341 SlotIndex UseIdx, const LiveRange &LR,
342 VirtRegOrUnit VRegOrUnit,
343 LaneBitmask LaneMask = LaneBitmask::getNone());
344 void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
345 SlotIndex DefIdx, const LiveRange &LR,
346 VirtRegOrUnit VRegOrUnit, bool SubRangeCheck = false,
347 LaneBitmask LaneMask = LaneBitmask::getNone());
348
349 void markReachable(const MachineBasicBlock *MBB);
350 void calcRegsPassed();
351 void checkPHIOps(const MachineBasicBlock &MBB);
352
353 void calcRegsRequired();
354 void verifyLiveIntervals();
355 void verifyLiveInterval(const LiveInterval &);
356 void verifyLiveRangeValue(const LiveRange &, const VNInfo *, VirtRegOrUnit,
357 LaneBitmask);
358 void verifyLiveRangeSegment(const LiveRange &,
359 const LiveRange::const_iterator I, VirtRegOrUnit,
360 LaneBitmask);
361 void verifyLiveRange(const LiveRange &, VirtRegOrUnit,
362 LaneBitmask LaneMask = LaneBitmask::getNone());
363
364 void verifyStackFrame();
365 /// Check that the stack protector is the top-most object in the stack.
366 void verifyStackProtector();
367
368 void verifySlotIndexes() const;
369 void verifyProperties(const MachineFunction &MF);
370};
371
372struct MachineVerifierLegacyPass : public MachineFunctionPass {
373 static char ID; // Pass ID, replacement for typeid
374
375 const std::string Banner;
376
377 MachineVerifierLegacyPass(std::string banner = std::string())
378 : MachineFunctionPass(ID), Banner(std::move(banner)) {}
379
380 void getAnalysisUsage(AnalysisUsage &AU) const override {
381 AU.addUsedIfAvailable<LiveStacksWrapperLegacy>();
382 AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
383 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
384 AU.setPreservesAll();
385 MachineFunctionPass::getAnalysisUsage(AU);
386 }
387
388 bool runOnMachineFunction(MachineFunction &MF) override {
389 // Skip functions that have known verification problems.
390 // FIXME: Remove this mechanism when all problematic passes have been
391 // fixed.
392 if (MF.getProperties().hasFailsVerification())
393 return false;
394
395 MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);
396 return false;
397 }
398};
399
400} // end anonymous namespace
401
402PreservedAnalyses
403MachineVerifierPass::run(MachineFunction &MF,
404 MachineFunctionAnalysisManager &MFAM) {
405 // Skip functions that have known verification problems.
406 // FIXME: Remove this mechanism when all problematic passes have been
407 // fixed.
408 if (MF.getProperties().hasFailsVerification())
409 return PreservedAnalyses::all();
410 MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);
411 return PreservedAnalyses::all();
412}
413
414char MachineVerifierLegacyPass::ID = 0;
415
416INITIALIZE_PASS(MachineVerifierLegacyPass, "machineverifier",
417 "Verify generated machine code", false, false)
418
419FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {
420 return new MachineVerifierLegacyPass(Banner);
421}
422
423void llvm::verifyMachineFunction(const std::string &Banner,
424 const MachineFunction &MF) {
425 // TODO: Use MFAM after porting below analyses.
426 // LiveIntervals *LiveInts;
427 // LiveStacks *LiveStks;
428 // SlotIndexes *Indexes;
429 MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);
430}
431
432bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,
433 bool AbortOnError) const {
434 return MachineVerifier(p, Banner, OS, AbortOnError).verify(MF: *this);
435}
436
437bool MachineFunction::verify(MachineFunctionAnalysisManager &MFAM,
438 const char *Banner, raw_ostream *OS,
439 bool AbortOnError) const {
440 return MachineVerifier(MFAM, Banner, OS, AbortOnError).verify(MF: *this);
441}
442
443bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
444 const char *Banner, raw_ostream *OS,
445 bool AbortOnError) const {
446 return MachineVerifier(Banner, LiveInts, /*LiveStks=*/nullptr, Indexes, OS,
447 AbortOnError)
448 .verify(MF: *this);
449}
450
451void MachineVerifier::verifySlotIndexes() const {
452 if (Indexes == nullptr)
453 return;
454
455 // Ensure the IdxMBB list is sorted by slot indexes.
456 SlotIndex Last;
457 for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),
458 E = Indexes->MBBIndexEnd(); I != E; ++I) {
459 assert(!Last.isValid() || I->first > Last);
460 Last = I->first;
461 }
462}
463
464void MachineVerifier::verifyProperties(const MachineFunction &MF) {
465 // If a pass has introduced virtual registers without clearing the
466 // NoVRegs property (or set it without allocating the vregs)
467 // then report an error.
468 if (MF.getProperties().hasNoVRegs() && MRI->getNumVirtRegs())
469 report(msg: "Function has NoVRegs property but there are VReg operands", MF: &MF);
470}
471
472bool MachineVerifier::verify(const MachineFunction &MF) {
473 this->MF = &MF;
474 TM = &MF.getTarget();
475 TII = MF.getSubtarget().getInstrInfo();
476 TRI = MF.getSubtarget().getRegisterInfo();
477 RBI = MF.getSubtarget().getRegBankInfo();
478 MRI = &MF.getRegInfo();
479
480 const MachineFunctionProperties &Props = MF.getProperties();
481 const bool isFunctionFailedISel = Props.hasFailedISel();
482
483 // If we're mid-GlobalISel and we already triggered the fallback path then
484 // it's expected that the MIR is somewhat broken but that's ok since we'll
485 // reset it and clear the FailedISel attribute in ResetMachineFunctions.
486 if (isFunctionFailedISel)
487 return true;
488
489 isFunctionRegBankSelected = Props.hasRegBankSelected();
490 isFunctionSelected = Props.hasSelected();
491 isFunctionTracksDebugUserValues = Props.hasTracksDebugUserValues();
492
493 if (PASS) {
494 auto *LISWrapper = PASS->getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
495 LiveInts = LISWrapper ? &LISWrapper->getLIS() : nullptr;
496 auto *LSWrapper = PASS->getAnalysisIfAvailable<LiveStacksWrapperLegacy>();
497 LiveStks = LSWrapper ? &LSWrapper->getLS() : nullptr;
498 auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>();
499 Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
500 }
501 if (MFAM) {
502 MachineFunction &Func = const_cast<MachineFunction &>(MF);
503 LiveInts = MFAM->getCachedResult<LiveIntervalsAnalysis>(IR&: Func);
504 // TODO: LiveStks = MFAM->getCachedResult<LiveStacksAnalysis>(Func);
505 Indexes = MFAM->getCachedResult<SlotIndexesAnalysis>(IR&: Func);
506 }
507
508 verifySlotIndexes();
509
510 verifyProperties(MF);
511
512 visitMachineFunctionBefore();
513 for (const MachineBasicBlock &MBB : MF) {
514 visitMachineBasicBlockBefore(MBB: &MBB);
515 // Keep track of the current bundle header.
516 const MachineInstr *CurBundle = nullptr;
517 // Do we expect the next instruction to be part of the same bundle?
518 bool InBundle = false;
519
520 for (const MachineInstr &MI : MBB.instrs()) {
521 if (MI.getParent() != &MBB) {
522 report(msg: "Bad instruction parent pointer", MBB: &MBB);
523 OS << "Instruction: " << MI;
524 continue;
525 }
526
527 // Check for consistent bundle flags.
528 if (InBundle && !MI.isBundledWithPred())
529 report(msg: "Missing BundledPred flag, "
530 "BundledSucc was set on predecessor",
531 MI: &MI);
532 if (!InBundle && MI.isBundledWithPred())
533 report(msg: "BundledPred flag is set, "
534 "but BundledSucc not set on predecessor",
535 MI: &MI);
536
537 // Is this a bundle header?
538 if (!MI.isInsideBundle()) {
539 if (CurBundle)
540 visitMachineBundleAfter(MI: CurBundle);
541 CurBundle = &MI;
542 visitMachineBundleBefore(MI: CurBundle);
543 } else if (!CurBundle)
544 report(msg: "No bundle header", MI: &MI);
545 visitMachineInstrBefore(MI: &MI);
546 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
547 const MachineOperand &Op = MI.getOperand(i: I);
548 if (Op.getParent() != &MI) {
549 // Make sure to use correct addOperand / removeOperand / ChangeTo
550 // functions when replacing operands of a MachineInstr.
551 report(msg: "Instruction has operand with wrong parent set", MI: &MI);
552 }
553
554 visitMachineOperand(MO: &Op, MONum: I);
555 }
556
557 // Was this the last bundled instruction?
558 InBundle = MI.isBundledWithSucc();
559 }
560 if (CurBundle)
561 visitMachineBundleAfter(MI: CurBundle);
562 if (InBundle)
563 report(msg: "BundledSucc flag set on last instruction in block", MI: &MBB.back());
564 visitMachineBasicBlockAfter(MBB: &MBB);
565 }
566 visitMachineFunctionAfter();
567
568 // Clean up.
569 regsLive.clear();
570 regsDefined.clear();
571 regsDead.clear();
572 regsKilled.clear();
573 regMasks.clear();
574 MBBInfoMap.clear();
575
576 return !ReportedErrs.hasError();
577}
578
579void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
580 assert(MF);
581 OS << '\n';
582 if (ReportedErrs.increment()) {
583 if (Banner)
584 OS << "# " << Banner << '\n';
585
586 if (LiveInts != nullptr)
587 LiveInts->print(O&: OS);
588 else
589 MF->print(OS, Indexes);
590 }
591
592 OS << "*** Bad machine code: " << msg << " ***\n"
593 << "- function: " << MF->getName() << '\n';
594}
595
596void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
597 assert(MBB);
598 report(msg, MF: MBB->getParent());
599 OS << "- basic block: " << printMBBReference(MBB: *MBB) << ' ' << MBB->getName()
600 << " (" << (const void *)MBB << ')';
601 if (Indexes)
602 OS << " [" << Indexes->getMBBStartIdx(mbb: MBB) << ';'
603 << Indexes->getMBBEndIdx(mbb: MBB) << ')';
604 OS << '\n';
605}
606
607void MachineVerifier::report(const char *msg, const MachineInstr *MI) {
608 assert(MI);
609 report(msg, MBB: MI->getParent());
610 OS << "- instruction: ";
611 if (Indexes && Indexes->hasIndex(instr: *MI))
612 OS << Indexes->getInstructionIndex(MI: *MI) << '\t';
613 MI->print(OS, /*IsStandalone=*/true);
614}
615
616void MachineVerifier::report(const char *msg, const MachineOperand *MO,
617 unsigned MONum, LLT MOVRegType) {
618 assert(MO);
619 report(msg, MI: MO->getParent());
620 OS << "- operand " << MONum << ": ";
621 MO->print(os&: OS, TypeToPrint: MOVRegType, TRI);
622 OS << '\n';
623}
624
625void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) {
626 report(msg: Msg.str().c_str(), MI);
627}
628
629void MachineVerifier::report_context(SlotIndex Pos) const {
630 OS << "- at: " << Pos << '\n';
631}
632
633void MachineVerifier::report_context(const LiveInterval &LI) const {
634 OS << "- interval: " << LI << '\n';
635}
636
637void MachineVerifier::report_context(const LiveRange &LR,
638 VirtRegOrUnit VRegOrUnit,
639 LaneBitmask LaneMask) const {
640 report_context_liverange(LR);
641 report_context_vreg_regunit(VRegOrUnit);
642 if (LaneMask.any())
643 report_context_lanemask(LaneMask);
644}
645
646void MachineVerifier::report_context(const LiveRange::Segment &S) const {
647 OS << "- segment: " << S << '\n';
648}
649
650void MachineVerifier::report_context(const VNInfo &VNI) const {
651 OS << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n";
652}
653
654void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
655 OS << "- liverange: " << LR << '\n';
656}
657
658void MachineVerifier::report_context(MCPhysReg PReg) const {
659 OS << "- p. register: " << printReg(Reg: PReg, TRI) << '\n';
660}
661
662void MachineVerifier::report_context_vreg(Register VReg) const {
663 OS << "- v. register: " << printReg(Reg: VReg, TRI) << '\n';
664}
665
666void MachineVerifier::report_context_vreg_regunit(
667 VirtRegOrUnit VRegOrUnit) const {
668 if (VRegOrUnit.isVirtualReg()) {
669 report_context_vreg(VReg: VRegOrUnit.asVirtualReg());
670 } else {
671 OS << "- regunit: " << printRegUnit(Unit: VRegOrUnit.asMCRegUnit(), TRI)
672 << '\n';
673 }
674}
675
676void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {
677 OS << "- lanemask: " << PrintLaneMask(LaneMask) << '\n';
678}
679
680void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
681 BBInfo &MInfo = MBBInfoMap[MBB];
682 if (!MInfo.reachable) {
683 MInfo.reachable = true;
684 for (const MachineBasicBlock *Succ : MBB->successors())
685 markReachable(MBB: Succ);
686 }
687}
688
689void MachineVerifier::visitMachineFunctionBefore() {
690 lastIndex = SlotIndex();
691 regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()
692 : TRI->getReservedRegs(MF: *MF);
693
694 if (!MF->empty())
695 markReachable(MBB: &MF->front());
696
697 // Build a set of the basic blocks in the function.
698 FunctionBlocks.clear();
699 for (const auto &MBB : *MF) {
700 FunctionBlocks.insert(Ptr: &MBB);
701 BBInfo &MInfo = MBBInfoMap[&MBB];
702
703 MInfo.Preds.insert_range(R: MBB.predecessors());
704 if (MInfo.Preds.size() != MBB.pred_size())
705 report(msg: "MBB has duplicate entries in its predecessor list.", MBB: &MBB);
706
707 MInfo.Succs.insert_range(R: MBB.successors());
708 if (MInfo.Succs.size() != MBB.succ_size())
709 report(msg: "MBB has duplicate entries in its successor list.", MBB: &MBB);
710 }
711
712 // Check that the register use lists are sane.
713 MRI->verifyUseLists();
714
715 if (!MF->empty()) {
716 verifyStackFrame();
717 verifyStackProtector();
718 }
719}
720
721static bool hasPHIs(const MachineFunction &MF) {
722 return !MF.getProperties().hasNoPHIs() &&
723 any_of(Range: MF, P: [](const MachineBasicBlock &MBB) {
724 return !MBB.phis().empty();
725 });
726}
727
728void
729MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
730 FirstTerminator = nullptr;
731 FirstNonPHI = nullptr;
732
733 if (MRI->tracksLiveness() && hasPHIs(MF: *MF)) {
734 // If this block has allocatable physical registers live-in, check that
735 // it is an entry block or landing pad.
736 for (const auto &LI : MBB->liveins()) {
737 if (isAllocatable(Reg: LI.PhysReg) && !MBB->isEHPad() &&
738 MBB->getIterator() != MBB->getParent()->begin() &&
739 !MBB->isInlineAsmBrIndirectTarget()) {
740 report(msg: "MBB has allocatable live-in, but isn't entry, landing-pad, or "
741 "inlineasm-br-indirect-target.",
742 MBB);
743 report_context(PReg: LI.PhysReg);
744 }
745 }
746 }
747
748 if (MBB->isIRBlockAddressTaken()) {
749 if (!MBB->getAddressTakenIRBlock()->hasAddressTaken())
750 report(msg: "ir-block-address-taken is associated with basic block not used by "
751 "a blockaddress.",
752 MBB);
753 }
754
755 // Count the number of landing pad successors.
756 SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs;
757 for (const auto *succ : MBB->successors()) {
758 if (succ->isEHPad())
759 LandingPadSuccs.insert(Ptr: succ);
760 if (!FunctionBlocks.count(Ptr: succ))
761 report(msg: "MBB has successor that isn't part of the function.", MBB);
762 if (!MBBInfoMap[succ].Preds.count(Ptr: MBB)) {
763 report(msg: "Inconsistent CFG", MBB);
764 OS << "MBB is not in the predecessor list of the successor "
765 << printMBBReference(MBB: *succ) << ".\n";
766 }
767 }
768
769 // Check the predecessor list.
770 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
771 if (!FunctionBlocks.count(Ptr: Pred))
772 report(msg: "MBB has predecessor that isn't part of the function.", MBB);
773 if (!MBBInfoMap[Pred].Succs.count(Ptr: MBB)) {
774 report(msg: "Inconsistent CFG", MBB);
775 OS << "MBB is not in the successor list of the predecessor "
776 << printMBBReference(MBB: *Pred) << ".\n";
777 }
778 }
779
780 const MCAsmInfo &AsmInfo = TM->getMCAsmInfo();
781 const BasicBlock *BB = MBB->getBasicBlock();
782 const Function &F = MF->getFunction();
783 if (LandingPadSuccs.size() > 1 &&
784 !(AsmInfo.getExceptionHandlingType() == ExceptionHandling::SjLj && BB &&
785 isa<SwitchInst>(Val: BB->getTerminator())) &&
786 !isScopedEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
787 report(msg: "MBB has more than one landing pad successor", MBB);
788
789 // Call analyzeBranch. If it succeeds, there several more conditions to check.
790 const MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
791 SmallVector<MachineOperand, 4> Cond;
792 if (!TII->analyzeBranch(MBB: *MBB, TBB, FBB, Cond)) {
793 // Ok, analyzeBranch thinks it knows what's going on with this block. Let's
794 // check whether its answers match up with reality.
795 if (!TBB && !FBB) {
796 // Block falls through to its successor.
797 if (!MBB->empty() && MBB->back().isBarrier() &&
798 !TII->isPredicated(MI: MBB->back())) {
799 report(msg: "MBB exits via unconditional fall-through but ends with a "
800 "barrier instruction!", MBB);
801 }
802 if (!Cond.empty()) {
803 report(msg: "MBB exits via unconditional fall-through but has a condition!",
804 MBB);
805 }
806 } else if (TBB && !FBB && Cond.empty()) {
807 // Block unconditionally branches somewhere.
808 if (MBB->empty()) {
809 report(msg: "MBB exits via unconditional branch but doesn't contain "
810 "any instructions!", MBB);
811 } else if (!MBB->back().isBarrier()) {
812 report(msg: "MBB exits via unconditional branch but doesn't end with a "
813 "barrier instruction!", MBB);
814 } else if (!MBB->back().isTerminator()) {
815 report(msg: "MBB exits via unconditional branch but the branch isn't a "
816 "terminator instruction!", MBB);
817 }
818 } else if (TBB && !FBB && !Cond.empty()) {
819 // Block conditionally branches somewhere, otherwise falls through.
820 if (MBB->empty()) {
821 report(msg: "MBB exits via conditional branch/fall-through but doesn't "
822 "contain any instructions!", MBB);
823 } else if (MBB->back().isBarrier()) {
824 report(msg: "MBB exits via conditional branch/fall-through but ends with a "
825 "barrier instruction!", MBB);
826 } else if (!MBB->back().isTerminator()) {
827 report(msg: "MBB exits via conditional branch/fall-through but the branch "
828 "isn't a terminator instruction!", MBB);
829 }
830 } else if (TBB && FBB) {
831 // Block conditionally branches somewhere, otherwise branches
832 // somewhere else.
833 if (MBB->empty()) {
834 report(msg: "MBB exits via conditional branch/branch but doesn't "
835 "contain any instructions!", MBB);
836 } else if (!MBB->back().isBarrier()) {
837 report(msg: "MBB exits via conditional branch/branch but doesn't end with a "
838 "barrier instruction!", MBB);
839 } else if (!MBB->back().isTerminator()) {
840 report(msg: "MBB exits via conditional branch/branch but the branch "
841 "isn't a terminator instruction!", MBB);
842 }
843 if (Cond.empty()) {
844 report(msg: "MBB exits via conditional branch/branch but there's no "
845 "condition!", MBB);
846 }
847 } else {
848 report(msg: "analyzeBranch returned invalid data!", MBB);
849 }
850
851 // Now check that the successors match up with the answers reported by
852 // analyzeBranch.
853 if (TBB && !MBB->isSuccessor(MBB: TBB))
854 report(msg: "MBB exits via jump or conditional branch, but its target isn't a "
855 "CFG successor!",
856 MBB);
857 if (FBB && !MBB->isSuccessor(MBB: FBB))
858 report(msg: "MBB exits via conditional branch, but its target isn't a CFG "
859 "successor!",
860 MBB);
861
862 // There might be a fallthrough to the next block if there's either no
863 // unconditional true branch, or if there's a condition, and one of the
864 // branches is missing.
865 bool Fallthrough = !TBB || (!Cond.empty() && !FBB);
866
867 // A conditional fallthrough must be an actual CFG successor, not
868 // unreachable. (Conversely, an unconditional fallthrough might not really
869 // be a successor, because the block might end in unreachable.)
870 if (!Cond.empty() && !FBB) {
871 MachineFunction::const_iterator MBBI = std::next(x: MBB->getIterator());
872 if (MBBI == MF->end()) {
873 report(msg: "MBB conditionally falls through out of function!", MBB);
874 } else if (!MBB->isSuccessor(MBB: &*MBBI))
875 report(msg: "MBB exits via conditional branch/fall-through but the CFG "
876 "successors don't match the actual successors!",
877 MBB);
878 }
879
880 // Verify that there aren't any extra un-accounted-for successors.
881 for (const MachineBasicBlock *SuccMBB : MBB->successors()) {
882 // If this successor is one of the branch targets, it's okay.
883 if (SuccMBB == TBB || SuccMBB == FBB)
884 continue;
885 // If we might have a fallthrough, and the successor is the fallthrough
886 // block, that's also ok.
887 if (Fallthrough && SuccMBB == MBB->getNextNode())
888 continue;
889 // Also accept successors which are for exception-handling or might be
890 // inlineasm_br targets.
891 if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
892 continue;
893 report(msg: "MBB has unexpected successors which are not branch targets, "
894 "fallthrough, EHPads, or inlineasm_br targets.",
895 MBB);
896 }
897 }
898
899 regsLive.clear();
900 if (MRI->tracksLiveness()) {
901 for (const auto &LI : MBB->liveins()) {
902 if (!LI.PhysReg.isPhysical()) {
903 report(msg: "MBB live-in list contains non-physical register", MBB);
904 continue;
905 }
906 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: LI.PhysReg));
907 }
908 }
909
910 const MachineFrameInfo &MFI = MF->getFrameInfo();
911 BitVector PR = MFI.getPristineRegs(MF: *MF);
912 for (unsigned I : PR.set_bits())
913 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: I));
914
915 regsKilled.clear();
916 regsDefined.clear();
917
918 if (Indexes)
919 lastIndex = Indexes->getMBBStartIdx(mbb: MBB);
920}
921
922// This function gets called for all bundle headers, including normal
923// stand-alone unbundled instructions.
924void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
925 if (Indexes && Indexes->hasIndex(instr: *MI)) {
926 SlotIndex idx = Indexes->getInstructionIndex(MI: *MI);
927 if (!(idx > lastIndex)) {
928 report(msg: "Instruction index out of order", MI);
929 OS << "Last instruction was at " << lastIndex << '\n';
930 }
931 lastIndex = idx;
932 }
933
934 // Ensure non-terminators don't follow terminators.
935 if (MI->isTerminator()) {
936 if (!FirstTerminator)
937 FirstTerminator = MI;
938 } else if (FirstTerminator) {
939 // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to
940 // precede non-terminators.
941 if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) {
942 report(msg: "Non-terminator instruction after the first terminator", MI);
943 OS << "First terminator was:\t" << *FirstTerminator;
944 }
945 }
946}
947
948// The operands on an INLINEASM instruction must follow a template.
949// Verify that the flag operands make sense.
950void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
951 // The first two operands on INLINEASM are the asm string and global flags.
952 if (MI->getNumOperands() < 2) {
953 report(msg: "Too few operands on inline asm", MI);
954 return;
955 }
956 if (!MI->getOperand(i: 0).isSymbol())
957 report(msg: "Asm string must be an external symbol", MI);
958 if (!MI->getOperand(i: 1).isImm())
959 report(msg: "Asm flags must be an immediate", MI);
960 // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
961 // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,
962 // and Extra_IsConvergent = 32, Extra_MayUnwind = 64.
963 if (!isUInt<7>(x: MI->getOperand(i: 1).getImm()))
964 report(msg: "Unknown asm flags", MO: &MI->getOperand(i: 1), MONum: 1);
965
966 static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");
967
968 unsigned OpNo = InlineAsm::MIOp_FirstOperand;
969 unsigned NumOps;
970 for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
971 const MachineOperand &MO = MI->getOperand(i: OpNo);
972 // There may be implicit ops after the fixed operands.
973 if (!MO.isImm())
974 break;
975 const InlineAsm::Flag F(MO.getImm());
976 NumOps = 1 + F.getNumOperandRegisters();
977 }
978
979 if (OpNo > MI->getNumOperands())
980 report(msg: "Missing operands in last group", MI);
981
982 // An optional MDNode follows the groups.
983 if (OpNo < MI->getNumOperands() && MI->getOperand(i: OpNo).isMetadata())
984 ++OpNo;
985
986 // All trailing operands must be implicit registers.
987 for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
988 const MachineOperand &MO = MI->getOperand(i: OpNo);
989 if (!MO.isReg() || !MO.isImplicit())
990 report(msg: "Expected implicit register after groups", MO: &MO, MONum: OpNo);
991 }
992
993 if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) {
994 const MachineBasicBlock *MBB = MI->getParent();
995
996 for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands();
997 i != e; ++i) {
998 const MachineOperand &MO = MI->getOperand(i);
999
1000 if (!MO.isMBB())
1001 continue;
1002
1003 // Check the successor & predecessor lists look ok, assume they are
1004 // not. Find the indirect target without going through the successors.
1005 const MachineBasicBlock *IndirectTargetMBB = MO.getMBB();
1006 if (!IndirectTargetMBB) {
1007 report(msg: "INLINEASM_BR indirect target does not exist", MO: &MO, MONum: i);
1008 break;
1009 }
1010
1011 if (!MBB->isSuccessor(MBB: IndirectTargetMBB))
1012 report(msg: "INLINEASM_BR indirect target missing from successor list", MO: &MO,
1013 MONum: i);
1014
1015 if (!IndirectTargetMBB->isPredecessor(MBB))
1016 report(msg: "INLINEASM_BR indirect target predecessor list missing parent",
1017 MO: &MO, MONum: i);
1018 }
1019 }
1020}
1021
1022bool MachineVerifier::verifyAllRegOpsScalar(const MachineInstr &MI,
1023 const MachineRegisterInfo &MRI) {
1024 if (none_of(Range: MI.explicit_operands(), P: [&MRI](const MachineOperand &Op) {
1025 if (!Op.isReg())
1026 return false;
1027 const auto Reg = Op.getReg();
1028 if (Reg.isPhysical())
1029 return false;
1030 return !MRI.getType(Reg).isScalar();
1031 }))
1032 return true;
1033 report(msg: "All register operands must have scalar types", MI: &MI);
1034 return false;
1035}
1036
1037/// Check that types are consistent when two operands need to have the same
1038/// number of vector elements.
1039/// \return true if the types are valid.
1040bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,
1041 const MachineInstr *MI) {
1042 if (Ty0.isVector() != Ty1.isVector()) {
1043 report(msg: "operand types must be all-vector or all-scalar", MI);
1044 // Generally we try to report as many issues as possible at once, but in
1045 // this case it's not clear what should we be comparing the size of the
1046 // scalar with: the size of the whole vector or its lane. Instead of
1047 // making an arbitrary choice and emitting not so helpful message, let's
1048 // avoid the extra noise and stop here.
1049 return false;
1050 }
1051
1052 if (Ty0.isVector() && Ty0.getElementCount() != Ty1.getElementCount()) {
1053 report(msg: "operand types must preserve number of vector elements", MI);
1054 return false;
1055 }
1056
1057 return true;
1058}
1059
1060bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
1061 auto Opcode = MI->getOpcode();
1062 bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
1063 Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
1064 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1065 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1066 AttributeSet Attrs = Intrinsic::getFnAttributes(
1067 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1068 bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
1069 if (NoSideEffects && DeclHasSideEffects) {
1070 report(Msg: Twine(TII->getName(Opcode),
1071 " used with intrinsic that accesses memory"),
1072 MI);
1073 return false;
1074 }
1075 if (!NoSideEffects && !DeclHasSideEffects) {
1076 report(Msg: Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI);
1077 return false;
1078 }
1079 }
1080
1081 return true;
1082}
1083
1084bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
1085 auto Opcode = MI->getOpcode();
1086 bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
1087 Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
1088 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1089 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1090 AttributeSet Attrs = Intrinsic::getFnAttributes(
1091 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1092 bool DeclIsConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
1093 if (NotConvergent && DeclIsConvergent) {
1094 report(Msg: Twine(TII->getName(Opcode), " used with a convergent intrinsic"),
1095 MI);
1096 return false;
1097 }
1098 if (!NotConvergent && !DeclIsConvergent) {
1099 report(
1100 Msg: Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"),
1101 MI);
1102 return false;
1103 }
1104 }
1105
1106 return true;
1107}
1108
1109void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
1110 if (isFunctionSelected)
1111 report(msg: "Unexpected generic instruction in a Selected function", MI);
1112
1113 const MCInstrDesc &MCID = MI->getDesc();
1114 unsigned NumOps = MI->getNumOperands();
1115
1116 // Branches must reference a basic block if they are not indirect
1117 if (MI->isBranch() && !MI->isIndirectBranch()) {
1118 bool HasMBB = false;
1119 for (const MachineOperand &Op : MI->operands()) {
1120 if (Op.isMBB()) {
1121 HasMBB = true;
1122 break;
1123 }
1124 }
1125
1126 if (!HasMBB) {
1127 report(msg: "Branch instruction is missing a basic block operand or "
1128 "isIndirectBranch property",
1129 MI);
1130 }
1131 }
1132
1133 // Check types.
1134 SmallVector<LLT, 4> Types;
1135 for (unsigned I = 0, E = std::min(a: MCID.getNumOperands(), b: NumOps);
1136 I != E; ++I) {
1137 if (!MCID.operands()[I].isGenericType())
1138 continue;
1139 // Generic instructions specify type equality constraints between some of
1140 // their operands. Make sure these are consistent.
1141 size_t TypeIdx = MCID.operands()[I].getGenericTypeIndex();
1142 Types.resize(N: std::max(a: TypeIdx + 1, b: Types.size()));
1143
1144 const MachineOperand *MO = &MI->getOperand(i: I);
1145 if (!MO->isReg()) {
1146 report(msg: "generic instruction must use register operands", MI);
1147 continue;
1148 }
1149
1150 LLT OpTy = MRI->getType(Reg: MO->getReg());
1151 // Don't report a type mismatch if there is no actual mismatch, only a
1152 // type missing, to reduce noise:
1153 if (OpTy.isValid()) {
1154 // Only the first valid type for a type index will be printed: don't
1155 // overwrite it later so it's always clear which type was expected:
1156 if (!Types[TypeIdx].isValid())
1157 Types[TypeIdx] = OpTy;
1158 else if (Types[TypeIdx] != OpTy)
1159 report(msg: "Type mismatch in generic instruction", MO, MONum: I, MOVRegType: OpTy);
1160 } else {
1161 // Generic instructions must have types attached to their operands.
1162 report(msg: "Generic instruction is missing a virtual register type", MO, MONum: I);
1163 }
1164 }
1165
1166 // Generic opcodes must not have physical register operands.
1167 for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
1168 const MachineOperand *MO = &MI->getOperand(i: I);
1169 if (MO->isReg() && MO->getReg().isPhysical())
1170 report(msg: "Generic instruction cannot have physical register", MO, MONum: I);
1171 }
1172
1173 // Avoid out of bounds in checks below. This was already reported earlier.
1174 if (MI->getNumOperands() < MCID.getNumOperands())
1175 return;
1176
1177 StringRef ErrorInfo;
1178 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
1179 report(msg: ErrorInfo.data(), MI);
1180
1181 // Verify properties of various specific instruction types
1182 unsigned Opc = MI->getOpcode();
1183 switch (Opc) {
1184 case TargetOpcode::G_ASSERT_SEXT:
1185 case TargetOpcode::G_ASSERT_ZEXT: {
1186 std::string OpcName =
1187 Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT";
1188 if (!MI->getOperand(i: 2).isImm()) {
1189 report(Msg: Twine(OpcName, " expects an immediate operand #2"), MI);
1190 break;
1191 }
1192
1193 Register Dst = MI->getOperand(i: 0).getReg();
1194 Register Src = MI->getOperand(i: 1).getReg();
1195 LLT SrcTy = MRI->getType(Reg: Src);
1196 int64_t Imm = MI->getOperand(i: 2).getImm();
1197 if (Imm <= 0) {
1198 report(Msg: Twine(OpcName, " size must be >= 1"), MI);
1199 break;
1200 }
1201
1202 if (Imm >= SrcTy.getScalarSizeInBits()) {
1203 report(Msg: Twine(OpcName, " size must be less than source bit width"), MI);
1204 break;
1205 }
1206
1207 const RegisterBank *SrcRB = RBI->getRegBank(Reg: Src, MRI: *MRI, TRI: *TRI);
1208 const RegisterBank *DstRB = RBI->getRegBank(Reg: Dst, MRI: *MRI, TRI: *TRI);
1209
1210 // Allow only the source bank to be set.
1211 if ((SrcRB && DstRB && SrcRB != DstRB) || (DstRB && !SrcRB)) {
1212 report(Msg: Twine(OpcName, " cannot change register bank"), MI);
1213 break;
1214 }
1215
1216 // Don't allow a class change. Do allow member class->regbank.
1217 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: Dst);
1218 if (DstRC && DstRC != MRI->getRegClassOrNull(Reg: Src)) {
1219 report(
1220 Msg: Twine(OpcName, " source and destination register classes must match"),
1221 MI);
1222 break;
1223 }
1224
1225 break;
1226 }
1227
1228 case TargetOpcode::G_CONSTANT:
1229 case TargetOpcode::G_FCONSTANT: {
1230 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1231 if (DstTy.isVector())
1232 report(msg: "Instruction cannot use a vector result type", MI);
1233
1234 if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {
1235 if (!MI->getOperand(i: 1).isCImm()) {
1236 report(msg: "G_CONSTANT operand must be cimm", MI);
1237 break;
1238 }
1239
1240 const ConstantInt *CI = MI->getOperand(i: 1).getCImm();
1241 if (CI->getBitWidth() != DstTy.getSizeInBits())
1242 report(msg: "inconsistent constant size", MI);
1243 } else {
1244 if (!MI->getOperand(i: 1).isFPImm()) {
1245 report(msg: "G_FCONSTANT operand must be fpimm", MI);
1246 break;
1247 }
1248 const ConstantFP *CF = MI->getOperand(i: 1).getFPImm();
1249
1250 if (APFloat::getSizeInBits(Sem: CF->getValueAPF().getSemantics()) !=
1251 DstTy.getSizeInBits()) {
1252 report(msg: "inconsistent constant size", MI);
1253 }
1254 }
1255
1256 break;
1257 }
1258 case TargetOpcode::G_LOAD:
1259 case TargetOpcode::G_STORE:
1260 case TargetOpcode::G_ZEXTLOAD:
1261 case TargetOpcode::G_SEXTLOAD:
1262 case TargetOpcode::G_FPEXTLOAD:
1263 case TargetOpcode::G_FPTRUNCSTORE: {
1264 LLT ValTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1265 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1266 if (!PtrTy.isPointer())
1267 report(msg: "Generic memory instruction must access a pointer", MI);
1268
1269 // Generic loads and stores must have a single MachineMemOperand
1270 // describing that access.
1271 if (!MI->hasOneMemOperand()) {
1272 report(msg: "Generic instruction accessing memory must have one mem operand",
1273 MI);
1274 } else {
1275 const MachineMemOperand &MMO = **MI->memoperands_begin();
1276 if (isa<GExtLoad>(Val: *MI)) {
1277 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1278 RHS: ValTy.getSizeInBits()))
1279 report(msg: "Generic extload must have a narrower memory type", MI);
1280 } else if (isa<GFPTruncStore>(Val: *MI)) {
1281 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1282 RHS: ValTy.getSizeInBits()))
1283 report(msg: "Generic truncstore must have a narrower memory type", MI);
1284 } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
1285 if (TypeSize::isKnownGT(LHS: MMO.getSize().getValue(),
1286 RHS: ValTy.getSizeInBytes()))
1287 report(msg: "load memory size cannot exceed result size", MI);
1288
1289 if (MMO.getRanges()) {
1290 ConstantInt *i =
1291 mdconst::extract<ConstantInt>(MD: MMO.getRanges()->getOperand(I: 0));
1292 const LLT RangeTy = LLT::scalar(SizeInBits: i->getIntegerType()->getBitWidth());
1293 const LLT MemTy = MMO.getMemoryType();
1294 if (MemTy.getScalarType() != RangeTy ||
1295 ValTy.isScalar() != MemTy.isScalar() ||
1296 (ValTy.isVector() &&
1297 ValTy.getNumElements() != MemTy.getNumElements())) {
1298 report(msg: "range is incompatible with the result type", MI);
1299 }
1300 }
1301 } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
1302 if (TypeSize::isKnownLT(LHS: ValTy.getSizeInBytes(),
1303 RHS: MMO.getSize().getValue()))
1304 report(msg: "store memory size cannot exceed value size", MI);
1305 }
1306
1307 const AtomicOrdering Order = MMO.getSuccessOrdering();
1308 if (isa<GAnyStore>(Val: *MI)) {
1309 if (Order == AtomicOrdering::Acquire ||
1310 Order == AtomicOrdering::AcquireRelease)
1311 report(msg: "atomic store cannot use acquire ordering", MI);
1312
1313 } else {
1314 if (Order == AtomicOrdering::Release ||
1315 Order == AtomicOrdering::AcquireRelease)
1316 report(msg: "atomic load cannot use release ordering", MI);
1317 }
1318 }
1319
1320 break;
1321 }
1322 case TargetOpcode::G_PHI: {
1323 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1324 if (!DstTy.isValid() || !all_of(Range: drop_begin(RangeOrContainer: MI->operands()),
1325 P: [this, &DstTy](const MachineOperand &MO) {
1326 if (!MO.isReg())
1327 return true;
1328 LLT Ty = MRI->getType(Reg: MO.getReg());
1329 if (!Ty.isValid() || (Ty != DstTy))
1330 return false;
1331 return true;
1332 }))
1333 report(msg: "Generic Instruction G_PHI has operands with incompatible/missing "
1334 "types",
1335 MI);
1336 break;
1337 }
1338 case TargetOpcode::G_BITCAST: {
1339 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1340 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1341 if (!DstTy.isValid() || !SrcTy.isValid())
1342 break;
1343
1344 if (SrcTy.isPointer() != DstTy.isPointer())
1345 report(msg: "bitcast cannot convert between pointers and other types", MI);
1346
1347 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1348 report(msg: "bitcast sizes must match", MI);
1349
1350 bool SameType = SrcTy.getKind() == DstTy.getKind();
1351 if (SameType && SrcTy.isPointerOrPointerVector())
1352 SameType &= SrcTy.getAddressSpace() == DstTy.getAddressSpace();
1353
1354 SameType &= SrcTy.getScalarSizeInBits() == DstTy.getScalarSizeInBits();
1355
1356 if (SameType && SrcTy.isVector())
1357 SameType &= SrcTy.getElementCount() == DstTy.getElementCount();
1358 if (SameType && SrcTy.isFloatOrFloatVector())
1359 SameType &= SrcTy.getFpSemantics() == DstTy.getFpSemantics();
1360
1361 if (SameType)
1362 report(msg: "bitcast must change the type", MI);
1363
1364 break;
1365 }
1366 case TargetOpcode::G_INTTOPTR:
1367 case TargetOpcode::G_PTRTOINT:
1368 case TargetOpcode::G_ADDRSPACE_CAST: {
1369 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1370 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1371 if (!DstTy.isValid() || !SrcTy.isValid())
1372 break;
1373
1374 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1375
1376 DstTy = DstTy.getScalarType();
1377 SrcTy = SrcTy.getScalarType();
1378
1379 if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {
1380 if (!DstTy.isPointer())
1381 report(msg: "inttoptr result type must be a pointer", MI);
1382 if (SrcTy.isPointer())
1383 report(msg: "inttoptr source type must not be a pointer", MI);
1384 } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {
1385 if (!SrcTy.isPointer())
1386 report(msg: "ptrtoint source type must be a pointer", MI);
1387 if (DstTy.isPointer())
1388 report(msg: "ptrtoint result type must not be a pointer", MI);
1389 } else {
1390 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
1391 if (!SrcTy.isPointer() || !DstTy.isPointer())
1392 report(msg: "addrspacecast types must be pointers", MI);
1393 else {
1394 if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())
1395 report(msg: "addrspacecast must convert different address spaces", MI);
1396 }
1397 }
1398
1399 break;
1400 }
1401 case TargetOpcode::G_PTR_ADD: {
1402 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1403 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1404 LLT OffsetTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1405 if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())
1406 break;
1407
1408 if (!PtrTy.isPointerOrPointerVector())
1409 report(msg: "gep first operand must be a pointer", MI);
1410
1411 if (OffsetTy.isPointerOrPointerVector())
1412 report(msg: "gep offset operand must not be a pointer", MI);
1413
1414 if (PtrTy.isPointerOrPointerVector()) {
1415 const DataLayout &DL = MF->getDataLayout();
1416 unsigned AS = PtrTy.getAddressSpace();
1417 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
1418 if (OffsetTy.getScalarSizeInBits() != IndexSizeInBits) {
1419 report(msg: "gep offset operand must match index size for address space",
1420 MI);
1421 }
1422 }
1423
1424 // TODO: Is the offset allowed to be a scalar with a vector?
1425 break;
1426 }
1427 case TargetOpcode::G_PTRMASK: {
1428 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1429 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1430 LLT MaskTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1431 if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid())
1432 break;
1433
1434 if (!DstTy.isPointerOrPointerVector())
1435 report(msg: "ptrmask result type must be a pointer", MI);
1436
1437 if (!MaskTy.getScalarType().isScalar())
1438 report(msg: "ptrmask mask type must be an integer", MI);
1439
1440 verifyVectorElementMatch(Ty0: DstTy, Ty1: MaskTy, MI);
1441 break;
1442 }
1443 case TargetOpcode::G_SEXT:
1444 case TargetOpcode::G_ZEXT:
1445 case TargetOpcode::G_ANYEXT:
1446 case TargetOpcode::G_TRUNC:
1447 case TargetOpcode::G_TRUNC_SSAT_S:
1448 case TargetOpcode::G_TRUNC_SSAT_U:
1449 case TargetOpcode::G_TRUNC_USAT_U:
1450 case TargetOpcode::G_FPEXT:
1451 case TargetOpcode::G_FPTRUNC: {
1452 // Number of operands and presense of types is already checked (and
1453 // reported in case of any issues), so no need to report them again. As
1454 // we're trying to report as many issues as possible at once, however, the
1455 // instructions aren't guaranteed to have the right number of operands or
1456 // types attached to them at this point
1457 assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
1458 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1459 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1460 if (!DstTy.isValid() || !SrcTy.isValid())
1461 break;
1462
1463 if (DstTy.isPointerOrPointerVector() || SrcTy.isPointerOrPointerVector())
1464 report(msg: "Generic extend/truncate can not operate on pointers", MI);
1465
1466 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1467
1468 unsigned DstSize = DstTy.getScalarSizeInBits();
1469 unsigned SrcSize = SrcTy.getScalarSizeInBits();
1470 switch (MI->getOpcode()) {
1471 default:
1472 if (DstSize <= SrcSize)
1473 report(msg: "Generic extend has destination type no larger than source", MI);
1474 break;
1475 case TargetOpcode::G_TRUNC:
1476 case TargetOpcode::G_TRUNC_SSAT_S:
1477 case TargetOpcode::G_TRUNC_SSAT_U:
1478 case TargetOpcode::G_TRUNC_USAT_U:
1479 case TargetOpcode::G_FPTRUNC:
1480 if (DstSize >= SrcSize)
1481 report(msg: "Generic truncate has destination type no smaller than source",
1482 MI);
1483 break;
1484 }
1485 break;
1486 }
1487 case TargetOpcode::G_SELECT: {
1488 LLT SelTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1489 LLT CondTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1490 if (!SelTy.isValid() || !CondTy.isValid())
1491 break;
1492
1493 // Scalar condition select on a vector is valid.
1494 if (CondTy.isVector())
1495 verifyVectorElementMatch(Ty0: SelTy, Ty1: CondTy, MI);
1496 break;
1497 }
1498 case TargetOpcode::G_MERGE_VALUES: {
1499 // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,
1500 // e.g. s2N = MERGE sN, sN
1501 // Merging multiple scalars into a vector is not allowed, should use
1502 // G_BUILD_VECTOR for that.
1503 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1504 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1505 if (DstTy.isVector() || SrcTy.isVector())
1506 report(msg: "G_MERGE_VALUES cannot operate on vectors", MI);
1507
1508 const unsigned NumOps = MI->getNumOperands();
1509 if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1))
1510 report(msg: "G_MERGE_VALUES result size is inconsistent", MI);
1511
1512 for (unsigned I = 2; I != NumOps; ++I) {
1513 if (MRI->getType(Reg: MI->getOperand(i: I).getReg()) != SrcTy)
1514 report(msg: "G_MERGE_VALUES source types do not match", MI);
1515 }
1516
1517 break;
1518 }
1519 case TargetOpcode::G_UNMERGE_VALUES: {
1520 unsigned NumDsts = MI->getNumOperands() - 1;
1521 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1522 for (unsigned i = 1; i < NumDsts; ++i) {
1523 if (MRI->getType(Reg: MI->getOperand(i).getReg()) != DstTy) {
1524 report(msg: "G_UNMERGE_VALUES destination types do not match", MI);
1525 break;
1526 }
1527 }
1528
1529 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: NumDsts).getReg());
1530 if (DstTy.isVector()) {
1531 // This case is the converse of G_CONCAT_VECTORS.
1532 if (!SrcTy.isVector() ||
1533 (SrcTy.getScalarType() != DstTy.getScalarType() &&
1534 !SrcTy.isPointerVector()) ||
1535 SrcTy.isScalableVector() != DstTy.isScalableVector() ||
1536 SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1537 report(msg: "G_UNMERGE_VALUES source operand does not match vector "
1538 "destination operands",
1539 MI);
1540 } else if (SrcTy.isVector()) {
1541 // This case is the converse of G_BUILD_VECTOR, but relaxed to allow
1542 // mismatched types as long as the total size matches:
1543 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<4 x s32>)
1544 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1545 report(msg: "G_UNMERGE_VALUES vector source operand does not match scalar "
1546 "destination operands",
1547 MI);
1548 } else {
1549 // This case is the converse of G_MERGE_VALUES.
1550 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits()) {
1551 report(msg: "G_UNMERGE_VALUES scalar source operand does not match scalar "
1552 "destination operands",
1553 MI);
1554 }
1555 }
1556 break;
1557 }
1558 case TargetOpcode::G_BUILD_VECTOR: {
1559 // Source types must be scalars, dest type a vector. Total size of scalars
1560 // must match the dest vector size.
1561 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1562 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1563 if (!DstTy.isVector() || SrcEltTy.isVector()) {
1564 report(msg: "G_BUILD_VECTOR must produce a vector from scalar operands", MI);
1565 break;
1566 }
1567
1568 if (DstTy.getElementType() != SrcEltTy)
1569 report(msg: "G_BUILD_VECTOR result element type must match source type", MI);
1570
1571 if (DstTy.getNumElements() != MI->getNumOperands() - 1)
1572 report(msg: "G_BUILD_VECTOR must have an operand for each element", MI);
1573
1574 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1575 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1576 report(msg: "G_BUILD_VECTOR source operand types are not homogeneous", MI);
1577
1578 break;
1579 }
1580 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1581 // Source types must be scalars, dest type a vector. Scalar types must be
1582 // larger than the dest vector elt type, as this is a truncating operation.
1583 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1584 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1585 if (!DstTy.isVector() || SrcEltTy.isVector())
1586 report(msg: "G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands",
1587 MI);
1588 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1589 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1590 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not homogeneous",
1591 MI);
1592 if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits())
1593 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not larger than "
1594 "dest elt type",
1595 MI);
1596 break;
1597 }
1598 case TargetOpcode::G_CONCAT_VECTORS: {
1599 // Source types should be vectors, and total size should match the dest
1600 // vector size.
1601 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1602 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1603 if (!DstTy.isVector() || !SrcTy.isVector())
1604 report(msg: "G_CONCAT_VECTOR requires vector source and destination operands",
1605 MI);
1606
1607 if (MI->getNumOperands() < 3)
1608 report(msg: "G_CONCAT_VECTOR requires at least 2 source operands", MI);
1609
1610 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1611 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1612 report(msg: "G_CONCAT_VECTOR source operand types are not homogeneous", MI);
1613 if (DstTy.getElementCount() !=
1614 SrcTy.getElementCount() * (MI->getNumOperands() - 1))
1615 report(msg: "G_CONCAT_VECTOR num dest and source elements should match", MI);
1616 break;
1617 }
1618 case TargetOpcode::G_ICMP:
1619 case TargetOpcode::G_FCMP: {
1620 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1621 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1622
1623 if ((DstTy.isVector() != SrcTy.isVector()) ||
1624 (DstTy.isVector() &&
1625 DstTy.getElementCount() != SrcTy.getElementCount()))
1626 report(msg: "Generic vector icmp/fcmp must preserve number of lanes", MI);
1627
1628 break;
1629 }
1630 case TargetOpcode::G_SCMP:
1631 case TargetOpcode::G_UCMP: {
1632 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1633 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1634
1635 if (SrcTy.isPointerOrPointerVector()) {
1636 report(msg: "Generic scmp/ucmp does not support pointers as operands", MI);
1637 break;
1638 }
1639
1640 if (DstTy.isPointerOrPointerVector()) {
1641 report(msg: "Generic scmp/ucmp does not support pointers as a result", MI);
1642 break;
1643 }
1644
1645 if (DstTy.getScalarSizeInBits() < 2) {
1646 report(msg: "Result type must be at least 2 bits wide", MI);
1647 break;
1648 }
1649
1650 if ((DstTy.isVector() != SrcTy.isVector()) ||
1651 (DstTy.isVector() &&
1652 DstTy.getElementCount() != SrcTy.getElementCount())) {
1653 report(msg: "Generic vector scmp/ucmp must preserve number of lanes", MI);
1654 break;
1655 }
1656
1657 break;
1658 }
1659 case TargetOpcode::G_EXTRACT: {
1660 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1661 if (!SrcOp.isReg()) {
1662 report(msg: "extract source must be a register", MI);
1663 break;
1664 }
1665
1666 const MachineOperand &OffsetOp = MI->getOperand(i: 2);
1667 if (!OffsetOp.isImm()) {
1668 report(msg: "extract offset must be a constant", MI);
1669 break;
1670 }
1671
1672 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1673 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1674 if (SrcSize == DstSize)
1675 report(msg: "extract source must be larger than result", MI);
1676
1677 if (DstSize + OffsetOp.getImm() > SrcSize)
1678 report(msg: "extract reads past end of register", MI);
1679 break;
1680 }
1681 case TargetOpcode::G_INSERT: {
1682 const MachineOperand &SrcOp = MI->getOperand(i: 2);
1683 if (!SrcOp.isReg()) {
1684 report(msg: "insert source must be a register", MI);
1685 break;
1686 }
1687
1688 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
1689 if (!OffsetOp.isImm()) {
1690 report(msg: "insert offset must be a constant", MI);
1691 break;
1692 }
1693
1694 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1695 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1696
1697 if (DstSize <= SrcSize)
1698 report(msg: "inserted size must be smaller than total register", MI);
1699
1700 if (SrcSize + OffsetOp.getImm() > DstSize)
1701 report(msg: "insert writes past end of register", MI);
1702
1703 break;
1704 }
1705 case TargetOpcode::G_JUMP_TABLE: {
1706 if (!MI->getOperand(i: 1).isJTI())
1707 report(msg: "G_JUMP_TABLE source operand must be a jump table index", MI);
1708 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1709 if (!DstTy.isPointer())
1710 report(msg: "G_JUMP_TABLE dest operand must have a pointer type", MI);
1711 break;
1712 }
1713 case TargetOpcode::G_BRJT: {
1714 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
1715 report(msg: "G_BRJT src operand 0 must be a pointer type", MI);
1716
1717 if (!MI->getOperand(i: 1).isJTI())
1718 report(msg: "G_BRJT src operand 1 must be a jump table index", MI);
1719
1720 const auto &IdxOp = MI->getOperand(i: 2);
1721 if (!IdxOp.isReg() || MRI->getType(Reg: IdxOp.getReg()).isPointer())
1722 report(msg: "G_BRJT src operand 2 must be a scalar reg type", MI);
1723 break;
1724 }
1725 case TargetOpcode::G_INTRINSIC:
1726 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1727 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1728 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
1729 // TODO: Should verify number of def and use operands, but the current
1730 // interface requires passing in IR types for mangling.
1731 const MachineOperand &IntrIDOp = MI->getOperand(i: MI->getNumExplicitDefs());
1732 if (!IntrIDOp.isIntrinsicID()) {
1733 report(msg: "G_INTRINSIC first src operand must be an intrinsic ID", MI);
1734 break;
1735 }
1736
1737 if (!verifyGIntrinsicSideEffects(MI))
1738 break;
1739 if (!verifyGIntrinsicConvergence(MI))
1740 break;
1741
1742 break;
1743 }
1744 case TargetOpcode::G_SEXT_INREG: {
1745 if (!MI->getOperand(i: 2).isImm()) {
1746 report(msg: "G_SEXT_INREG expects an immediate operand #2", MI);
1747 break;
1748 }
1749
1750 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1751 int64_t Imm = MI->getOperand(i: 2).getImm();
1752 if (Imm <= 0)
1753 report(msg: "G_SEXT_INREG size must be >= 1", MI);
1754 if (Imm >= SrcTy.getScalarSizeInBits())
1755 report(msg: "G_SEXT_INREG size must be less than source bit width", MI);
1756 break;
1757 }
1758 case TargetOpcode::G_BSWAP: {
1759 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1760 if (DstTy.getScalarSizeInBits() % 16 != 0)
1761 report(msg: "G_BSWAP size must be a multiple of 16 bits", MI);
1762 break;
1763 }
1764 case TargetOpcode::G_VSCALE: {
1765 if (!MI->getOperand(i: 1).isCImm()) {
1766 report(msg: "G_VSCALE operand must be cimm", MI);
1767 break;
1768 }
1769 if (MI->getOperand(i: 1).getCImm()->isZero()) {
1770 report(msg: "G_VSCALE immediate cannot be zero", MI);
1771 break;
1772 }
1773 break;
1774 }
1775 case TargetOpcode::G_STEP_VECTOR: {
1776 if (!MI->getOperand(i: 1).isCImm()) {
1777 report(msg: "operand must be cimm", MI);
1778 break;
1779 }
1780
1781 if (!MI->getOperand(i: 1).getCImm()->getValue().isStrictlyPositive()) {
1782 report(msg: "step must be > 0", MI);
1783 break;
1784 }
1785
1786 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1787 if (!DstTy.isScalableVector()) {
1788 report(msg: "Destination type must be a scalable vector", MI);
1789 break;
1790 }
1791
1792 // <vscale x 2 x p0>
1793 if (!DstTy.getElementType().isScalar()) {
1794 report(msg: "Destination element type must be scalar", MI);
1795 break;
1796 }
1797
1798 if (MI->getOperand(i: 1).getCImm()->getBitWidth() !=
1799 DstTy.getElementType().getScalarSizeInBits()) {
1800 report(msg: "step bitwidth differs from result type element bitwidth", MI);
1801 break;
1802 }
1803 break;
1804 }
1805 case TargetOpcode::G_INSERT_SUBVECTOR: {
1806 const MachineOperand &Src0Op = MI->getOperand(i: 1);
1807 if (!Src0Op.isReg()) {
1808 report(msg: "G_INSERT_SUBVECTOR first source must be a register", MI);
1809 break;
1810 }
1811
1812 const MachineOperand &Src1Op = MI->getOperand(i: 2);
1813 if (!Src1Op.isReg()) {
1814 report(msg: "G_INSERT_SUBVECTOR second source must be a register", MI);
1815 break;
1816 }
1817
1818 const MachineOperand &IndexOp = MI->getOperand(i: 3);
1819 if (!IndexOp.isImm()) {
1820 report(msg: "G_INSERT_SUBVECTOR index must be an immediate", MI);
1821 break;
1822 }
1823
1824 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1825 LLT Src1Ty = MRI->getType(Reg: Src1Op.getReg());
1826
1827 if (!DstTy.isVector()) {
1828 report(msg: "Destination type must be a vector", MI);
1829 break;
1830 }
1831
1832 if (!Src1Ty.isVector()) {
1833 report(msg: "Second source must be a vector", MI);
1834 break;
1835 }
1836
1837 if (DstTy.getElementType() != Src1Ty.getElementType()) {
1838 report(msg: "Element type of vectors must be the same", MI);
1839 break;
1840 }
1841
1842 if (!DstTy.isScalable() && Src1Ty.isScalable()) {
1843 report(msg: "Cannot insert a scalable vector into a fixed length vector", MI);
1844 break;
1845 }
1846
1847 bool IsMixedFixedIntoScalable =
1848 DstTy.isScalableVector() && Src1Ty.isFixedVector();
1849
1850 if (!IsMixedFixedIntoScalable &&
1851 ElementCount::isKnownGT(LHS: Src1Ty.getElementCount(),
1852 RHS: DstTy.getElementCount())) {
1853 report(msg: "Second source must be smaller than destination vector", MI);
1854 break;
1855 }
1856
1857 uint64_t Idx = IndexOp.getImm();
1858 uint64_t Src1MinLen = Src1Ty.getElementCount().getKnownMinValue();
1859 if (IndexOp.getImm() % Src1MinLen != 0) {
1860 report(msg: "Index must be a multiple of the second source vector's "
1861 "minimum vector length",
1862 MI);
1863 break;
1864 }
1865
1866 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1867 if (Idx >= DstMinLen ||
1868 (!IsMixedFixedIntoScalable && Idx + Src1MinLen > DstMinLen)) {
1869 report(msg: "Subvector type and index must not cause insert to overrun the "
1870 "vector being inserted into",
1871 MI);
1872 break;
1873 }
1874
1875 break;
1876 }
1877 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1878 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1879 if (!SrcOp.isReg()) {
1880 report(msg: "G_EXTRACT_SUBVECTOR first source must be a register", MI);
1881 break;
1882 }
1883
1884 const MachineOperand &IndexOp = MI->getOperand(i: 2);
1885 if (!IndexOp.isImm()) {
1886 report(msg: "G_EXTRACT_SUBVECTOR index must be an immediate", MI);
1887 break;
1888 }
1889
1890 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1891 LLT SrcTy = MRI->getType(Reg: SrcOp.getReg());
1892
1893 if (!DstTy.isVector()) {
1894 report(msg: "Destination type must be a vector", MI);
1895 break;
1896 }
1897
1898 if (!SrcTy.isVector()) {
1899 report(msg: "Source must be a vector", MI);
1900 break;
1901 }
1902
1903 if (DstTy.getElementType() != SrcTy.getElementType()) {
1904 report(msg: "Element type of vectors must be the same", MI);
1905 break;
1906 }
1907
1908 if (DstTy.isScalable() && !SrcTy.isScalable()) {
1909 report(msg: "Cannot extract a scalable vector from a fixed length vector", MI);
1910 break;
1911 }
1912
1913 if (ElementCount::isKnownGT(LHS: DstTy.getElementCount(),
1914 RHS: SrcTy.getElementCount())) {
1915 report(msg: "Destination vector must be smaller than source vector", MI);
1916 break;
1917 }
1918
1919 uint64_t Idx = IndexOp.getImm();
1920 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1921 if (Idx % DstMinLen != 0) {
1922 report(msg: "Index must be a multiple of the destination vector's minimum "
1923 "vector length",
1924 MI);
1925 break;
1926 }
1927
1928 bool IsMixedFixedFromScalable =
1929 DstTy.isFixedVector() && SrcTy.isScalableVector();
1930 uint64_t SrcMinLen = SrcTy.getElementCount().getKnownMinValue();
1931 if (Idx >= SrcMinLen ||
1932 (!IsMixedFixedFromScalable && Idx + DstMinLen > SrcMinLen)) {
1933 report(msg: "Destination type and index must not cause extract to overrun the "
1934 "source vector",
1935 MI);
1936 break;
1937 }
1938
1939 break;
1940 }
1941 case TargetOpcode::G_SHUFFLE_VECTOR: {
1942 const MachineOperand &MaskOp = MI->getOperand(i: 3);
1943 if (!MaskOp.isShuffleMask()) {
1944 report(msg: "Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);
1945 break;
1946 }
1947
1948 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1949 LLT Src0Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1950 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1951
1952 if (Src0Ty != Src1Ty)
1953 report(msg: "Source operands must be the same type", MI);
1954
1955 if (Src0Ty.getScalarType() != DstTy.getScalarType()) {
1956 report(msg: "G_SHUFFLE_VECTOR cannot change element type", MI);
1957 break;
1958 }
1959 if (!Src0Ty.isVector()) {
1960 report(msg: "G_SHUFFLE_VECTOR must have vector src", MI);
1961 break;
1962 }
1963 if (!DstTy.isVector()) {
1964 report(msg: "G_SHUFFLE_VECTOR must have vector dst", MI);
1965 break;
1966 }
1967
1968 // Don't check that all operands are vector because scalars are used in
1969 // place of 1 element vectors.
1970 int SrcNumElts = Src0Ty.getNumElements();
1971 int DstNumElts = DstTy.getNumElements();
1972
1973 ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();
1974
1975 if (static_cast<int>(MaskIdxes.size()) != DstNumElts)
1976 report(msg: "Wrong result type for shufflemask", MI);
1977
1978 for (int Idx : MaskIdxes) {
1979 if (Idx < 0)
1980 continue;
1981
1982 if (Idx >= 2 * SrcNumElts)
1983 report(msg: "Out of bounds shuffle index", MI);
1984 }
1985
1986 break;
1987 }
1988
1989 case TargetOpcode::G_SPLAT_VECTOR: {
1990 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1991 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1992
1993 if (!DstTy.isScalableVector()) {
1994 report(msg: "Destination type must be a scalable vector", MI);
1995 break;
1996 }
1997
1998 if (!SrcTy.isScalar() && !SrcTy.isPointer()) {
1999 report(msg: "Source type must be a scalar or pointer", MI);
2000 break;
2001 }
2002
2003 if (TypeSize::isKnownGT(LHS: DstTy.getElementType().getSizeInBits(),
2004 RHS: SrcTy.getSizeInBits())) {
2005 report(msg: "Element type of the destination must be the same size or smaller "
2006 "than the source type",
2007 MI);
2008 break;
2009 }
2010
2011 break;
2012 }
2013 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2014 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2015 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2016 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2017
2018 if (!DstTy.isScalar() && !DstTy.isPointer()) {
2019 report(msg: "Destination type must be a scalar or pointer", MI);
2020 break;
2021 }
2022
2023 if (!SrcTy.isVector()) {
2024 report(msg: "First source must be a vector", MI);
2025 break;
2026 }
2027
2028 auto TLI = MF->getSubtarget().getTargetLowering();
2029 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2030 report(msg: "Index type must match VectorIdxTy", MI);
2031 break;
2032 }
2033
2034 break;
2035 }
2036 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2037 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2038 LLT VecTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2039 LLT ScaTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2040 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 3).getReg());
2041
2042 if (!DstTy.isVector()) {
2043 report(msg: "Destination type must be a vector", MI);
2044 break;
2045 }
2046
2047 if (VecTy != DstTy) {
2048 report(msg: "Destination type and vector type must match", MI);
2049 break;
2050 }
2051
2052 if (!ScaTy.isScalar() && !ScaTy.isPointer()) {
2053 report(msg: "Inserted element must be a scalar or pointer", MI);
2054 break;
2055 }
2056
2057 auto TLI = MF->getSubtarget().getTargetLowering();
2058 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2059 report(msg: "Index type must match VectorIdxTy", MI);
2060 break;
2061 }
2062
2063 break;
2064 }
2065 case TargetOpcode::G_DYN_STACKALLOC: {
2066 const MachineOperand &DstOp = MI->getOperand(i: 0);
2067 const MachineOperand &AllocOp = MI->getOperand(i: 1);
2068 const MachineOperand &AlignOp = MI->getOperand(i: 2);
2069
2070 if (!DstOp.isReg() || !MRI->getType(Reg: DstOp.getReg()).isPointer()) {
2071 report(msg: "dst operand 0 must be a pointer type", MI);
2072 break;
2073 }
2074
2075 if (!AllocOp.isReg() || !MRI->getType(Reg: AllocOp.getReg()).isScalar()) {
2076 report(msg: "src operand 1 must be a scalar reg type", MI);
2077 break;
2078 }
2079
2080 if (!AlignOp.isImm()) {
2081 report(msg: "src operand 2 must be an immediate type", MI);
2082 break;
2083 }
2084 break;
2085 }
2086 case TargetOpcode::G_MEMCPY_INLINE:
2087 case TargetOpcode::G_MEMCPY:
2088 case TargetOpcode::G_MEMMOVE: {
2089 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2090 if (MMOs.size() != 2) {
2091 report(msg: "memcpy/memmove must have 2 memory operands", MI);
2092 break;
2093 }
2094
2095 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) ||
2096 (MMOs[1]->isStore() || !MMOs[1]->isLoad())) {
2097 report(msg: "wrong memory operand types", MI);
2098 break;
2099 }
2100
2101 if (MMOs[0]->getSize() != MMOs[1]->getSize())
2102 report(msg: "inconsistent memory operand sizes", MI);
2103
2104 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2105 LLT SrcPtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2106
2107 if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) {
2108 report(msg: "memory instruction operand must be a pointer", MI);
2109 break;
2110 }
2111
2112 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2113 report(msg: "inconsistent store address space", MI);
2114 if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())
2115 report(msg: "inconsistent load address space", MI);
2116
2117 if (Opc != TargetOpcode::G_MEMCPY_INLINE)
2118 if (!MI->getOperand(i: 3).isImm() || (MI->getOperand(i: 3).getImm() & ~1LL))
2119 report(msg: "'tail' flag (operand 3) must be an immediate 0 or 1", MI);
2120
2121 break;
2122 }
2123 case TargetOpcode::G_BZERO:
2124 case TargetOpcode::G_MEMSET:
2125 case TargetOpcode::G_MEMSET_INLINE: {
2126 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2127 std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset"
2128 : Opc == TargetOpcode::G_MEMSET_INLINE ? "memset_inline"
2129 : "bzero";
2130 if (MMOs.size() != 1) {
2131 report(Msg: Twine(Name, " must have 1 memory operand"), MI);
2132 break;
2133 }
2134
2135 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
2136 report(Msg: Twine(Name, " memory operand must be a store"), MI);
2137 break;
2138 }
2139
2140 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2141 if (!DstPtrTy.isPointer()) {
2142 report(Msg: Twine(Name, " operand must be a pointer"), MI);
2143 break;
2144 }
2145
2146 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2147 report(Msg: "inconsistent " + Twine(Name, " address space"), MI);
2148
2149 if (Opc != TargetOpcode::G_MEMSET_INLINE) {
2150 if (!MI->getOperand(i: MI->getNumOperands() - 1).isImm() ||
2151 (MI->getOperand(i: MI->getNumOperands() - 1).getImm() & ~1LL))
2152 report(msg: "'tail' flag (last operand) must be an immediate 0 or 1", MI);
2153 }
2154
2155 break;
2156 }
2157 case TargetOpcode::G_UBSANTRAP: {
2158 const MachineOperand &KindOp = MI->getOperand(i: 0);
2159 if (!MI->getOperand(i: 0).isImm()) {
2160 report(msg: "Crash kind must be an immediate", MO: &KindOp, MONum: 0);
2161 break;
2162 }
2163 int64_t Kind = MI->getOperand(i: 0).getImm();
2164 if (!isInt<8>(x: Kind))
2165 report(msg: "Crash kind must be 8 bit wide", MO: &KindOp, MONum: 0);
2166 break;
2167 }
2168 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
2169 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
2170 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2171 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2172 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2173 if (!DstTy.isScalar())
2174 report(msg: "Vector reduction requires a scalar destination type", MI);
2175 if (!Src1Ty.isScalar())
2176 report(msg: "Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI);
2177 if (!Src2Ty.isVector())
2178 report(msg: "Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);
2179 break;
2180 }
2181 case TargetOpcode::G_VECREDUCE_FADD:
2182 case TargetOpcode::G_VECREDUCE_FMUL:
2183 case TargetOpcode::G_VECREDUCE_FMAX:
2184 case TargetOpcode::G_VECREDUCE_FMIN:
2185 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
2186 case TargetOpcode::G_VECREDUCE_FMINIMUM:
2187 case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
2188 case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
2189 case TargetOpcode::G_VECREDUCE_ADD:
2190 case TargetOpcode::G_VECREDUCE_MUL:
2191 case TargetOpcode::G_VECREDUCE_AND:
2192 case TargetOpcode::G_VECREDUCE_OR:
2193 case TargetOpcode::G_VECREDUCE_XOR:
2194 case TargetOpcode::G_VECREDUCE_SMAX:
2195 case TargetOpcode::G_VECREDUCE_SMIN:
2196 case TargetOpcode::G_VECREDUCE_UMAX:
2197 case TargetOpcode::G_VECREDUCE_UMIN: {
2198 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2199 if (!DstTy.isScalar())
2200 report(msg: "Vector reduction requires a scalar destination type", MI);
2201 break;
2202 }
2203
2204 case TargetOpcode::G_SBFX:
2205 case TargetOpcode::G_UBFX: {
2206 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2207 if (DstTy.isVector()) {
2208 report(msg: "Bitfield extraction is not supported on vectors", MI);
2209 break;
2210 }
2211 break;
2212 }
2213 case TargetOpcode::G_SHL:
2214 case TargetOpcode::G_LSHR:
2215 case TargetOpcode::G_ASHR:
2216 case TargetOpcode::G_ROTR:
2217 case TargetOpcode::G_ROTL: {
2218 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2219 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2220 if (Src1Ty.isVector() != Src2Ty.isVector()) {
2221 report(msg: "Shifts and rotates require operands to be either all scalars or "
2222 "all vectors",
2223 MI);
2224 break;
2225 }
2226 break;
2227 }
2228 case TargetOpcode::G_LLROUND:
2229 case TargetOpcode::G_LROUND: {
2230 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2231 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2232 if (!DstTy.isValid() || !SrcTy.isValid())
2233 break;
2234 if (SrcTy.isPointer() || DstTy.isPointer()) {
2235 StringRef Op = SrcTy.isPointer() ? "Source" : "Destination";
2236 report(Msg: Twine(Op, " operand must not be a pointer type"), MI);
2237 } else if (SrcTy.isScalar()) {
2238 verifyAllRegOpsScalar(MI: *MI, MRI: *MRI);
2239 break;
2240 } else if (SrcTy.isVector()) {
2241 verifyVectorElementMatch(Ty0: SrcTy, Ty1: DstTy, MI);
2242 break;
2243 }
2244 break;
2245 }
2246 case TargetOpcode::G_IS_FPCLASS: {
2247 LLT DestTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2248 LLT DestEltTy = DestTy.getScalarType();
2249 if (!DestEltTy.isScalar()) {
2250 report(msg: "Destination must be a scalar or vector of scalars", MI);
2251 break;
2252 }
2253 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2254 LLT SrcEltTy = SrcTy.getScalarType();
2255 if (!SrcEltTy.isScalar()) {
2256 report(msg: "Source must be a scalar or vector of scalars", MI);
2257 break;
2258 }
2259 if (!verifyVectorElementMatch(Ty0: DestTy, Ty1: SrcTy, MI))
2260 break;
2261 const MachineOperand &TestMO = MI->getOperand(i: 2);
2262 if (!TestMO.isImm()) {
2263 report(msg: "floating-point class set (operand 2) must be an immediate", MI);
2264 break;
2265 }
2266 int64_t Test = TestMO.getImm();
2267 if (Test < 0 || Test > fcAllFlags) {
2268 report(msg: "Incorrect floating-point class set (operand 2)", MI);
2269 break;
2270 }
2271 break;
2272 }
2273 case TargetOpcode::G_PREFETCH: {
2274 const MachineOperand &AddrOp = MI->getOperand(i: 0);
2275 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer()) {
2276 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 0);
2277 break;
2278 }
2279 const MachineOperand &RWOp = MI->getOperand(i: 1);
2280 if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {
2281 report(msg: "rw operand must be an immediate 0-1", MO: &RWOp, MONum: 1);
2282 break;
2283 }
2284 const MachineOperand &LocalityOp = MI->getOperand(i: 2);
2285 if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {
2286 report(msg: "locality operand must be an immediate 0-3", MO: &LocalityOp, MONum: 2);
2287 break;
2288 }
2289 const MachineOperand &CacheTypeOp = MI->getOperand(i: 3);
2290 if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {
2291 report(msg: "cache type operand must be an immediate 0-1", MO: &CacheTypeOp, MONum: 3);
2292 break;
2293 }
2294 break;
2295 }
2296 case TargetOpcode::G_ASSERT_ALIGN: {
2297 if (MI->getOperand(i: 2).getImm() < 1)
2298 report(msg: "alignment immediate must be >= 1", MI);
2299 break;
2300 }
2301 case TargetOpcode::G_CONSTANT_POOL: {
2302 if (!MI->getOperand(i: 1).isCPI())
2303 report(msg: "Src operand 1 must be a constant pool index", MI);
2304 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
2305 report(msg: "Dst operand 0 must be a pointer", MI);
2306 break;
2307 }
2308 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {
2309 const MachineOperand &AddrOp = MI->getOperand(i: 1);
2310 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer())
2311 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 1);
2312 break;
2313 }
2314 case TargetOpcode::G_SMIN:
2315 case TargetOpcode::G_SMAX:
2316 case TargetOpcode::G_UMIN:
2317 case TargetOpcode::G_UMAX: {
2318 const LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2319 if (DstTy.isPointerOrPointerVector())
2320 report(msg: "Generic smin/smax/umin/umax does not support pointer operands",
2321 MI);
2322 break;
2323 }
2324 default:
2325 break;
2326 }
2327}
2328
2329void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
2330 const MCInstrDesc &MCID = MI->getDesc();
2331 if (MI->getNumOperands() < MCID.getNumOperands()) {
2332 report(msg: "Too few operands", MI);
2333 OS << MCID.getNumOperands() << " operands expected, but "
2334 << MI->getNumOperands() << " given.\n";
2335 }
2336
2337 if (MI->getFlag(Flag: MachineInstr::NoConvergent) && !MCID.isConvergent())
2338 report(msg: "NoConvergent flag expected only on convergent instructions.", MI);
2339
2340 if (MI->isPHI()) {
2341 if (MF->getProperties().hasNoPHIs())
2342 report(msg: "Found PHI instruction with NoPHIs property set", MI);
2343
2344 if (FirstNonPHI)
2345 report(msg: "Found PHI instruction after non-PHI", MI);
2346 } else if (FirstNonPHI == nullptr)
2347 FirstNonPHI = MI;
2348
2349 // Check the tied operands.
2350 if (MI->isInlineAsm())
2351 verifyInlineAsm(MI);
2352
2353 // Check that unspillable terminators define a reg and have at most one use.
2354 if (TII->isUnspillableTerminator(MI)) {
2355 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
2356 report(msg: "Unspillable Terminator does not define a reg", MI);
2357 Register Def = MI->getOperand(i: 0).getReg();
2358 if (Def.isVirtual() && hasPHIs(MF: *MF) &&
2359 std::distance(first: MRI->use_nodbg_begin(RegNo: Def), last: MRI->use_nodbg_end()) > 1)
2360 report(msg: "Unspillable Terminator expected to have at most one use!", MI);
2361 }
2362
2363 // A fully-formed DBG_VALUE must have a location. Ignore partially formed
2364 // DBG_VALUEs: these are convenient to use in tests, but should never get
2365 // generated.
2366 if (MI->isDebugValue() && MI->getNumOperands() == 4)
2367 if (!MI->getDebugLoc())
2368 report(msg: "Missing DebugLoc for debug instruction", MI);
2369
2370 // Meta instructions should never be the subject of debug value tracking,
2371 // they don't create a value in the output program at all.
2372 if (MI->isMetaInstruction() && MI->peekDebugInstrNum())
2373 report(msg: "Metadata instruction should not have a value tracking number", MI);
2374
2375 // Check the MachineMemOperands for basic consistency.
2376 for (MachineMemOperand *Op : MI->memoperands()) {
2377 if (Op->isLoad() && !MI->mayLoad())
2378 report(msg: "Missing mayLoad flag", MI);
2379 if (Op->isStore() && !MI->mayStore())
2380 report(msg: "Missing mayStore flag", MI);
2381 }
2382
2383 // Debug values must not have a slot index.
2384 // Other instructions must have one, unless they are inside a bundle.
2385 if (LiveInts) {
2386 bool mapped = !LiveInts->isNotInMIMap(Instr: *MI);
2387 if (MI->isDebugOrPseudoInstr()) {
2388 if (mapped)
2389 report(msg: "Debug instruction has a slot index", MI);
2390 } else if (MI->isInsideBundle()) {
2391 if (mapped)
2392 report(msg: "Instruction inside bundle has a slot index", MI);
2393 } else {
2394 if (!mapped)
2395 report(msg: "Missing slot index", MI);
2396 }
2397 }
2398
2399 unsigned Opc = MCID.getOpcode();
2400 if (isPreISelGenericOpcode(Opcode: Opc) || isPreISelGenericOptimizationHint(Opcode: Opc)) {
2401 verifyPreISelGenericInstruction(MI);
2402 return;
2403 }
2404
2405 StringRef ErrorInfo;
2406 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
2407 report(msg: ErrorInfo.data(), MI);
2408
2409 // Verify properties of various specific instruction types
2410 switch (MI->getOpcode()) {
2411 case TargetOpcode::COPY: {
2412 const MachineOperand &DstOp = MI->getOperand(i: 0);
2413 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2414 const Register SrcReg = SrcOp.getReg();
2415 const Register DstReg = DstOp.getReg();
2416
2417 LLT DstTy = MRI->getType(Reg: DstReg);
2418 LLT SrcTy = MRI->getType(Reg: SrcReg);
2419 if (SrcTy.isValid() && DstTy.isValid()) {
2420 // If both types are valid, check that the types are the same.
2421 if (SrcTy != DstTy) {
2422 report(msg: "Copy Instruction is illegal with mismatching types", MI);
2423 OS << "Def = " << DstTy << ", Src = " << SrcTy << '\n';
2424 }
2425
2426 break;
2427 }
2428
2429 if (!SrcTy.isValid() && !DstTy.isValid())
2430 break;
2431
2432 // If we have only one valid type, this is likely a copy between a virtual
2433 // and physical register.
2434 TypeSize SrcSize = TypeSize::getZero();
2435 TypeSize DstSize = TypeSize::getZero();
2436 if (SrcReg.isPhysical() && DstTy.isValid()) {
2437 if (!hasPhysRegClassForType(TRI: *TRI, Reg: SrcReg, Ty: DstTy))
2438 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2439 } else {
2440 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2441 }
2442
2443 if (DstReg.isPhysical() && SrcTy.isValid()) {
2444 if (!hasPhysRegClassForType(TRI: *TRI, Reg: DstReg, Ty: SrcTy))
2445 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2446 } else {
2447 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2448 }
2449
2450 // The next two checks allow COPY between physical and virtual registers,
2451 // when the virtual register has a scalable size and the physical register
2452 // has a fixed size. These checks allow COPY between *potentially*
2453 // mismatched sizes. However, once RegisterBankSelection occurs,
2454 // MachineVerifier should be able to resolve a fixed size for the scalable
2455 // vector, and at that point this function will know for sure whether the
2456 // sizes are mismatched and correctly report a size mismatch.
2457 if (SrcReg.isPhysical() && DstReg.isVirtual() && DstSize.isScalable() &&
2458 !SrcSize.isScalable())
2459 break;
2460 if (SrcReg.isVirtual() && DstReg.isPhysical() && SrcSize.isScalable() &&
2461 !DstSize.isScalable())
2462 break;
2463
2464 if (SrcSize.isNonZero() && DstSize.isNonZero() && SrcSize != DstSize) {
2465 if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {
2466 report(msg: "Copy Instruction is illegal with mismatching sizes", MI);
2467 OS << "Def Size = " << DstSize << ", Src Size = " << SrcSize << '\n';
2468 }
2469 }
2470 break;
2471 }
2472 case TargetOpcode::COPY_LANEMASK: {
2473 const MachineOperand &DstOp = MI->getOperand(i: 0);
2474 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2475 const MachineOperand &LaneMaskOp = MI->getOperand(i: 2);
2476 const Register SrcReg = SrcOp.getReg();
2477 const LaneBitmask LaneMask = LaneMaskOp.getLaneMask();
2478 LaneBitmask SrcMaxLaneMask = LaneBitmask::getAll();
2479
2480 if (DstOp.getSubReg())
2481 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &DstOp, MONum: 0);
2482
2483 if (SrcOp.getSubReg())
2484 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &SrcOp, MONum: 1);
2485
2486 if (LaneMask.none())
2487 report(msg: "COPY_LANEMASK must read at least one lane", MI);
2488
2489 if (SrcReg.isPhysical()) {
2490 const TargetRegisterClass *SrcRC = TRI->getMinimalPhysRegClass(Reg: SrcReg);
2491 if (SrcRC)
2492 SrcMaxLaneMask = SrcRC->getLaneMask();
2493 } else {
2494 SrcMaxLaneMask = MRI->getMaxLaneMaskForVReg(Reg: SrcReg);
2495 }
2496
2497 // COPY_LANEMASK should be used only for partial copy. For full
2498 // copy, one should strictly use the COPY instruction.
2499 if (SrcMaxLaneMask == LaneMask)
2500 report(msg: "COPY_LANEMASK cannot be used to do full copy", MI);
2501
2502 // If LaneMask is greater than the SrcMaxLaneMask, it implies
2503 // COPY_LANEMASK is attempting to read from the lanes that
2504 // don't exists in the source register.
2505 if (SrcMaxLaneMask < LaneMask)
2506 report(msg: "COPY_LANEMASK attempts to read from the lanes that "
2507 "don't exist in the source register",
2508 MI);
2509
2510 break;
2511 }
2512 case TargetOpcode::STATEPOINT: {
2513 StatepointOpers SO(MI);
2514 if (!MI->getOperand(i: SO.getIDPos()).isImm() ||
2515 !MI->getOperand(i: SO.getNBytesPos()).isImm() ||
2516 !MI->getOperand(i: SO.getNCallArgsPos()).isImm()) {
2517 report(msg: "meta operands to STATEPOINT not constant!", MI);
2518 break;
2519 }
2520
2521 auto VerifyStackMapConstant = [&](unsigned Offset) {
2522 if (Offset >= MI->getNumOperands()) {
2523 report(msg: "stack map constant to STATEPOINT is out of range!", MI);
2524 return;
2525 }
2526 if (!MI->getOperand(i: Offset - 1).isImm() ||
2527 MI->getOperand(i: Offset - 1).getImm() != StackMaps::ConstantOp ||
2528 !MI->getOperand(i: Offset).isImm())
2529 report(msg: "stack map constant to STATEPOINT not well formed!", MI);
2530 };
2531 VerifyStackMapConstant(SO.getCCIdx());
2532 VerifyStackMapConstant(SO.getFlagsIdx());
2533 VerifyStackMapConstant(SO.getNumDeoptArgsIdx());
2534 VerifyStackMapConstant(SO.getNumGCPtrIdx());
2535 VerifyStackMapConstant(SO.getNumAllocaIdx());
2536 VerifyStackMapConstant(SO.getNumGcMapEntriesIdx());
2537
2538 // Verify that all explicit statepoint defs are tied to gc operands as
2539 // they are expected to be a relocation of gc operands.
2540 unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx();
2541 unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2;
2542 for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) {
2543 unsigned UseOpIdx;
2544 if (!MI->isRegTiedToUseOperand(DefOpIdx: Idx, UseOpIdx: &UseOpIdx)) {
2545 report(msg: "STATEPOINT defs expected to be tied", MI);
2546 break;
2547 }
2548 if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) {
2549 report(msg: "STATEPOINT def tied to non-gc operand", MI);
2550 break;
2551 }
2552 }
2553
2554 // TODO: verify we have properly encoded deopt arguments
2555 } break;
2556 case TargetOpcode::INSERT_SUBREG: {
2557 unsigned InsertedSize;
2558 if (unsigned SubIdx = MI->getOperand(i: 2).getSubReg())
2559 InsertedSize = TRI->getSubRegIdxSize(Idx: SubIdx);
2560 else
2561 InsertedSize = TRI->getRegSizeInBits(Reg: MI->getOperand(i: 2).getReg(), MRI: *MRI);
2562 unsigned SubRegSize = TRI->getSubRegIdxSize(Idx: MI->getOperand(i: 3).getImm());
2563 if (SubRegSize < InsertedSize) {
2564 report(msg: "INSERT_SUBREG expected inserted value to have equal or lesser "
2565 "size than the subreg it was inserted into", MI);
2566 break;
2567 }
2568 } break;
2569 case TargetOpcode::REG_SEQUENCE: {
2570 unsigned NumOps = MI->getNumOperands();
2571 if (!(NumOps & 1)) {
2572 report(msg: "Invalid number of operands for REG_SEQUENCE", MI);
2573 break;
2574 }
2575
2576 for (unsigned I = 1; I != NumOps; I += 2) {
2577 const MachineOperand &RegOp = MI->getOperand(i: I);
2578 const MachineOperand &SubRegOp = MI->getOperand(i: I + 1);
2579
2580 if (!RegOp.isReg())
2581 report(msg: "Invalid register operand for REG_SEQUENCE", MO: &RegOp, MONum: I);
2582
2583 if (!SubRegOp.isImm() || SubRegOp.getImm() == 0 ||
2584 SubRegOp.getImm() >= TRI->getNumSubRegIndices()) {
2585 report(msg: "Invalid subregister index operand for REG_SEQUENCE",
2586 MO: &SubRegOp, MONum: I + 1);
2587 }
2588 }
2589
2590 Register DstReg = MI->getOperand(i: 0).getReg();
2591 if (DstReg.isPhysical())
2592 report(msg: "REG_SEQUENCE does not support physical register results", MI);
2593
2594 if (MI->getOperand(i: 0).getSubReg())
2595 report(msg: "Invalid subreg result for REG_SEQUENCE", MI);
2596
2597 break;
2598 }
2599 }
2600}
2601
2602void
2603MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
2604 const MachineInstr *MI = MO->getParent();
2605 const MCInstrDesc &MCID = MI->getDesc();
2606 unsigned NumDefs = MCID.getNumDefs();
2607 if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)
2608 NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;
2609
2610 // The first MCID.NumDefs operands must be explicit register defines
2611 if (MONum < NumDefs) {
2612 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2613 if (!MO->isReg())
2614 report(msg: "Explicit definition must be a register", MO, MONum);
2615 else if (!MO->isDef() && !MCOI.isOptionalDef())
2616 report(msg: "Explicit definition marked as use", MO, MONum);
2617 else if (MO->isImplicit())
2618 report(msg: "Explicit definition marked as implicit", MO, MONum);
2619 } else if (MONum < MCID.getNumOperands()) {
2620 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2621 // Don't check if it's the last operand in a variadic instruction. See,
2622 // e.g., LDM_RET in the arm back end. Check non-variadic operands only.
2623 bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1;
2624 if (!IsOptional) {
2625 if (MO->isReg()) {
2626 if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs())
2627 report(msg: "Explicit operand marked as def", MO, MONum);
2628 if (MO->isImplicit())
2629 report(msg: "Explicit operand marked as implicit", MO, MONum);
2630 }
2631
2632 // Check that an instruction has register operands only as expected.
2633 if (MCOI.OperandType == MCOI::OPERAND_REGISTER &&
2634 !MO->isReg() && !MO->isFI())
2635 report(msg: "Expected a register operand.", MO, MONum);
2636 if (MO->isReg()) {
2637 if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE ||
2638 (MCOI.OperandType == MCOI::OPERAND_PCREL &&
2639 !TII->isPCRelRegisterOperandLegal(MI: *MI, OpIdx: MONum)))
2640 report(msg: "Expected a non-register operand.", MO, MONum);
2641 }
2642 }
2643
2644 int TiedTo = MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::TIED_TO);
2645 if (TiedTo != -1) {
2646 if (!MO->isReg())
2647 report(msg: "Tied use must be a register", MO, MONum);
2648 else if (!MO->isTied())
2649 report(msg: "Operand should be tied", MO, MONum);
2650 else if (unsigned(TiedTo) != MI->findTiedOperandIdx(OpIdx: MONum))
2651 report(msg: "Tied def doesn't match MCInstrDesc", MO, MONum);
2652 else if (MO->getReg().isPhysical()) {
2653 const MachineOperand &MOTied = MI->getOperand(i: TiedTo);
2654 if (!MOTied.isReg())
2655 report(msg: "Tied counterpart must be a register", MO: &MOTied, MONum: TiedTo);
2656 else if (MOTied.getReg().isPhysical() &&
2657 MO->getReg() != MOTied.getReg())
2658 report(msg: "Tied physical registers must match.", MO: &MOTied, MONum: TiedTo);
2659 }
2660 } else if (MO->isReg() && MO->isTied())
2661 report(msg: "Explicit operand should not be tied", MO, MONum);
2662 } else if (!MI->isVariadic()) {
2663 // ARM adds %reg0 operands to indicate predicates. We'll allow that.
2664 if (!MO->isValidExcessOperand())
2665 report(msg: "Extra explicit operand on non-variadic instruction", MO, MONum);
2666 }
2667
2668 // Verify earlyClobber def operand
2669 if (MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::EARLY_CLOBBER) != -1) {
2670 if (!MO->isReg())
2671 report(msg: "Early clobber must be a register", MI);
2672 if (!MO->isEarlyClobber())
2673 report(msg: "Missing earlyClobber flag", MI);
2674 }
2675
2676 switch (MO->getType()) {
2677 case MachineOperand::MO_Register: {
2678 // Verify debug flag on debug instructions. Check this first because reg0
2679 // indicates an undefined debug value.
2680 if (MI->isDebugInstr() && MO->isUse()) {
2681 if (!MO->isDebug())
2682 report(msg: "Register operand must be marked debug", MO, MONum);
2683 } else if (MO->isDebug()) {
2684 report(msg: "Register operand must not be marked debug", MO, MONum);
2685 }
2686
2687 const Register Reg = MO->getReg();
2688 if (!Reg)
2689 return;
2690 if (MRI->tracksLiveness() && !MI->isDebugInstr())
2691 checkLiveness(MO, MONum);
2692
2693 if (MO->isDef() && MO->isUndef() && !MO->getSubReg() &&
2694 MO->getReg().isVirtual()) // TODO: Apply to physregs too
2695 report(msg: "Undef virtual register def operands require a subregister", MO, MONum);
2696
2697 // Verify the consistency of tied operands.
2698 if (MO->isTied()) {
2699 unsigned OtherIdx = MI->findTiedOperandIdx(OpIdx: MONum);
2700 const MachineOperand &OtherMO = MI->getOperand(i: OtherIdx);
2701 if (!OtherMO.isReg())
2702 report(msg: "Must be tied to a register", MO, MONum);
2703 if (!OtherMO.isTied())
2704 report(msg: "Missing tie flags on tied operand", MO, MONum);
2705 if (MI->findTiedOperandIdx(OpIdx: OtherIdx) != MONum)
2706 report(msg: "Inconsistent tie links", MO, MONum);
2707
2708 // See IsUndef in MachineOperand.h.
2709 if (MO->isUse() && MO->isUndef() && Reg.isVirtual() &&
2710 OtherMO.getReg() != Reg &&
2711 any_of(Range: MI->all_uses(), P: [&](const MachineOperand &Other) {
2712 return &Other != MO && Other.isUndef() && Other.getReg() == Reg &&
2713 Other.getSubReg() == MO->getSubReg();
2714 }))
2715 report(msg: "Tied undef use shares a virtual register with another read", MO,
2716 MONum);
2717
2718 if (MONum < MCID.getNumDefs()) {
2719 if (OtherIdx < MCID.getNumOperands()) {
2720 if (-1 == MCID.getOperandConstraint(OpNum: OtherIdx, Constraint: MCOI::TIED_TO))
2721 report(msg: "Explicit def tied to explicit use without tie constraint",
2722 MO, MONum);
2723 } else {
2724 if (!OtherMO.isImplicit())
2725 report(msg: "Explicit def should be tied to implicit use", MO, MONum);
2726 }
2727 }
2728 }
2729
2730 // Verify two-address constraints after the twoaddressinstruction pass.
2731 // Both twoaddressinstruction pass and phi-node-elimination pass call
2732 // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification
2733 // after twoaddressinstruction pass not after phi-node-elimination pass. So
2734 // we shouldn't use the IsSSA as the condition, we should based on
2735 // TiedOpsRewritten property to verify two-address constraints, this
2736 // property will be set in twoaddressinstruction pass.
2737 unsigned DefIdx;
2738 if (MF->getProperties().hasTiedOpsRewritten() && MO->isUse() &&
2739 MI->isRegTiedToDefOperand(UseOpIdx: MONum, DefOpIdx: &DefIdx) &&
2740 Reg != MI->getOperand(i: DefIdx).getReg())
2741 report(msg: "Two-address instruction operands must be identical", MO, MONum);
2742
2743 // Check register classes.
2744 unsigned SubIdx = MO->getSubReg();
2745
2746 if (Reg.isPhysical()) {
2747 if (SubIdx) {
2748 report(msg: "Illegal subregister index for physical register", MO, MONum);
2749 return;
2750 }
2751 if (MONum < MCID.getNumOperands()) {
2752 if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum)) {
2753 if (!DRC->contains(Reg)) {
2754 report(msg: "Illegal physical register for instruction", MO, MONum);
2755 OS << printReg(Reg, TRI) << " is not a "
2756 << TRI->getRegClassName(Class: DRC) << " register.\n";
2757 }
2758 }
2759 }
2760 if (MO->isRenamable()) {
2761 if (MRI->isReserved(PhysReg: Reg)) {
2762 report(msg: "isRenamable set on reserved register", MO, MONum);
2763 return;
2764 }
2765 }
2766 } else {
2767 // Virtual register.
2768 const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);
2769 if (!RC) {
2770 // This is a generic virtual register.
2771
2772 // Do not allow undef uses for generic virtual registers. This ensures
2773 // getVRegDef can never fail and return null on a generic register.
2774 //
2775 // FIXME: This restriction should probably be broadened to all SSA
2776 // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still
2777 // run on the SSA function just before phi elimination.
2778 if (MO->isUndef())
2779 report(msg: "Generic virtual register use cannot be undef", MO, MONum);
2780
2781 // Debug value instruction is permitted to use undefined vregs.
2782 // This is a performance measure to skip the overhead of immediately
2783 // pruning unused debug operands. The final undef substitution occurs
2784 // when debug values are allocated in LDVImpl::handleDebugValue, so
2785 // these verifications always apply after this pass.
2786 if (isFunctionTracksDebugUserValues || !MO->isUse() ||
2787 !MI->isDebugValue() || !MRI->def_empty(RegNo: Reg)) {
2788 // If we're post-Select, we can't have gvregs anymore.
2789 if (isFunctionSelected) {
2790 report(msg: "Generic virtual register invalid in a Selected function",
2791 MO, MONum);
2792 return;
2793 }
2794
2795 // The gvreg must have a type and it must not have a SubIdx.
2796 LLT Ty = MRI->getType(Reg);
2797 if (!Ty.isValid()) {
2798 report(msg: "Generic virtual register must have a valid type", MO,
2799 MONum);
2800 return;
2801 }
2802
2803 const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
2804 const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo();
2805
2806 // If we're post-RegBankSelect, the gvreg must have a bank.
2807 if (!RegBank && isFunctionRegBankSelected) {
2808 report(msg: "Generic virtual register must have a bank in a "
2809 "RegBankSelected function",
2810 MO, MONum);
2811 return;
2812 }
2813
2814 // Make sure the register fits into its register bank if any.
2815 if (RegBank && Ty.isValid() && !Ty.isScalableVector() &&
2816 RBI->getMaximumSize(RegBankID: RegBank->getID()) < Ty.getSizeInBits()) {
2817 report(msg: "Register bank is too small for virtual register", MO,
2818 MONum);
2819 OS << "Register bank " << RegBank->getName() << " too small("
2820 << RBI->getMaximumSize(RegBankID: RegBank->getID()) << ") to fit "
2821 << Ty.getSizeInBits() << "-bits\n";
2822 return;
2823 }
2824 }
2825
2826 if (SubIdx) {
2827 report(msg: "Generic virtual register does not allow subregister index", MO,
2828 MONum);
2829 return;
2830 }
2831
2832 // If this is a target specific instruction and this operand
2833 // has register class constraint, the virtual register must
2834 // comply to it.
2835 if (!isPreISelGenericOpcode(Opcode: MCID.getOpcode()) &&
2836 MONum < MCID.getNumOperands() && TII->getRegClass(MCID, OpNum: MONum)) {
2837 report(msg: "Virtual register does not match instruction constraint", MO,
2838 MONum);
2839 OS << "Expect register class "
2840 << TRI->getRegClassName(Class: TII->getRegClass(MCID, OpNum: MONum))
2841 << " but got nothing\n";
2842 return;
2843 }
2844
2845 break;
2846 }
2847 // Validate that SubIdx can be applied to the virtual register.
2848 if (!TRI->isSubRegValidForRegClass(RC, Idx: SubIdx)) {
2849 report(msg: "Invalid subregister index for virtual register", MO, MONum);
2850 OS << "Register class " << TRI->getRegClassName(Class: RC)
2851 << " does not support subreg index "
2852 << TRI->getSubRegIndexName(SubIdx) << '\n';
2853 return;
2854 }
2855 if (MONum >= MCID.getNumOperands())
2856 break;
2857 const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum);
2858 if (!DRC)
2859 break;
2860
2861 // If SubIdx is used, verify that RC with SubIdx can be used for an
2862 // operand of class DRC. This is valid if for every register in RC, the
2863 // register obtained by applying SubIdx to it is in DRC.
2864 if (SubIdx && TRI->getMatchingSuperRegClass(A: RC, B: DRC, Idx: SubIdx) != RC) {
2865 report(msg: "Illegal virtual register for instruction", MO, MONum);
2866 OS << TRI->getRegClassName(Class: RC) << "." << TRI->getSubRegIndexName(SubIdx)
2867 << " cannot be used for " << TRI->getRegClassName(Class: DRC)
2868 << " operands.";
2869 }
2870
2871 // If no SubIdx is used, verify that RC is a sub-class of DRC.
2872 if (!SubIdx && !RC->hasSuperClassEq(RC: DRC)) {
2873 report(msg: "Illegal virtual register for instruction", MO, MONum);
2874 OS << "Expected a " << TRI->getRegClassName(Class: DRC)
2875 << " register, but got a " << TRI->getRegClassName(Class: RC)
2876 << " register\n";
2877 }
2878 }
2879 break;
2880 }
2881
2882 case MachineOperand::MO_RegisterMask:
2883 regMasks.push_back(Elt: MO->getRegMask());
2884 break;
2885
2886 case MachineOperand::MO_MachineBasicBlock:
2887 if (MI->isPHI() && !MO->getMBB()->isSuccessor(MBB: MI->getParent()))
2888 report(msg: "PHI operand is not in the CFG", MO, MONum);
2889 break;
2890
2891 case MachineOperand::MO_FrameIndex:
2892 if (LiveStks && LiveStks->hasInterval(Slot: MO->getIndex()) &&
2893 LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
2894 int FI = MO->getIndex();
2895 LiveInterval &LI = LiveStks->getInterval(Slot: FI);
2896 SlotIndex Idx = LiveInts->getInstructionIndex(Instr: *MI);
2897
2898 bool MayStore = MI->mayStore();
2899 bool MayLoad = MI->mayLoad();
2900 // For a memory-to-memory move, we need to check if the frame
2901 // index is used for storing or loading, by inspecting the
2902 // memory operands.
2903 if (MayStore && MayLoad) {
2904 for (const MachineMemOperand *MMO : MI->memoperands()) {
2905 const auto *Value = dyn_cast_if_present<FixedStackPseudoSourceValue>(
2906 Val: MMO->getPseudoValue());
2907 if (!Value || Value->getFrameIndex() != FI)
2908 continue;
2909
2910 if (MMO->isStore())
2911 MayLoad = false;
2912 else
2913 MayStore = false;
2914 break;
2915 }
2916 if (MayLoad == MayStore)
2917 report(msg: "Missing fixed stack memoperand.", MI);
2918 }
2919 if (MayLoad && !LI.liveAt(index: Idx.getRegSlot(EC: true))) {
2920 report(msg: "Instruction loads from dead spill slot", MO, MONum);
2921 OS << "Live stack: " << LI << '\n';
2922 }
2923 if (MayStore && !LI.liveAt(index: Idx.getRegSlot())) {
2924 report(msg: "Instruction stores to dead spill slot", MO, MONum);
2925 OS << "Live stack: " << LI << '\n';
2926 }
2927 }
2928 break;
2929
2930 case MachineOperand::MO_CFIIndex:
2931 if (MO->getCFIIndex() >= MF->getFrameInstructions().size())
2932 report(msg: "CFI instruction has invalid index", MO, MONum);
2933 break;
2934
2935 default:
2936 break;
2937 }
2938}
2939
2940void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
2941 unsigned MONum, SlotIndex UseIdx,
2942 const LiveRange &LR,
2943 VirtRegOrUnit VRegOrUnit,
2944 LaneBitmask LaneMask) {
2945 const MachineInstr *MI = MO->getParent();
2946
2947 if (!LR.verify()) {
2948 report(msg: "invalid live range", MO, MONum);
2949 report_context_liverange(LR);
2950 report_context_vreg_regunit(VRegOrUnit);
2951 report_context(Pos: UseIdx);
2952 return;
2953 }
2954
2955 LiveQueryResult LRQ = LR.Query(Idx: UseIdx);
2956 bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut());
2957 // Check if we have a segment at the use, note however that we only need one
2958 // live subregister range, the others may be dead.
2959 if (!HasValue && LaneMask.none()) {
2960 report(msg: "No live segment at use", MO, MONum);
2961 report_context_liverange(LR);
2962 report_context_vreg_regunit(VRegOrUnit);
2963 report_context(Pos: UseIdx);
2964 }
2965 if (MO->isKill() && !LRQ.isKill()) {
2966 report(msg: "Live range continues after kill flag", MO, MONum);
2967 report_context_liverange(LR);
2968 report_context_vreg_regunit(VRegOrUnit);
2969 if (LaneMask.any())
2970 report_context_lanemask(LaneMask);
2971 report_context(Pos: UseIdx);
2972 }
2973}
2974
2975void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
2976 unsigned MONum, SlotIndex DefIdx,
2977 const LiveRange &LR,
2978 VirtRegOrUnit VRegOrUnit,
2979 bool SubRangeCheck,
2980 LaneBitmask LaneMask) {
2981 if (!LR.verify()) {
2982 report(msg: "invalid live range", MO, MONum);
2983 report_context_liverange(LR);
2984 report_context_vreg_regunit(VRegOrUnit);
2985 if (LaneMask.any())
2986 report_context_lanemask(LaneMask);
2987 report_context(Pos: DefIdx);
2988 }
2989
2990 if (const VNInfo *VNI = LR.getVNInfoAt(Idx: DefIdx)) {
2991 // The LR can correspond to the whole reg and its def slot is not obliged
2992 // to be the same as the MO' def slot. E.g. when we check here "normal"
2993 // subreg MO but there is other EC subreg MO in the same instruction so the
2994 // whole reg has EC def slot and differs from the currently checked MO' def
2995 // slot. For example:
2996 // %0 [16e,32r:0) 0@16e L..3 [16e,32r:0) 0@16e L..C [16r,32r:0) 0@16r
2997 // Check that there is an early-clobber def of the same superregister
2998 // somewhere is performed in visitMachineFunctionAfter()
2999 if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) ||
3000 !SlotIndex::isSameInstr(A: VNI->def, B: DefIdx) ||
3001 (VNI->def != DefIdx &&
3002 (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) {
3003 report(msg: "Inconsistent valno->def", MO, MONum);
3004 report_context_liverange(LR);
3005 report_context_vreg_regunit(VRegOrUnit);
3006 if (LaneMask.any())
3007 report_context_lanemask(LaneMask);
3008 report_context(VNI: *VNI);
3009 report_context(Pos: DefIdx);
3010 }
3011 } else {
3012 report(msg: "No live segment at def", MO, MONum);
3013 report_context_liverange(LR);
3014 report_context_vreg_regunit(VRegOrUnit);
3015 if (LaneMask.any())
3016 report_context_lanemask(LaneMask);
3017 report_context(Pos: DefIdx);
3018 }
3019 // Check that, if the dead def flag is present, LiveInts agree.
3020 if (MO->isDead()) {
3021 LiveQueryResult LRQ = LR.Query(Idx: DefIdx);
3022 if (!LRQ.isDeadDef()) {
3023 assert(VRegOrUnit.isVirtualReg() && "Expecting a virtual register.");
3024 // A dead subreg def only tells us that the specific subreg is dead. There
3025 // could be other non-dead defs of other subregs, or we could have other
3026 // parts of the register being live through the instruction. So unless we
3027 // are checking liveness for a subrange it is ok for the live range to
3028 // continue, given that we have a dead def of a subregister.
3029 if (SubRangeCheck || MO->getSubReg() == 0) {
3030 report(msg: "Live range continues after dead def flag", MO, MONum);
3031 report_context_liverange(LR);
3032 report_context_vreg_regunit(VRegOrUnit);
3033 if (LaneMask.any())
3034 report_context_lanemask(LaneMask);
3035 }
3036 }
3037 }
3038}
3039
3040void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
3041 const MachineInstr *MI = MO->getParent();
3042 const Register Reg = MO->getReg();
3043 const unsigned SubRegIdx = MO->getSubReg();
3044
3045 const LiveInterval *LI = nullptr;
3046 if (LiveInts && Reg.isVirtual()) {
3047 if (LiveInts->hasInterval(Reg)) {
3048 LI = &LiveInts->getInterval(Reg);
3049 if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&
3050 !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(VReg: Reg))
3051 report(msg: "Live interval for subreg operand has no subranges", MO, MONum);
3052 } else {
3053 report(msg: "Virtual register has no live interval", MO, MONum);
3054 }
3055 }
3056
3057 // Both use and def operands can read a register.
3058 if (MO->readsReg()) {
3059 if (MO->isKill())
3060 addRegWithSubRegs(RV&: regsKilled, Reg);
3061
3062 // Check LiveInts liveness and kill.
3063 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3064 SlotIndex UseIdx;
3065 if (MI->isPHI()) {
3066 // PHI use occurs on the edge, so check for live out here instead.
3067 UseIdx = LiveInts->getMBBEndIdx(
3068 mbb: MI->getOperand(i: MONum + 1).getMBB()).getPrevSlot();
3069 } else {
3070 UseIdx = LiveInts->getInstructionIndex(Instr: *MI);
3071 }
3072 // Check the cached regunit intervals.
3073 if (Reg.isPhysical() && !isReserved(Reg)) {
3074 for (MCRegUnit Unit : TRI->regunits(Reg: Reg.asMCReg())) {
3075 if (MRI->isReservedRegUnit(Unit))
3076 continue;
3077 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3078 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LR, VRegOrUnit: VirtRegOrUnit(Unit));
3079 }
3080 }
3081
3082 if (Reg.isVirtual()) {
3083 // This is a virtual register interval.
3084 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3085
3086 if (LI->hasSubRanges() && !MO->isDef()) {
3087 LaneBitmask MOMask = SubRegIdx != 0
3088 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3089 : MRI->getMaxLaneMaskForVReg(Reg);
3090 LaneBitmask LiveInMask;
3091 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3092 if ((MOMask & SR.LaneMask).none())
3093 continue;
3094 checkLivenessAtUse(MO, MONum, UseIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg),
3095 LaneMask: SR.LaneMask);
3096 LiveQueryResult LRQ = SR.Query(Idx: UseIdx);
3097 if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))
3098 LiveInMask |= SR.LaneMask;
3099 }
3100 // At least parts of the register has to be live at the use.
3101 if ((LiveInMask & MOMask).none()) {
3102 report(msg: "No live subrange at use", MO, MONum);
3103 report_context(LI: *LI);
3104 report_context(Pos: UseIdx);
3105 }
3106 // For PHIs all lanes should be live
3107 if (MI->isPHI() && LiveInMask != MOMask) {
3108 report(msg: "Not all lanes of PHI source live at use", MO, MONum);
3109 report_context(LI: *LI);
3110 report_context(Pos: UseIdx);
3111 }
3112 }
3113 }
3114 }
3115
3116 // Use of a dead register.
3117 if (!regsLive.count(V: Reg)) {
3118 if (Reg.isPhysical()) {
3119 // Reserved registers may be used even when 'dead'.
3120 bool Bad = !isReserved(Reg);
3121 // We are fine if just any subregister has a defined value.
3122 if (Bad) {
3123
3124 for (const MCPhysReg &SubReg : TRI->subregs(Reg)) {
3125 if (regsLive.count(V: SubReg)) {
3126 Bad = false;
3127 break;
3128 }
3129 }
3130 }
3131 // If there is an additional implicit-use of a super register we stop
3132 // here. By definition we are fine if the super register is not
3133 // (completely) dead, if the complete super register is dead we will
3134 // get a report for its operand.
3135 if (Bad) {
3136 for (const MachineOperand &MOP : MI->uses()) {
3137 if (!MOP.isReg() || !MOP.isImplicit())
3138 continue;
3139
3140 if (!MOP.getReg().isPhysical())
3141 continue;
3142
3143 if (MOP.getReg() != Reg &&
3144 all_of(Range: TRI->regunits(Reg), P: [&](const MCRegUnit RegUnit) {
3145 return llvm::is_contained(Range: TRI->regunits(Reg: MOP.getReg()),
3146 Element: RegUnit);
3147 }))
3148 Bad = false;
3149 }
3150 }
3151 if (Bad)
3152 report(msg: "Using an undefined physical register", MO, MONum);
3153 } else if (MRI->def_empty(RegNo: Reg)) {
3154 report(msg: "Reading virtual register without a def", MO, MONum);
3155 } else {
3156 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3157 // We don't know which virtual registers are live in, so only complain
3158 // if vreg was killed in this MBB. Otherwise keep track of vregs that
3159 // must be live in. PHI instructions are handled separately.
3160 if (MInfo.regsKilled.count(V: Reg))
3161 report(msg: "Using a killed virtual register", MO, MONum);
3162 else if (!MI->isPHI())
3163 MInfo.vregsLiveIn.insert(KV: std::make_pair(x: Reg, y&: MI));
3164 }
3165 }
3166 }
3167
3168 if (MO->isDef()) {
3169 // Register defined.
3170 // TODO: verify that earlyclobber ops are not used.
3171 if (MO->isDead())
3172 addRegWithSubRegs(RV&: regsDead, Reg);
3173 else
3174 addRegWithSubRegs(RV&: regsDefined, Reg);
3175
3176 // Verify SSA form.
3177 if (MRI->isSSA() && Reg.isVirtual()) {
3178 if (!MRI->hasOneDef(RegNo: Reg))
3179 report(msg: "Multiple virtual register defs in SSA form", MO, MONum);
3180 if (MO->getSubReg())
3181 report(msg: "Subreg def in SSA form", MO, MONum);
3182 }
3183
3184 // Check LiveInts for a live segment, but only for virtual registers.
3185 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3186 SlotIndex DefIdx = LiveInts->getInstructionIndex(Instr: *MI);
3187 DefIdx = DefIdx.getRegSlot(EC: MO->isEarlyClobber());
3188
3189 if (Reg.isVirtual()) {
3190 checkLivenessAtDef(MO, MONum, DefIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3191
3192 if (LI->hasSubRanges()) {
3193 LaneBitmask MOMask = SubRegIdx != 0
3194 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3195 : MRI->getMaxLaneMaskForVReg(Reg);
3196 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3197 if ((SR.LaneMask & MOMask).none())
3198 continue;
3199 checkLivenessAtDef(MO, MONum, DefIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg), SubRangeCheck: true,
3200 LaneMask: SR.LaneMask);
3201 }
3202 }
3203 }
3204 }
3205 }
3206}
3207
3208// This function gets called after visiting all instructions in a bundle. The
3209// argument points to the bundle header.
3210// Normal stand-alone instructions are also considered 'bundles', and this
3211// function is called for all of them.
3212void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {
3213 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3214 set_union(S1&: MInfo.regsKilled, S2: regsKilled);
3215 set_subtract(S1&: regsLive, S2: regsKilled); regsKilled.clear();
3216 // Kill any masked registers.
3217 while (!regMasks.empty()) {
3218 const uint32_t *Mask = regMasks.pop_back_val();
3219 for (Register Reg : regsLive)
3220 if (Reg.isPhysical() &&
3221 MachineOperand::clobbersPhysReg(RegMask: Mask, PhysReg: Reg.asMCReg()))
3222 regsDead.push_back(Elt: Reg);
3223 }
3224 set_subtract(S1&: regsLive, S2: regsDead); regsDead.clear();
3225 set_union(S1&: regsLive, S2: regsDefined); regsDefined.clear();
3226}
3227
3228void
3229MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {
3230 MBBInfoMap[MBB].regsLiveOut = regsLive;
3231 regsLive.clear();
3232
3233 if (Indexes) {
3234 SlotIndex stop = Indexes->getMBBEndIdx(mbb: MBB);
3235 if (!(stop > lastIndex)) {
3236 report(msg: "Block ends before last instruction index", MBB);
3237 OS << "Block ends at " << stop << " last instruction was at " << lastIndex
3238 << '\n';
3239 }
3240 lastIndex = stop;
3241 }
3242}
3243
3244namespace {
3245// This implements a set of registers that serves as a filter: can filter other
3246// sets by passing through elements not in the filter and blocking those that
3247// are. Any filter implicitly includes the full set of physical registers upon
3248// creation, thus filtering them all out. The filter itself as a set only grows,
3249// and needs to be as efficient as possible.
3250struct VRegFilter {
3251 // Add elements to the filter itself. \pre Input set \p FromRegSet must have
3252 // no duplicates. Both virtual and physical registers are fine.
3253 template <typename RegSetT> void add(const RegSetT &FromRegSet) {
3254 SmallVector<Register, 0> VRegsBuffer;
3255 filterAndAdd(FromRegSet, VRegsBuffer);
3256 }
3257 // Filter \p FromRegSet through the filter and append passed elements into \p
3258 // ToVRegs. All elements appended are then added to the filter itself.
3259 // \returns true if anything changed.
3260 template <typename RegSetT>
3261 bool filterAndAdd(const RegSetT &FromRegSet,
3262 SmallVectorImpl<Register> &ToVRegs) {
3263 unsigned SparseUniverse = Sparse.size();
3264 unsigned NewSparseUniverse = SparseUniverse;
3265 unsigned NewDenseSize = Dense.size();
3266 size_t Begin = ToVRegs.size();
3267 for (Register Reg : FromRegSet) {
3268 if (!Reg.isVirtual())
3269 continue;
3270 unsigned Index = Reg.virtRegIndex();
3271 if (Index < SparseUniverseMax) {
3272 if (Index < SparseUniverse && Sparse.test(Idx: Index))
3273 continue;
3274 NewSparseUniverse = std::max(a: NewSparseUniverse, b: Index + 1);
3275 } else {
3276 if (Dense.count(V: Reg))
3277 continue;
3278 ++NewDenseSize;
3279 }
3280 ToVRegs.push_back(Elt: Reg);
3281 }
3282 size_t End = ToVRegs.size();
3283 if (Begin == End)
3284 return false;
3285 // Reserving space in sets once performs better than doing so continuously
3286 // and pays easily for double look-ups (even in Dense with SparseUniverseMax
3287 // tuned all the way down) and double iteration (the second one is over a
3288 // SmallVector, which is a lot cheaper compared to DenseSet or BitVector).
3289 Sparse.resize(N: NewSparseUniverse);
3290 Dense.reserve(Size: NewDenseSize);
3291 for (unsigned I = Begin; I < End; ++I) {
3292 Register Reg = ToVRegs[I];
3293 unsigned Index = Reg.virtRegIndex();
3294 if (Index < SparseUniverseMax)
3295 Sparse.set(Index);
3296 else
3297 Dense.insert(V: Reg);
3298 }
3299 return true;
3300 }
3301
3302private:
3303 static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8;
3304 // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyond
3305 // are tracked by Dense. The only purpose of the threshold and the Dense set
3306 // is to have a reasonably growing memory usage in pathological cases (large
3307 // number of very sparse VRegFilter instances live at the same time). In
3308 // practice even in the worst-by-execution time cases having all elements
3309 // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more
3310 // space efficient than if tracked by Dense. The threshold is set to keep the
3311 // worst-case memory usage within 2x of figures determined empirically for
3312 // "all Dense" scenario in such worst-by-execution-time cases.
3313 BitVector Sparse;
3314 DenseSet<Register> Dense;
3315};
3316
3317// Implements both a transfer function and a (binary, in-place) join operator
3318// for a dataflow over register sets with set union join and filtering transfer
3319// (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.
3320// Maintains out_b as its state, allowing for O(n) iteration over it at any
3321// time, where n is the size of the set (as opposed to O(U) where U is the
3322// universe). filter_b implicitly contains all physical registers at all times.
3323class FilteringVRegSet {
3324 VRegFilter Filter;
3325 SmallVector<Register, 0> VRegs;
3326
3327public:
3328 // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.
3329 // Both virtual and physical registers are fine.
3330 template <typename RegSetT> void addToFilter(const RegSetT &RS) {
3331 Filter.add(RS);
3332 }
3333 // Passes \p RS through the filter_b (transfer function) and adds what's left
3334 // to itself (out_b).
3335 template <typename RegSetT> bool add(const RegSetT &RS) {
3336 // Double-duty the Filter: to maintain VRegs a set (and the join operation
3337 // a set union) just add everything being added here to the Filter as well.
3338 return Filter.filterAndAdd(RS, VRegs);
3339 }
3340 using const_iterator = decltype(VRegs)::const_iterator;
3341 const_iterator begin() const { return VRegs.begin(); }
3342 const_iterator end() const { return VRegs.end(); }
3343 size_t size() const { return VRegs.size(); }
3344};
3345} // namespace
3346
3347// Calculate the largest possible vregsPassed sets. These are the registers that
3348// can pass through an MBB live, but may not be live every time. It is assumed
3349// that all vregsPassed sets are empty before the call.
3350void MachineVerifier::calcRegsPassed() {
3351 if (MF->empty())
3352 // ReversePostOrderTraversal doesn't handle empty functions.
3353 return;
3354
3355 for (const MachineBasicBlock *MB :
3356 ReversePostOrderTraversal<const MachineFunction *>(MF)) {
3357 FilteringVRegSet VRegs;
3358 BBInfo &Info = MBBInfoMap[MB];
3359 assert(Info.reachable);
3360
3361 VRegs.addToFilter(RS: Info.regsKilled);
3362 VRegs.addToFilter(RS: Info.regsLiveOut);
3363 for (const MachineBasicBlock *Pred : MB->predecessors()) {
3364 const BBInfo &PredInfo = MBBInfoMap[Pred];
3365 if (!PredInfo.reachable)
3366 continue;
3367
3368 VRegs.add(RS: PredInfo.regsLiveOut);
3369 VRegs.add(RS: PredInfo.vregsPassed);
3370 }
3371 Info.vregsPassed.reserve(Size: VRegs.size());
3372 Info.vregsPassed.insert_range(R&: VRegs);
3373 }
3374}
3375
3376// Calculate the set of virtual registers that must be passed through each basic
3377// block in order to satisfy the requirements of successor blocks. This is very
3378// similar to calcRegsPassed, only backwards.
3379void MachineVerifier::calcRegsRequired() {
3380 // First push live-in regs to predecessors' vregsRequired.
3381 SmallPtrSet<const MachineBasicBlock*, 8> todo;
3382 for (const auto &MBB : *MF) {
3383 BBInfo &MInfo = MBBInfoMap[&MBB];
3384 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3385 BBInfo &PInfo = MBBInfoMap[Pred];
3386 if (PInfo.addRequired(RM: MInfo.vregsLiveIn))
3387 todo.insert(Ptr: Pred);
3388 }
3389
3390 // Handle the PHI node.
3391 for (const MachineInstr &MI : MBB.phis()) {
3392 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3393 // Skip those Operands which are undef regs or not regs.
3394 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())
3395 continue;
3396
3397 // Get register and predecessor for one PHI edge.
3398 Register Reg = MI.getOperand(i).getReg();
3399 const MachineBasicBlock *Pred = MI.getOperand(i: i + 1).getMBB();
3400
3401 BBInfo &PInfo = MBBInfoMap[Pred];
3402 if (PInfo.addRequired(Reg))
3403 todo.insert(Ptr: Pred);
3404 }
3405 }
3406 }
3407
3408 // Iteratively push vregsRequired to predecessors. This will converge to the
3409 // same final state regardless of DenseSet iteration order.
3410 while (!todo.empty()) {
3411 const MachineBasicBlock *MBB = *todo.begin();
3412 todo.erase(Ptr: MBB);
3413 BBInfo &MInfo = MBBInfoMap[MBB];
3414 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
3415 if (Pred == MBB)
3416 continue;
3417 BBInfo &SInfo = MBBInfoMap[Pred];
3418 if (SInfo.addRequired(RS: MInfo.vregsRequired))
3419 todo.insert(Ptr: Pred);
3420 }
3421 }
3422}
3423
3424// Check PHI instructions at the beginning of MBB. It is assumed that
3425// calcRegsPassed has been run so BBInfo::isLiveOut is valid.
3426void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {
3427 BBInfo &MInfo = MBBInfoMap[&MBB];
3428
3429 SmallPtrSet<const MachineBasicBlock*, 8> seen;
3430 for (const MachineInstr &Phi : MBB) {
3431 if (!Phi.isPHI())
3432 break;
3433 seen.clear();
3434
3435 const MachineOperand &MODef = Phi.getOperand(i: 0);
3436 if (!MODef.isReg() || !MODef.isDef()) {
3437 report(msg: "Expected first PHI operand to be a register def", MO: &MODef, MONum: 0);
3438 continue;
3439 }
3440 if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||
3441 MODef.isEarlyClobber() || MODef.isDebug())
3442 report(msg: "Unexpected flag on PHI operand", MO: &MODef, MONum: 0);
3443 Register DefReg = MODef.getReg();
3444 if (!DefReg.isVirtual())
3445 report(msg: "Expected first PHI operand to be a virtual register", MO: &MODef, MONum: 0);
3446
3447 for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {
3448 const MachineOperand &MO0 = Phi.getOperand(i: I);
3449 if (!MO0.isReg()) {
3450 report(msg: "Expected PHI operand to be a register", MO: &MO0, MONum: I);
3451 continue;
3452 }
3453 if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||
3454 MO0.isDebug() || MO0.isTied())
3455 report(msg: "Unexpected flag on PHI operand", MO: &MO0, MONum: I);
3456
3457 const MachineOperand &MO1 = Phi.getOperand(i: I + 1);
3458 if (!MO1.isMBB()) {
3459 report(msg: "Expected PHI operand to be a basic block", MO: &MO1, MONum: I + 1);
3460 continue;
3461 }
3462
3463 const MachineBasicBlock &Pre = *MO1.getMBB();
3464 if (!Pre.isSuccessor(MBB: &MBB)) {
3465 report(msg: "PHI input is not a predecessor block", MO: &MO1, MONum: I + 1);
3466 continue;
3467 }
3468
3469 if (MInfo.reachable) {
3470 seen.insert(Ptr: &Pre);
3471 BBInfo &PrInfo = MBBInfoMap[&Pre];
3472 if (!MO0.isUndef() && PrInfo.reachable &&
3473 !PrInfo.isLiveOut(Reg: MO0.getReg()))
3474 report(msg: "PHI operand is not live-out from predecessor", MO: &MO0, MONum: I);
3475 }
3476 }
3477
3478 // Did we see all predecessors?
3479 if (MInfo.reachable) {
3480 for (MachineBasicBlock *Pred : MBB.predecessors()) {
3481 if (!seen.count(Ptr: Pred)) {
3482 report(msg: "Missing PHI operand", MI: &Phi);
3483 OS << printMBBReference(MBB: *Pred)
3484 << " is a predecessor according to the CFG.\n";
3485 }
3486 }
3487 }
3488 }
3489}
3490
3491static void
3492verifyConvergenceControl(const MachineFunction &MF, MachineDominatorTree &DT,
3493 std::function<void(const Twine &Message)> FailureCB,
3494 raw_ostream &OS) {
3495 MachineConvergenceVerifier CV;
3496 CV.initialize(OS: &OS, FailureCB, F: MF);
3497
3498 for (const auto &MBB : MF) {
3499 CV.visit(BB: MBB);
3500 for (const auto &MI : MBB.instrs())
3501 CV.visit(I: MI);
3502 }
3503
3504 if (CV.sawTokens()) {
3505 DT.recalculate(Func&: const_cast<MachineFunction &>(MF));
3506 CV.verify(DT);
3507 }
3508}
3509
3510void MachineVerifier::visitMachineFunctionAfter() {
3511 auto FailureCB = [this](const Twine &Message) {
3512 report(msg: Message.str().c_str(), MF);
3513 };
3514 verifyConvergenceControl(MF: *MF, DT, FailureCB, OS);
3515
3516 calcRegsPassed();
3517
3518 for (const MachineBasicBlock &MBB : *MF)
3519 checkPHIOps(MBB);
3520
3521 // Now check liveness info if available
3522 calcRegsRequired();
3523
3524 // Check for killed virtual registers that should be live out.
3525 for (const auto &MBB : *MF) {
3526 BBInfo &MInfo = MBBInfoMap[&MBB];
3527 for (Register VReg : MInfo.vregsRequired)
3528 if (MInfo.regsKilled.count(V: VReg)) {
3529 report(msg: "Virtual register killed in block, but needed live out.", MBB: &MBB);
3530 OS << "Virtual register " << printReg(Reg: VReg)
3531 << " is used after the block.\n";
3532 }
3533 }
3534
3535 if (!MF->empty()) {
3536 BBInfo &MInfo = MBBInfoMap[&MF->front()];
3537 for (Register VReg : MInfo.vregsRequired) {
3538 report(msg: "Virtual register defs don't dominate all uses.", MF);
3539 report_context_vreg(VReg);
3540 }
3541 }
3542
3543 if (LiveInts)
3544 verifyLiveIntervals();
3545
3546 // Check live-in list of each MBB. If a register is live into MBB, check
3547 // that the register is in regsLiveOut of each predecessor block. Since
3548 // this must come from a definition in the predecessor or its live-in
3549 // list, this will catch a live-through case where the predecessor does not
3550 // have the register in its live-in list. This currently only checks
3551 // registers that have no aliases, are not allocatable and are not
3552 // reserved, which could mean a condition code register for instance.
3553 if (MRI->tracksLiveness())
3554 for (const auto &MBB : *MF)
3555 for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) {
3556 MCRegister LiveInReg = P.PhysReg;
3557 bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid();
3558 if (hasAliases || isAllocatable(Reg: LiveInReg) || isReserved(Reg: LiveInReg))
3559 continue;
3560 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3561 BBInfo &PInfo = MBBInfoMap[Pred];
3562 if (!PInfo.regsLiveOut.count(V: LiveInReg)) {
3563 report(msg: "Live in register not found to be live out from predecessor.",
3564 MBB: &MBB);
3565 OS << TRI->getName(RegNo: LiveInReg) << " not found to be live out from "
3566 << printMBBReference(MBB: *Pred) << '\n';
3567 }
3568 }
3569 }
3570
3571 for (auto CSInfo : MF->getCallSitesInfo())
3572 if (!CSInfo.first->isCall())
3573 report(msg: "Call site info referencing instruction that is not call", MF);
3574
3575 // If there's debug-info, check that we don't have any duplicate value
3576 // tracking numbers.
3577 if (MF->getFunction().getSubprogram()) {
3578 DenseSet<unsigned> SeenNumbers;
3579 for (const auto &MBB : *MF) {
3580 for (const auto &MI : MBB) {
3581 if (auto Num = MI.peekDebugInstrNum()) {
3582 auto Result = SeenNumbers.insert(V: (unsigned)Num);
3583 if (!Result.second)
3584 report(msg: "Instruction has a duplicated value tracking number", MI: &MI);
3585 }
3586 }
3587 }
3588 }
3589}
3590
3591void MachineVerifier::verifyLiveIntervals() {
3592 assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
3593 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3594 Register Reg = Register::index2VirtReg(Index: I);
3595
3596 // Spilling and splitting may leave unused registers around. Skip them.
3597 if (MRI->reg_nodbg_empty(RegNo: Reg))
3598 continue;
3599
3600 if (!LiveInts->hasInterval(Reg)) {
3601 report(msg: "Missing live interval for virtual register", MF);
3602 OS << printReg(Reg, TRI) << " still has defs or uses\n";
3603 continue;
3604 }
3605
3606 const LiveInterval &LI = LiveInts->getInterval(Reg);
3607 assert(Reg == LI.reg() && "Invalid reg to interval mapping");
3608 verifyLiveInterval(LI);
3609 }
3610
3611 // Verify all the cached regunit intervals.
3612 for (MCRegUnit Unit : TRI->regunits())
3613 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3614 verifyLiveRange(*LR, VirtRegOrUnit(Unit));
3615}
3616
3617void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
3618 const VNInfo *VNI,
3619 VirtRegOrUnit VRegOrUnit,
3620 LaneBitmask LaneMask) {
3621 if (VNI->isUnused())
3622 return;
3623
3624 const VNInfo *DefVNI = LR.getVNInfoAt(Idx: VNI->def);
3625
3626 if (!DefVNI) {
3627 report(msg: "Value not live at VNInfo def and not marked unused", MF);
3628 report_context(LR, VRegOrUnit, LaneMask);
3629 report_context(VNI: *VNI);
3630 return;
3631 }
3632
3633 if (DefVNI != VNI) {
3634 report(msg: "Live segment at def has different VNInfo", MF);
3635 report_context(LR, VRegOrUnit, LaneMask);
3636 report_context(VNI: *VNI);
3637 return;
3638 }
3639
3640 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: VNI->def);
3641 if (!MBB) {
3642 report(msg: "Invalid VNInfo definition index", MF);
3643 report_context(LR, VRegOrUnit, LaneMask);
3644 report_context(VNI: *VNI);
3645 return;
3646 }
3647
3648 if (VNI->isPHIDef()) {
3649 if (VNI->def != LiveInts->getMBBStartIdx(mbb: MBB)) {
3650 report(msg: "PHIDef VNInfo is not defined at MBB start", MBB);
3651 report_context(LR, VRegOrUnit, LaneMask);
3652 report_context(VNI: *VNI);
3653 }
3654 return;
3655 }
3656
3657 // Non-PHI def.
3658 const MachineInstr *MI = LiveInts->getInstructionFromIndex(index: VNI->def);
3659 if (!MI) {
3660 report(msg: "No instruction at VNInfo def index", MBB);
3661 report_context(LR, VRegOrUnit, LaneMask);
3662 report_context(VNI: *VNI);
3663 return;
3664 }
3665
3666 bool hasDef = false;
3667 bool isEarlyClobber = false;
3668 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3669 if (!MOI->isReg() || !MOI->isDef())
3670 continue;
3671 if (VRegOrUnit.isVirtualReg()) {
3672 if (MOI->getReg() != VRegOrUnit.asVirtualReg())
3673 continue;
3674 } else {
3675 if (!MOI->getReg().isPhysical() ||
3676 !TRI->hasRegUnit(Reg: MOI->getReg(), RegUnit: VRegOrUnit.asMCRegUnit()))
3677 continue;
3678 }
3679 if (LaneMask.any() &&
3680 (TRI->getSubRegIndexLaneMask(SubIdx: MOI->getSubReg()) & LaneMask).none())
3681 continue;
3682 hasDef = true;
3683 if (MOI->isEarlyClobber())
3684 isEarlyClobber = true;
3685 }
3686
3687 if (!hasDef) {
3688 report(msg: "Defining instruction does not modify register", MI);
3689 report_context(LR, VRegOrUnit, LaneMask);
3690 report_context(VNI: *VNI);
3691 }
3692
3693 // Early clobber defs begin at USE slots, but other defs must begin at
3694 // DEF slots.
3695 if (isEarlyClobber) {
3696 if (!VNI->def.isEarlyClobber()) {
3697 report(msg: "Early clobber def must be at an early-clobber slot", MBB);
3698 report_context(LR, VRegOrUnit, LaneMask);
3699 report_context(VNI: *VNI);
3700 }
3701 } else if (!VNI->def.isRegister()) {
3702 report(msg: "Non-PHI, non-early clobber def must be at a register slot", MBB);
3703 report_context(LR, VRegOrUnit, LaneMask);
3704 report_context(VNI: *VNI);
3705 }
3706}
3707
3708void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
3709 const LiveRange::const_iterator I,
3710 VirtRegOrUnit VRegOrUnit,
3711 LaneBitmask LaneMask) {
3712 const LiveRange::Segment &S = *I;
3713 const VNInfo *VNI = S.valno;
3714 assert(VNI && "Live segment has no valno");
3715
3716 if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(ValNo: VNI->id)) {
3717 report(msg: "Foreign valno in live segment", MF);
3718 report_context(LR, VRegOrUnit, LaneMask);
3719 report_context(S);
3720 report_context(VNI: *VNI);
3721 }
3722
3723 if (VNI->isUnused()) {
3724 report(msg: "Live segment valno is marked unused", MF);
3725 report_context(LR, VRegOrUnit, LaneMask);
3726 report_context(S);
3727 }
3728
3729 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: S.start);
3730 if (!MBB) {
3731 report(msg: "Bad start of live segment, no basic block", MF);
3732 report_context(LR, VRegOrUnit, LaneMask);
3733 report_context(S);
3734 return;
3735 }
3736 SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(mbb: MBB);
3737 if (S.start != MBBStartIdx && S.start != VNI->def) {
3738 report(msg: "Live segment must begin at MBB entry or valno def", MBB);
3739 report_context(LR, VRegOrUnit, LaneMask);
3740 report_context(S);
3741 }
3742
3743 const MachineBasicBlock *EndMBB =
3744 LiveInts->getMBBFromIndex(index: S.end.getPrevSlot());
3745 if (!EndMBB) {
3746 report(msg: "Bad end of live segment, no basic block", MF);
3747 report_context(LR, VRegOrUnit, LaneMask);
3748 report_context(S);
3749 return;
3750 }
3751
3752 // Checks for non-live-out segments.
3753 if (S.end != LiveInts->getMBBEndIdx(mbb: EndMBB)) {
3754 // RegUnit intervals are allowed dead phis.
3755 if (!VRegOrUnit.isVirtualReg() && VNI->isPHIDef() && S.start == VNI->def &&
3756 S.end == VNI->def.getDeadSlot())
3757 return;
3758
3759 // The live segment is ending inside EndMBB
3760 const MachineInstr *MI =
3761 LiveInts->getInstructionFromIndex(index: S.end.getPrevSlot());
3762 if (!MI) {
3763 report(msg: "Live segment doesn't end at a valid instruction", MBB: EndMBB);
3764 report_context(LR, VRegOrUnit, LaneMask);
3765 report_context(S);
3766 return;
3767 }
3768
3769 // The block slot must refer to a basic block boundary.
3770 if (S.end.isBlock()) {
3771 report(msg: "Live segment ends at B slot of an instruction", MBB: EndMBB);
3772 report_context(LR, VRegOrUnit, LaneMask);
3773 report_context(S);
3774 }
3775
3776 if (S.end.isDead()) {
3777 // Segment ends on the dead slot.
3778 // That means there must be a dead def.
3779 if (!SlotIndex::isSameInstr(A: S.start, B: S.end)) {
3780 report(msg: "Live segment ending at dead slot spans instructions", MBB: EndMBB);
3781 report_context(LR, VRegOrUnit, LaneMask);
3782 report_context(S);
3783 }
3784 }
3785
3786 // After tied operands are rewritten, a live segment can only end at an
3787 // early-clobber slot if it is being redefined by an early-clobber def.
3788 // TODO: Before tied operands are rewritten, a live segment can only end at
3789 // an early-clobber slot if the last use is tied to an early-clobber def.
3790 if (MF->getProperties().hasTiedOpsRewritten() && S.end.isEarlyClobber()) {
3791 if (I + 1 == LR.end() || (I + 1)->start != S.end) {
3792 report(msg: "Live segment ending at early clobber slot must be "
3793 "redefined by an EC def in the same instruction",
3794 MBB: EndMBB);
3795 report_context(LR, VRegOrUnit, LaneMask);
3796 report_context(S);
3797 }
3798 }
3799
3800 // The following checks only apply to virtual registers. Physreg liveness
3801 // is too weird to check.
3802 if (VRegOrUnit.isVirtualReg()) {
3803 // A live segment can end with either a redefinition, a kill flag on a
3804 // use, or a dead flag on a def.
3805 bool hasRead = false;
3806 bool hasSubRegDef = false;
3807 bool hasDeadDef = false;
3808 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3809 if (!MOI->isReg() || MOI->getReg() != VRegOrUnit.asVirtualReg())
3810 continue;
3811 unsigned Sub = MOI->getSubReg();
3812 LaneBitmask SLM =
3813 Sub != 0 ? TRI->getSubRegIndexLaneMask(SubIdx: Sub) : LaneBitmask::getAll();
3814 if (MOI->isDef()) {
3815 if (Sub != 0) {
3816 hasSubRegDef = true;
3817 // An operand %0:sub0 reads %0:sub1..n. Invert the lane
3818 // mask for subregister defs. Read-undef defs will be handled by
3819 // readsReg below.
3820 SLM = ~SLM;
3821 }
3822 if (MOI->isDead())
3823 hasDeadDef = true;
3824 }
3825 if (LaneMask.any() && (LaneMask & SLM).none())
3826 continue;
3827 if (MOI->readsReg())
3828 hasRead = true;
3829 }
3830 if (S.end.isDead()) {
3831 // Make sure that the corresponding machine operand for a "dead" live
3832 // range has the dead flag. We cannot perform this check for subregister
3833 // liveranges as partially dead values are allowed.
3834 if (LaneMask.none() && !hasDeadDef) {
3835 report(
3836 msg: "Instruction ending live segment on dead slot has no dead flag",
3837 MI);
3838 report_context(LR, VRegOrUnit, LaneMask);
3839 report_context(S);
3840 }
3841 } else {
3842 if (!hasRead) {
3843 // When tracking subregister liveness, the main range must start new
3844 // values on partial register writes, even if there is no read.
3845 if (!MRI->shouldTrackSubRegLiveness(VReg: VRegOrUnit.asVirtualReg()) ||
3846 LaneMask.any() || !hasSubRegDef) {
3847 report(msg: "Instruction ending live segment doesn't read the register",
3848 MI);
3849 report_context(LR, VRegOrUnit, LaneMask);
3850 report_context(S);
3851 }
3852 }
3853 }
3854 }
3855 }
3856
3857 // Now check all the basic blocks in this live segment.
3858 MachineFunction::const_iterator MFI = MBB->getIterator();
3859 // Is this live segment the beginning of a non-PHIDef VN?
3860 if (S.start == VNI->def && !VNI->isPHIDef()) {
3861 // Not live-in to any blocks.
3862 if (MBB == EndMBB)
3863 return;
3864 // Skip this block.
3865 ++MFI;
3866 }
3867
3868 SmallVector<SlotIndex, 4> Undefs;
3869 if (LaneMask.any()) {
3870 LiveInterval &OwnerLI = LiveInts->getInterval(Reg: VRegOrUnit.asVirtualReg());
3871 OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, MRI: *MRI, Indexes: *Indexes);
3872 }
3873
3874 while (true) {
3875 assert(LiveInts->isLiveInToMBB(LR, &*MFI));
3876 // We don't know how to track physregs into a landing pad.
3877 if (!VRegOrUnit.isVirtualReg() && MFI->isEHPad()) {
3878 if (&*MFI == EndMBB)
3879 break;
3880 ++MFI;
3881 continue;
3882 }
3883
3884 // Is VNI a PHI-def in the current block?
3885 bool IsPHI = VNI->isPHIDef() &&
3886 VNI->def == LiveInts->getMBBStartIdx(mbb: &*MFI);
3887
3888 // Check that VNI is live-out of all predecessors.
3889 for (const MachineBasicBlock *Pred : MFI->predecessors()) {
3890 SlotIndex PEnd = LiveInts->getMBBEndIdx(mbb: Pred);
3891 // Predecessor of landing pad live-out on last call.
3892 if (MFI->isEHPad()) {
3893 for (const MachineInstr &MI : llvm::reverse(C: *Pred)) {
3894 if (MI.isCall()) {
3895 PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex();
3896 break;
3897 }
3898 }
3899 }
3900 const VNInfo *PVNI = LR.getVNInfoBefore(Idx: PEnd);
3901
3902 // All predecessors must have a live-out value. However for a phi
3903 // instruction with subregister intervals
3904 // only one of the subregisters (not necessarily the current one) needs to
3905 // be defined.
3906 if (!PVNI && (LaneMask.none() || !IsPHI)) {
3907 if (LiveRangeCalc::isJointlyDominated(MBB: Pred, Defs: Undefs, Indexes: *Indexes))
3908 continue;
3909 report(msg: "Register not marked live out of predecessor", MBB: Pred);
3910 report_context(LR, VRegOrUnit, LaneMask);
3911 report_context(VNI: *VNI);
3912 OS << " live into " << printMBBReference(MBB: *MFI) << '@'
3913 << LiveInts->getMBBStartIdx(mbb: &*MFI) << ", not live before " << PEnd
3914 << '\n';
3915 continue;
3916 }
3917
3918 // Only PHI-defs can take different predecessor values.
3919 if (!IsPHI && PVNI != VNI) {
3920 report(msg: "Different value live out of predecessor", MBB: Pred);
3921 report_context(LR, VRegOrUnit, LaneMask);
3922 OS << "Valno #" << PVNI->id << " live out of "
3923 << printMBBReference(MBB: *Pred) << '@' << PEnd << "\nValno #" << VNI->id
3924 << " live into " << printMBBReference(MBB: *MFI) << '@'
3925 << LiveInts->getMBBStartIdx(mbb: &*MFI) << '\n';
3926 }
3927 }
3928 if (&*MFI == EndMBB)
3929 break;
3930 ++MFI;
3931 }
3932}
3933
3934void MachineVerifier::verifyLiveRange(const LiveRange &LR,
3935 VirtRegOrUnit VRegOrUnit,
3936 LaneBitmask LaneMask) {
3937 for (const VNInfo *VNI : LR.valnos)
3938 verifyLiveRangeValue(LR, VNI, VRegOrUnit, LaneMask);
3939
3940 for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
3941 verifyLiveRangeSegment(LR, I, VRegOrUnit, LaneMask);
3942}
3943
3944void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
3945 Register Reg = LI.reg();
3946 assert(Reg.isVirtual());
3947 verifyLiveRange(LR: LI, VRegOrUnit: VirtRegOrUnit(Reg));
3948
3949 if (LI.hasSubRanges()) {
3950 LaneBitmask Mask;
3951 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
3952 for (const LiveInterval::SubRange &SR : LI.subranges()) {
3953 if ((Mask & SR.LaneMask).any()) {
3954 report(msg: "Lane masks of sub ranges overlap in live interval", MF);
3955 report_context(LI);
3956 }
3957 if ((SR.LaneMask & ~MaxMask).any()) {
3958 report(msg: "Subrange lanemask is invalid", MF);
3959 report_context(LI);
3960 }
3961 if (SR.empty()) {
3962 report(msg: "Subrange must not be empty", MF);
3963 report_context(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
3964 }
3965 Mask |= SR.LaneMask;
3966 verifyLiveRange(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
3967 if (!LI.covers(Other: SR)) {
3968 report(msg: "A Subrange is not covered by the main range", MF);
3969 report_context(LI);
3970 }
3971 }
3972 }
3973
3974 // Check the LI only has one connected component.
3975 ConnectedVNInfoEqClasses ConEQ(*LiveInts);
3976 unsigned NumComp = ConEQ.Classify(LR: LI);
3977 if (NumComp > 1) {
3978 report(msg: "Multiple connected components in live interval", MF);
3979 report_context(LI);
3980 for (unsigned comp = 0; comp != NumComp; ++comp) {
3981 OS << comp << ": valnos";
3982 for (const VNInfo *I : LI.valnos)
3983 if (comp == ConEQ.getEqClass(VNI: I))
3984 OS << ' ' << I->id;
3985 OS << '\n';
3986 }
3987 }
3988}
3989
3990namespace {
3991
3992 // FrameSetup and FrameDestroy can have zero adjustment, so using a single
3993 // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the
3994 // value is zero.
3995 // We use a bool plus an integer to capture the stack state.
3996struct StackStateOfBB {
3997 StackStateOfBB() = default;
3998 StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup)
3999 : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),
4000 ExitIsSetup(ExitSetup) {}
4001
4002 // Can be negative, which means we are setting up a frame.
4003 int EntryValue = 0;
4004 int ExitValue = 0;
4005 bool EntryIsSetup = false;
4006 bool ExitIsSetup = false;
4007};
4008
4009} // end anonymous namespace
4010
4011/// Make sure on every path through the CFG, a FrameSetup <n> is always followed
4012/// by a FrameDestroy <n>, stack adjustments are identical on all
4013/// CFG edges to a merge point, and frame is destroyed at end of a return block.
4014void MachineVerifier::verifyStackFrame() {
4015 unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
4016 unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
4017 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
4018 return;
4019
4020 SmallVector<StackStateOfBB, 8> SPState;
4021 SPState.resize(N: MF->getNumBlockIDs());
4022 df_iterator_default_set<const MachineBasicBlock*> Reachable;
4023
4024 // Visit the MBBs in DFS order.
4025 for (df_ext_iterator<const MachineFunction *,
4026 df_iterator_default_set<const MachineBasicBlock *>>
4027 DFI = df_ext_begin(G: MF, S&: Reachable), DFE = df_ext_end(G: MF, S&: Reachable);
4028 DFI != DFE; ++DFI) {
4029 const MachineBasicBlock *MBB = *DFI;
4030
4031 StackStateOfBB BBState;
4032 // Check the exit state of the DFS stack predecessor.
4033 if (DFI.getPathLength() >= 2) {
4034 const MachineBasicBlock *StackPred = DFI.getPath(n: DFI.getPathLength() - 2);
4035 assert(Reachable.count(StackPred) &&
4036 "DFS stack predecessor is already visited.\n");
4037 BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;
4038 BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;
4039 BBState.ExitValue = BBState.EntryValue;
4040 BBState.ExitIsSetup = BBState.EntryIsSetup;
4041 }
4042
4043 if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) {
4044 report(msg: "Call frame size on entry does not match value computed from "
4045 "predecessor",
4046 MBB);
4047 OS << "Call frame size on entry " << MBB->getCallFrameSize()
4048 << " does not match value computed from predecessor "
4049 << -BBState.EntryValue << '\n';
4050 }
4051
4052 // Update stack state by checking contents of MBB.
4053 for (const auto &I : *MBB) {
4054 if (I.getOpcode() == FrameSetupOpcode) {
4055 if (BBState.ExitIsSetup)
4056 report(msg: "FrameSetup is after another FrameSetup", MI: &I);
4057 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4058 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4059 "instruction.", MI: &I);
4060 BBState.ExitValue -= TII->getFrameTotalSize(I);
4061 BBState.ExitIsSetup = true;
4062 }
4063
4064 if (I.getOpcode() == FrameDestroyOpcode) {
4065 int Size = TII->getFrameTotalSize(I);
4066 if (!BBState.ExitIsSetup)
4067 report(msg: "FrameDestroy is not after a FrameSetup", MI: &I);
4068 int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :
4069 BBState.ExitValue;
4070 if (BBState.ExitIsSetup && AbsSPAdj != Size) {
4071 report(msg: "FrameDestroy <n> is after FrameSetup <m>", MI: &I);
4072 OS << "FrameDestroy <" << Size << "> is after FrameSetup <"
4073 << AbsSPAdj << ">.\n";
4074 }
4075 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4076 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4077 "instruction.", MI: &I);
4078 BBState.ExitValue += Size;
4079 BBState.ExitIsSetup = false;
4080 }
4081 }
4082 SPState[MBB->getNumber()] = BBState;
4083
4084 // Make sure the exit state of any predecessor is consistent with the entry
4085 // state.
4086 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
4087 if (Reachable.count(Ptr: Pred) &&
4088 (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue ||
4089 SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {
4090 report(msg: "The exit stack state of a predecessor is inconsistent.", MBB);
4091 OS << "Predecessor " << printMBBReference(MBB: *Pred) << " has exit state ("
4092 << SPState[Pred->getNumber()].ExitValue << ", "
4093 << SPState[Pred->getNumber()].ExitIsSetup << "), while "
4094 << printMBBReference(MBB: *MBB) << " has entry state ("
4095 << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";
4096 }
4097 }
4098
4099 // Make sure the entry state of any successor is consistent with the exit
4100 // state.
4101 for (const MachineBasicBlock *Succ : MBB->successors()) {
4102 if (Reachable.count(Ptr: Succ) &&
4103 (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue ||
4104 SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {
4105 report(msg: "The entry stack state of a successor is inconsistent.", MBB);
4106 OS << "Successor " << printMBBReference(MBB: *Succ) << " has entry state ("
4107 << SPState[Succ->getNumber()].EntryValue << ", "
4108 << SPState[Succ->getNumber()].EntryIsSetup << "), while "
4109 << printMBBReference(MBB: *MBB) << " has exit state ("
4110 << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";
4111 }
4112 }
4113
4114 // Make sure a basic block with return ends with zero stack adjustment.
4115 if (!MBB->empty() && MBB->back().isReturn()) {
4116 if (BBState.ExitIsSetup)
4117 report(msg: "A return block ends with a FrameSetup.", MBB);
4118 if (BBState.ExitValue)
4119 report(msg: "A return block ends with a nonzero stack adjustment.", MBB);
4120 }
4121 }
4122}
4123
4124void MachineVerifier::verifyStackProtector() {
4125 const MachineFrameInfo &MFI = MF->getFrameInfo();
4126 if (!MFI.hasStackProtectorIndex())
4127 return;
4128 // Only applicable when the offsets of frame objects have been determined,
4129 // which is indicated by a non-zero stack size.
4130 if (!MFI.getStackSize())
4131 return;
4132 const TargetFrameLowering &TFI = *MF->getSubtarget().getFrameLowering();
4133 bool StackGrowsDown =
4134 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
4135 unsigned FI = MFI.getStackProtectorIndex();
4136 int64_t SPStart = MFI.getObjectOffset(ObjectIdx: FI);
4137 int64_t SPEnd = SPStart + MFI.getObjectSize(ObjectIdx: FI);
4138 for (unsigned I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
4139 if (I == FI)
4140 continue;
4141 if (MFI.isDeadObjectIndex(ObjectIdx: I))
4142 continue;
4143 // FIXME: Skip non-default stack objects, as some targets may place them
4144 // above the stack protector. This is a workaround for the fact that
4145 // backends such as AArch64 may place SVE stack objects *above* the stack
4146 // protector.
4147 if (MFI.getStackID(ObjectIdx: I) != TargetStackID::Default)
4148 continue;
4149 // Skip variable-sized objects because they do not have a fixed offset.
4150 if (MFI.isVariableSizedObjectIndex(ObjectIdx: I))
4151 continue;
4152 // FIXME: Skip spill slots which may be allocated above the stack protector.
4153 // Ideally this would only skip callee-saved registers, but we don't have
4154 // that information here. For example, spill-slots used for scavenging are
4155 // not described in CalleeSavedInfo.
4156 if (MFI.isSpillSlotObjectIndex(ObjectIdx: I))
4157 continue;
4158 int64_t ObjStart = MFI.getObjectOffset(ObjectIdx: I);
4159 int64_t ObjEnd = ObjStart + MFI.getObjectSize(ObjectIdx: I);
4160 if (SPStart < ObjEnd && ObjStart < SPEnd) {
4161 report(msg: "Stack protector overlaps with another stack object", MF);
4162 break;
4163 }
4164 if ((StackGrowsDown && SPStart <= ObjStart) ||
4165 (!StackGrowsDown && SPStart >= ObjStart)) {
4166 report(msg: "Stack protector is not the top-most object on the stack", MF);
4167 break;
4168 }
4169 }
4170}
4171