1//===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Pass to verify generated machine code. The following is checked:
10//
11// Operand counts: All explicit operands must be present.
12//
13// Register classes: All physical and virtual register operands must be
14// compatible with the register class required by the instruction descriptor.
15//
16// Register live intervals: Registers must be defined only once, and must be
17// defined before use.
18//
19// The machine code verifier is enabled with the command-line option
20// -verify-machineinstrs.
21//===----------------------------------------------------------------------===//
22
23#include "llvm/CodeGen/MachineVerifier.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/ADT/DenseSet.h"
27#include "llvm/ADT/DepthFirstIterator.h"
28#include "llvm/ADT/PostOrderIterator.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SetOperations.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/StringRef.h"
34#include "llvm/ADT/Twine.h"
35#include "llvm/CodeGen/CodeGenCommonISel.h"
36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
37#include "llvm/CodeGen/LiveInterval.h"
38#include "llvm/CodeGen/LiveIntervals.h"
39#include "llvm/CodeGen/LiveRangeCalc.h"
40#include "llvm/CodeGen/LiveStacks.h"
41#include "llvm/CodeGen/MachineBasicBlock.h"
42#include "llvm/CodeGen/MachineConvergenceVerifier.h"
43#include "llvm/CodeGen/MachineDominators.h"
44#include "llvm/CodeGen/MachineFrameInfo.h"
45#include "llvm/CodeGen/MachineFunction.h"
46#include "llvm/CodeGen/MachineFunctionPass.h"
47#include "llvm/CodeGen/MachineInstr.h"
48#include "llvm/CodeGen/MachineInstrBundle.h"
49#include "llvm/CodeGen/MachineMemOperand.h"
50#include "llvm/CodeGen/MachineOperand.h"
51#include "llvm/CodeGen/MachineRegisterInfo.h"
52#include "llvm/CodeGen/PseudoSourceValue.h"
53#include "llvm/CodeGen/RegisterBank.h"
54#include "llvm/CodeGen/RegisterBankInfo.h"
55#include "llvm/CodeGen/SlotIndexes.h"
56#include "llvm/CodeGen/StackMaps.h"
57#include "llvm/CodeGen/TargetInstrInfo.h"
58#include "llvm/CodeGen/TargetLowering.h"
59#include "llvm/CodeGen/TargetOpcodes.h"
60#include "llvm/CodeGen/TargetRegisterInfo.h"
61#include "llvm/CodeGen/TargetSubtargetInfo.h"
62#include "llvm/CodeGenTypes/LowLevelType.h"
63#include "llvm/IR/BasicBlock.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/EHPersonalities.h"
66#include "llvm/IR/Function.h"
67#include "llvm/IR/InlineAsm.h"
68#include "llvm/IR/Instructions.h"
69#include "llvm/InitializePasses.h"
70#include "llvm/MC/LaneBitmask.h"
71#include "llvm/MC/MCAsmInfo.h"
72#include "llvm/MC/MCDwarf.h"
73#include "llvm/MC/MCInstrDesc.h"
74#include "llvm/MC/MCRegisterInfo.h"
75#include "llvm/MC/MCTargetOptions.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/ErrorHandling.h"
79#include "llvm/Support/ManagedStatic.h"
80#include "llvm/Support/MathExtras.h"
81#include "llvm/Support/ModRef.h"
82#include "llvm/Support/Mutex.h"
83#include "llvm/Support/raw_ostream.h"
84#include "llvm/Target/TargetMachine.h"
85#include <algorithm>
86#include <cassert>
87#include <cstddef>
88#include <cstdint>
89#include <iterator>
90#include <string>
91#include <utility>
92
93using namespace llvm;
94
95namespace {
96
97/// Used the by the ReportedErrors class to guarantee only one error is reported
98/// at one time.
99static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock;
100
101static bool hasPhysRegClassForType(const TargetRegisterInfo &TRI,
102 MCRegister Reg, LLT Ty) {
103 assert(Reg.isPhysical() && "reg must be a physical register");
104 assert(Ty.isValid() && "expected a valid type");
105
106 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
107 if (TRI.isTypeLegalForClass(RC: *RC, T: Ty))
108 return true;
109
110 return llvm::any_of(Range: TRI.regclasses(), P: [&](const TargetRegisterClass &RC) {
111 return RC.contains(Reg) && TRI.isTypeLegalForClass(RC, T: Ty);
112 });
113}
114
115struct MachineVerifier {
116 MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b,
117 raw_ostream *OS, bool AbortOnError = true)
118 : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b),
119 ReportedErrs(AbortOnError) {}
120
121 MachineVerifier(Pass *pass, const char *b, raw_ostream *OS,
122 bool AbortOnError = true)
123 : PASS(pass), OS(OS ? *OS : nulls()), Banner(b),
124 ReportedErrs(AbortOnError) {}
125
126 MachineVerifier(const char *b, LiveIntervals *LiveInts, LiveStacks *LiveStks,
127 SlotIndexes *Indexes, raw_ostream *OS,
128 bool AbortOnError = true)
129 : OS(OS ? *OS : nulls()), Banner(b), LiveInts(LiveInts),
130 LiveStks(LiveStks), Indexes(Indexes), ReportedErrs(AbortOnError) {}
131
132 /// \returns true if no problems were found.
133 bool verify(const MachineFunction &MF);
134
135 MachineFunctionAnalysisManager *MFAM = nullptr;
136 Pass *const PASS = nullptr;
137 raw_ostream &OS;
138 const char *Banner;
139 const MachineFunction *MF = nullptr;
140 const TargetMachine *TM = nullptr;
141 const TargetInstrInfo *TII = nullptr;
142 const TargetRegisterInfo *TRI = nullptr;
143 const MachineRegisterInfo *MRI = nullptr;
144 const RegisterBankInfo *RBI = nullptr;
145
146 // Avoid querying the MachineFunctionProperties for each operand.
147 bool isFunctionRegBankSelected = false;
148 bool isFunctionSelected = false;
149 bool isFunctionTracksDebugUserValues = false;
150
151 using RegVector = SmallVector<Register, 16>;
152 using RegMaskVector = SmallVector<const uint32_t *, 4>;
153 using RegSet = DenseSet<Register>;
154 using RegMap = DenseMap<Register, const MachineInstr *>;
155 using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;
156
157 const MachineInstr *FirstNonPHI = nullptr;
158 const MachineInstr *FirstTerminator = nullptr;
159 BlockSet FunctionBlocks;
160
161 BitVector regsReserved;
162 RegSet regsLive;
163 RegVector regsDefined, regsDead, regsKilled;
164 RegMaskVector regMasks;
165
166 // Live physical register defs of the current instruction, and the operand
167 // indices of its dead physical register defs.
168 SmallVector<MCRegister, 4> LivePhysDefs;
169 SmallVector<unsigned, 4> DeadPhysDefOps;
170
171 SlotIndex lastIndex;
172
173 // Add Reg and any sub-registers to RV
174 void addRegWithSubRegs(RegVector &RV, Register Reg) {
175 RV.push_back(Elt: Reg);
176 if (Reg.isPhysical())
177 append_range(C&: RV, R: TRI->subregs(Reg: Reg.asMCReg()));
178 }
179
180 struct BBInfo {
181 // Is this MBB reachable from the MF entry point?
182 bool reachable = false;
183
184 // Vregs that must be live in because they are used without being
185 // defined. Map value is the user. vregsLiveIn doesn't include regs
186 // that only are used by PHI nodes.
187 RegMap vregsLiveIn;
188
189 // Regs killed in MBB. They may be defined again, and will then be in both
190 // regsKilled and regsLiveOut.
191 RegSet regsKilled;
192
193 // Regs defined in MBB and live out. Note that vregs passing through may
194 // be live out without being mentioned here.
195 RegSet regsLiveOut;
196
197 // Vregs that pass through MBB untouched. This set is disjoint from
198 // regsKilled and regsLiveOut.
199 RegSet vregsPassed;
200
201 // Vregs that must pass through MBB because they are needed by a successor
202 // block. This set is disjoint from regsLiveOut.
203 RegSet vregsRequired;
204
205 // Set versions of block's predecessor and successor lists.
206 BlockSet Preds, Succs;
207
208 BBInfo() = default;
209
210 // Add register to vregsRequired if it belongs there. Return true if
211 // anything changed.
212 bool addRequired(Register Reg) {
213 if (!Reg.isVirtual())
214 return false;
215 if (regsLiveOut.count(V: Reg))
216 return false;
217 return vregsRequired.insert(V: Reg).second;
218 }
219
220 // Same for a full set.
221 bool addRequired(const RegSet &RS) {
222 bool Changed = false;
223 for (Register Reg : RS)
224 Changed |= addRequired(Reg);
225 return Changed;
226 }
227
228 // Same for a full map.
229 bool addRequired(const RegMap &RM) {
230 bool Changed = false;
231 for (const auto &I : RM)
232 Changed |= addRequired(Reg: I.first);
233 return Changed;
234 }
235
236 // Live-out registers are either in regsLiveOut or vregsPassed.
237 bool isLiveOut(Register Reg) const {
238 return regsLiveOut.count(V: Reg) || vregsPassed.count(V: Reg);
239 }
240 };
241
242 // Extra register info per MBB.
243 DenseMap<const MachineBasicBlock *, BBInfo> MBBInfoMap;
244
245 bool isReserved(Register Reg) {
246 return Reg.id() < regsReserved.size() && regsReserved.test(Idx: Reg.id());
247 }
248
249 bool isAllocatable(Register Reg) const {
250 return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(RegNo: Reg) &&
251 !regsReserved.test(Idx: Reg.id());
252 }
253
254 // Analysis information if available
255 LiveIntervals *LiveInts = nullptr;
256 LiveStacks *LiveStks = nullptr;
257 SlotIndexes *Indexes = nullptr;
258
259 /// A class to track the number of reported error and to guarantee that only
260 /// one error is reported at one time.
261 class ReportedErrors {
262 unsigned NumReported = 0;
263 bool AbortOnError;
264
265 public:
266 /// \param AbortOnError -- If set, abort after printing the first error.
267 ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {}
268
269 ~ReportedErrors() {
270 if (!hasError())
271 return;
272 if (AbortOnError)
273 report_fatal_error(reason: "Found " + Twine(NumReported) +
274 " machine code errors.");
275 // Since we haven't aborted, release the lock to allow other threads to
276 // report errors.
277 ReportedErrorsLock->unlock();
278 }
279
280 /// Increment the number of reported errors.
281 /// \returns true if this is the first reported error.
282 bool increment() {
283 // If this is the first error this thread has encountered, grab the lock
284 // to prevent other threads from reporting errors at the same time.
285 // Otherwise we assume we already have the lock.
286 if (!hasError())
287 ReportedErrorsLock->lock();
288 ++NumReported;
289 return NumReported == 1;
290 }
291
292 /// \returns true if an error was reported.
293 bool hasError() { return NumReported; }
294 };
295 ReportedErrors ReportedErrs;
296
297 // This is calculated only when trying to verify convergence control tokens.
298 // Similar to the LLVM IR verifier, we calculate this locally instead of
299 // relying on the pass manager.
300 MachineDominatorTree DT;
301
302 void visitMachineFunctionBefore();
303 void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
304 void visitMachineBundleBefore(const MachineInstr *MI);
305
306 /// Verify that all of \p MI's virtual register operands are scalars.
307 /// \returns True if all virtual register operands are scalar. False
308 /// otherwise.
309 bool verifyAllRegOpsScalar(const MachineInstr &MI,
310 const MachineRegisterInfo &MRI);
311 bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);
312
313 bool verifyGIntrinsicSideEffects(const MachineInstr *MI);
314 bool verifyGIntrinsicConvergence(const MachineInstr *MI);
315 void verifyPreISelGenericInstruction(const MachineInstr *MI);
316
317 void visitMachineInstrBefore(const MachineInstr *MI);
318 void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
319 void visitMachineInstrAfter(const MachineInstr *MI);
320 void visitMachineBundleAfter(const MachineInstr *MI);
321 void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);
322 void visitMachineFunctionAfter();
323
324 void report(const char *msg, const MachineFunction *MF);
325 void report(const char *msg, const MachineBasicBlock *MBB);
326 void report(const char *msg, const MachineInstr *MI);
327 void report(const char *msg, const MachineOperand *MO, unsigned MONum,
328 LLT MOVRegType = LLT{});
329 void report(const Twine &Msg, const MachineInstr *MI);
330
331 void report_context(const LiveInterval &LI) const;
332 void report_context(const LiveRange &LR, VirtRegOrUnit VRegOrUnit,
333 LaneBitmask LaneMask) const;
334 void report_context(const LiveRange::Segment &S) const;
335 void report_context(const VNInfo &VNI) const;
336 void report_context(SlotIndex Pos) const;
337 void report_context(MCPhysReg PhysReg) const;
338 void report_context_liverange(const LiveRange &LR) const;
339 void report_context_lanemask(LaneBitmask LaneMask) const;
340 void report_context_vreg(Register VReg) const;
341 void report_context_vreg_regunit(VirtRegOrUnit VRegOrUnit) const;
342
343 void verifyInlineAsm(const MachineInstr *MI);
344
345 void checkLiveness(const MachineOperand *MO, unsigned MONum);
346 void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
347 SlotIndex UseIdx, const LiveRange &LR,
348 VirtRegOrUnit VRegOrUnit,
349 LaneBitmask LaneMask = LaneBitmask::getNone());
350 void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
351 SlotIndex DefIdx, const LiveRange &LR,
352 VirtRegOrUnit VRegOrUnit, bool SubRangeCheck = false,
353 LaneBitmask LaneMask = LaneBitmask::getNone());
354
355 void markReachable(const MachineBasicBlock *MBB);
356 void calcRegsPassed();
357 void checkPHIOps(const MachineBasicBlock &MBB);
358
359 void calcRegsRequired();
360 void verifyLiveIntervals();
361 void verifyLiveInterval(const LiveInterval &);
362 void verifyLiveRangeValue(const LiveRange &, const VNInfo *, VirtRegOrUnit,
363 LaneBitmask);
364 void verifyLiveRangeSegment(const LiveRange &,
365 const LiveRange::const_iterator I, VirtRegOrUnit,
366 LaneBitmask);
367 void verifyLiveRange(const LiveRange &, VirtRegOrUnit,
368 LaneBitmask LaneMask = LaneBitmask::getNone());
369
370 void verifyStackFrame();
371 /// Check that the stack protector is the top-most object in the stack.
372 void verifyStackProtector();
373
374 void verifySlotIndexes() const;
375 void verifyProperties(const MachineFunction &MF);
376};
377
378struct MachineVerifierLegacyPass : public MachineFunctionPass {
379 static char ID; // Pass ID, replacement for typeid
380
381 const std::string Banner;
382
383 MachineVerifierLegacyPass(std::string banner = std::string())
384 : MachineFunctionPass(ID), Banner(std::move(banner)) {}
385
386 void getAnalysisUsage(AnalysisUsage &AU) const override {
387 AU.addUsedIfAvailable<LiveStacksWrapperLegacy>();
388 AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
389 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
390 AU.setPreservesAll();
391 MachineFunctionPass::getAnalysisUsage(AU);
392 }
393
394 bool runOnMachineFunction(MachineFunction &MF) override {
395 // Skip functions that have known verification problems.
396 // FIXME: Remove this mechanism when all problematic passes have been
397 // fixed.
398 if (MF.getProperties().hasFailsVerification())
399 return false;
400
401 MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);
402 return false;
403 }
404};
405
406} // end anonymous namespace
407
408PreservedAnalyses
409MachineVerifierPass::run(MachineFunction &MF,
410 MachineFunctionAnalysisManager &MFAM) {
411 // Skip functions that have known verification problems.
412 // FIXME: Remove this mechanism when all problematic passes have been
413 // fixed.
414 if (MF.getProperties().hasFailsVerification())
415 return PreservedAnalyses::all();
416 MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);
417 return PreservedAnalyses::all();
418}
419
420char MachineVerifierLegacyPass::ID = 0;
421
422INITIALIZE_PASS(MachineVerifierLegacyPass, "machineverifier",
423 "Verify generated machine code", false, false)
424
425FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {
426 return new MachineVerifierLegacyPass(Banner);
427}
428
429void llvm::verifyMachineFunction(const std::string &Banner,
430 const MachineFunction &MF) {
431 // TODO: Use MFAM after porting below analyses.
432 // LiveIntervals *LiveInts;
433 // LiveStacks *LiveStks;
434 // SlotIndexes *Indexes;
435 MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);
436}
437
438bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,
439 bool AbortOnError) const {
440 return MachineVerifier(p, Banner, OS, AbortOnError).verify(MF: *this);
441}
442
443bool MachineFunction::verify(MachineFunctionAnalysisManager &MFAM,
444 const char *Banner, raw_ostream *OS,
445 bool AbortOnError) const {
446 return MachineVerifier(MFAM, Banner, OS, AbortOnError).verify(MF: *this);
447}
448
449bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
450 const char *Banner, raw_ostream *OS,
451 bool AbortOnError) const {
452 return MachineVerifier(Banner, LiveInts, /*LiveStks=*/nullptr, Indexes, OS,
453 AbortOnError)
454 .verify(MF: *this);
455}
456
457void MachineVerifier::verifySlotIndexes() const {
458 if (Indexes == nullptr)
459 return;
460
461 // Ensure the IdxMBB list is sorted by slot indexes.
462 SlotIndex Last;
463 for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),
464 E = Indexes->MBBIndexEnd(); I != E; ++I) {
465 assert(!Last.isValid() || I->first > Last);
466 Last = I->first;
467 }
468}
469
470void MachineVerifier::verifyProperties(const MachineFunction &MF) {
471 // If a pass has introduced virtual registers without clearing the
472 // NoVRegs property (or set it without allocating the vregs)
473 // then report an error.
474 if (MF.getProperties().hasNoVRegs() && MRI->getNumVirtRegs())
475 report(msg: "Function has NoVRegs property but there are VReg operands", MF: &MF);
476}
477
478bool MachineVerifier::verify(const MachineFunction &MF) {
479 this->MF = &MF;
480 TM = &MF.getTarget();
481 TII = MF.getSubtarget().getInstrInfo();
482 TRI = MF.getSubtarget().getRegisterInfo();
483 RBI = MF.getSubtarget().getRegBankInfo();
484 MRI = &MF.getRegInfo();
485
486 const MachineFunctionProperties &Props = MF.getProperties();
487 const bool isFunctionFailedISel = Props.hasFailedISel();
488
489 // If we're mid-GlobalISel and we already triggered the fallback path then
490 // it's expected that the MIR is somewhat broken but that's ok since we'll
491 // reset it and clear the FailedISel attribute in ResetMachineFunctions.
492 if (isFunctionFailedISel)
493 return true;
494
495 isFunctionRegBankSelected = Props.hasRegBankSelected();
496 isFunctionSelected = Props.hasSelected();
497 isFunctionTracksDebugUserValues = Props.hasTracksDebugUserValues();
498
499 if (PASS) {
500 auto *LISWrapper = PASS->getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
501 LiveInts = LISWrapper ? &LISWrapper->getLIS() : nullptr;
502 auto *LSWrapper = PASS->getAnalysisIfAvailable<LiveStacksWrapperLegacy>();
503 LiveStks = LSWrapper ? &LSWrapper->getLS() : nullptr;
504 auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>();
505 Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
506 }
507 if (MFAM) {
508 MachineFunction &Func = const_cast<MachineFunction &>(MF);
509 LiveInts = MFAM->getCachedResult<LiveIntervalsAnalysis>(IR&: Func);
510 // TODO: LiveStks = MFAM->getCachedResult<LiveStacksAnalysis>(Func);
511 Indexes = MFAM->getCachedResult<SlotIndexesAnalysis>(IR&: Func);
512 }
513
514 verifySlotIndexes();
515
516 verifyProperties(MF);
517
518 visitMachineFunctionBefore();
519 for (const MachineBasicBlock &MBB : MF) {
520 visitMachineBasicBlockBefore(MBB: &MBB);
521 // Keep track of the current bundle header.
522 const MachineInstr *CurBundle = nullptr;
523 // Do we expect the next instruction to be part of the same bundle?
524 bool InBundle = false;
525
526 for (const MachineInstr &MI : MBB.instrs()) {
527 if (MI.getParent() != &MBB) {
528 report(msg: "Bad instruction parent pointer", MBB: &MBB);
529 OS << "Instruction: " << MI;
530 continue;
531 }
532
533 // Check for consistent bundle flags.
534 if (InBundle && !MI.isBundledWithPred())
535 report(msg: "Missing BundledPred flag, "
536 "BundledSucc was set on predecessor",
537 MI: &MI);
538 if (!InBundle && MI.isBundledWithPred())
539 report(msg: "BundledPred flag is set, "
540 "but BundledSucc not set on predecessor",
541 MI: &MI);
542
543 // Is this a bundle header?
544 if (!MI.isInsideBundle()) {
545 if (CurBundle)
546 visitMachineBundleAfter(MI: CurBundle);
547 CurBundle = &MI;
548 visitMachineBundleBefore(MI: CurBundle);
549 } else if (!CurBundle)
550 report(msg: "No bundle header", MI: &MI);
551 visitMachineInstrBefore(MI: &MI);
552 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
553 const MachineOperand &Op = MI.getOperand(i: I);
554 if (Op.getParent() != &MI) {
555 // Make sure to use correct addOperand / removeOperand / ChangeTo
556 // functions when replacing operands of a MachineInstr.
557 report(msg: "Instruction has operand with wrong parent set", MI: &MI);
558 }
559
560 visitMachineOperand(MO: &Op, MONum: I);
561 }
562 visitMachineInstrAfter(MI: &MI);
563
564 // Was this the last bundled instruction?
565 InBundle = MI.isBundledWithSucc();
566 }
567 if (CurBundle)
568 visitMachineBundleAfter(MI: CurBundle);
569 if (InBundle)
570 report(msg: "BundledSucc flag set on last instruction in block", MI: &MBB.back());
571 visitMachineBasicBlockAfter(MBB: &MBB);
572 }
573 visitMachineFunctionAfter();
574
575 // Clean up.
576 regsLive.clear();
577 regsDefined.clear();
578 regsDead.clear();
579 regsKilled.clear();
580 regMasks.clear();
581 MBBInfoMap.clear();
582
583 return !ReportedErrs.hasError();
584}
585
586void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
587 assert(MF);
588 OS << '\n';
589 if (ReportedErrs.increment()) {
590 if (Banner)
591 OS << "# " << Banner << '\n';
592
593 if (LiveInts != nullptr)
594 LiveInts->print(O&: OS);
595 else
596 MF->print(OS, Indexes);
597 }
598
599 OS << "*** Bad machine code: " << msg << " ***\n"
600 << "- function: " << MF->getName() << '\n';
601}
602
603void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
604 assert(MBB);
605 report(msg, MF: MBB->getParent());
606 OS << "- basic block: " << printMBBReference(MBB: *MBB) << ' ' << MBB->getName()
607 << " (" << (const void *)MBB << ')';
608 if (Indexes)
609 OS << " [" << Indexes->getMBBStartIdx(mbb: MBB) << ';'
610 << Indexes->getMBBEndIdx(mbb: MBB) << ')';
611 OS << '\n';
612}
613
614void MachineVerifier::report(const char *msg, const MachineInstr *MI) {
615 assert(MI);
616 report(msg, MBB: MI->getParent());
617 OS << "- instruction: ";
618 if (Indexes && Indexes->hasIndex(instr: *MI))
619 OS << Indexes->getInstructionIndex(MI: *MI) << '\t';
620 MI->print(OS, /*IsStandalone=*/true);
621}
622
623void MachineVerifier::report(const char *msg, const MachineOperand *MO,
624 unsigned MONum, LLT MOVRegType) {
625 assert(MO);
626 report(msg, MI: MO->getParent());
627 OS << "- operand " << MONum << ": ";
628 MO->print(os&: OS, TypeToPrint: MOVRegType, TRI);
629 OS << '\n';
630}
631
632void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) {
633 report(msg: Msg.str().c_str(), MI);
634}
635
636void MachineVerifier::report_context(SlotIndex Pos) const {
637 OS << "- at: " << Pos << '\n';
638}
639
640void MachineVerifier::report_context(const LiveInterval &LI) const {
641 OS << "- interval: " << LI << '\n';
642}
643
644void MachineVerifier::report_context(const LiveRange &LR,
645 VirtRegOrUnit VRegOrUnit,
646 LaneBitmask LaneMask) const {
647 report_context_liverange(LR);
648 report_context_vreg_regunit(VRegOrUnit);
649 if (LaneMask.any())
650 report_context_lanemask(LaneMask);
651}
652
653void MachineVerifier::report_context(const LiveRange::Segment &S) const {
654 OS << "- segment: " << S << '\n';
655}
656
657void MachineVerifier::report_context(const VNInfo &VNI) const {
658 OS << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n";
659}
660
661void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
662 OS << "- liverange: " << LR << '\n';
663}
664
665void MachineVerifier::report_context(MCPhysReg PReg) const {
666 OS << "- p. register: " << printReg(Reg: PReg, TRI) << '\n';
667}
668
669void MachineVerifier::report_context_vreg(Register VReg) const {
670 OS << "- v. register: " << printReg(Reg: VReg, TRI) << '\n';
671}
672
673void MachineVerifier::report_context_vreg_regunit(
674 VirtRegOrUnit VRegOrUnit) const {
675 if (VRegOrUnit.isVirtualReg()) {
676 report_context_vreg(VReg: VRegOrUnit.asVirtualReg());
677 } else {
678 OS << "- regunit: " << printRegUnit(Unit: VRegOrUnit.asMCRegUnit(), TRI)
679 << '\n';
680 }
681}
682
683void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {
684 OS << "- lanemask: " << PrintLaneMask(LaneMask) << '\n';
685}
686
687void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
688 BBInfo &MInfo = MBBInfoMap[MBB];
689 if (!MInfo.reachable) {
690 MInfo.reachable = true;
691 for (const MachineBasicBlock *Succ : MBB->successors())
692 markReachable(MBB: Succ);
693 }
694}
695
696void MachineVerifier::visitMachineFunctionBefore() {
697 lastIndex = SlotIndex();
698 regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()
699 : TRI->getReservedRegs(MF: *MF);
700
701 if (!MF->empty())
702 markReachable(MBB: &MF->front());
703
704 // Build a set of the basic blocks in the function.
705 FunctionBlocks.clear();
706 for (const auto &MBB : *MF) {
707 FunctionBlocks.insert(Ptr: &MBB);
708 BBInfo &MInfo = MBBInfoMap[&MBB];
709
710 MInfo.Preds.insert_range(R: MBB.predecessors());
711 if (MInfo.Preds.size() != MBB.pred_size())
712 report(msg: "MBB has duplicate entries in its predecessor list.", MBB: &MBB);
713
714 MInfo.Succs.insert_range(R: MBB.successors());
715 if (MInfo.Succs.size() != MBB.succ_size())
716 report(msg: "MBB has duplicate entries in its successor list.", MBB: &MBB);
717 }
718
719 // Check that the register use lists are sane.
720 MRI->verifyUseLists();
721
722 if (!MF->empty()) {
723 verifyStackFrame();
724 verifyStackProtector();
725 }
726}
727
728static bool hasPHIs(const MachineFunction &MF) {
729 return !MF.getProperties().hasNoPHIs() &&
730 any_of(Range: MF, P: [](const MachineBasicBlock &MBB) {
731 return !MBB.phis().empty();
732 });
733}
734
735void
736MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
737 FirstTerminator = nullptr;
738 FirstNonPHI = nullptr;
739
740 if (MRI->tracksLiveness() && hasPHIs(MF: *MF)) {
741 // If this block has allocatable physical registers live-in, check that
742 // it is an entry block or landing pad.
743 for (const auto &LI : MBB->liveins()) {
744 if (isAllocatable(Reg: LI.PhysReg) && !MBB->isEHPad() &&
745 MBB->getIterator() != MBB->getParent()->begin() &&
746 !MBB->isInlineAsmBrIndirectTarget()) {
747 report(msg: "MBB has allocatable live-in, but isn't entry, landing-pad, or "
748 "inlineasm-br-indirect-target.",
749 MBB);
750 report_context(PReg: LI.PhysReg);
751 }
752 }
753 }
754
755 if (MBB->isIRBlockAddressTaken()) {
756 if (!MBB->getAddressTakenIRBlock()->hasAddressTaken())
757 report(msg: "ir-block-address-taken is associated with basic block not used by "
758 "a blockaddress.",
759 MBB);
760 }
761
762 // Count the number of landing pad successors.
763 SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs;
764 for (const auto *succ : MBB->successors()) {
765 if (succ->isEHPad())
766 LandingPadSuccs.insert(Ptr: succ);
767 if (!FunctionBlocks.count(Ptr: succ))
768 report(msg: "MBB has successor that isn't part of the function.", MBB);
769 if (!MBBInfoMap[succ].Preds.count(Ptr: MBB)) {
770 report(msg: "Inconsistent CFG", MBB);
771 OS << "MBB is not in the predecessor list of the successor "
772 << printMBBReference(MBB: *succ) << ".\n";
773 }
774 }
775
776 // Check the predecessor list.
777 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
778 if (!FunctionBlocks.count(Ptr: Pred))
779 report(msg: "MBB has predecessor that isn't part of the function.", MBB);
780 if (!MBBInfoMap[Pred].Succs.count(Ptr: MBB)) {
781 report(msg: "Inconsistent CFG", MBB);
782 OS << "MBB is not in the successor list of the predecessor "
783 << printMBBReference(MBB: *Pred) << ".\n";
784 }
785 }
786
787 const MCAsmInfo &AsmInfo = TM->getMCAsmInfo();
788 const BasicBlock *BB = MBB->getBasicBlock();
789 const Function &F = MF->getFunction();
790 if (LandingPadSuccs.size() > 1 &&
791 !(AsmInfo.getExceptionHandlingType() == ExceptionHandling::SjLj && BB &&
792 isa<SwitchInst>(Val: BB->getTerminator())) &&
793 !isScopedEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
794 report(msg: "MBB has more than one landing pad successor", MBB);
795
796 // Call analyzeBranch. If it succeeds, there several more conditions to check.
797 const MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
798 SmallVector<MachineOperand, 4> Cond;
799 if (!TII->analyzeBranch(MBB: *MBB, TBB, FBB, Cond)) {
800 // Ok, analyzeBranch thinks it knows what's going on with this block. Let's
801 // check whether its answers match up with reality.
802 if (!TBB && !FBB) {
803 // Block falls through to its successor.
804 if (!MBB->empty() && MBB->back().isBarrier() &&
805 !TII->isPredicated(MI: MBB->back())) {
806 report(msg: "MBB exits via unconditional fall-through but ends with a "
807 "barrier instruction!", MBB);
808 }
809 if (!Cond.empty()) {
810 report(msg: "MBB exits via unconditional fall-through but has a condition!",
811 MBB);
812 }
813 } else if (TBB && !FBB && Cond.empty()) {
814 // Block unconditionally branches somewhere.
815 if (MBB->empty()) {
816 report(msg: "MBB exits via unconditional branch but doesn't contain "
817 "any instructions!", MBB);
818 } else if (!MBB->back().isBarrier()) {
819 report(msg: "MBB exits via unconditional branch but doesn't end with a "
820 "barrier instruction!", MBB);
821 } else if (!MBB->back().isTerminator()) {
822 report(msg: "MBB exits via unconditional branch but the branch isn't a "
823 "terminator instruction!", MBB);
824 }
825 } else if (TBB && !FBB && !Cond.empty()) {
826 // Block conditionally branches somewhere, otherwise falls through.
827 if (MBB->empty()) {
828 report(msg: "MBB exits via conditional branch/fall-through but doesn't "
829 "contain any instructions!", MBB);
830 } else if (MBB->back().isBarrier()) {
831 report(msg: "MBB exits via conditional branch/fall-through but ends with a "
832 "barrier instruction!", MBB);
833 } else if (!MBB->back().isTerminator()) {
834 report(msg: "MBB exits via conditional branch/fall-through but the branch "
835 "isn't a terminator instruction!", MBB);
836 }
837 } else if (TBB && FBB) {
838 // Block conditionally branches somewhere, otherwise branches
839 // somewhere else.
840 if (MBB->empty()) {
841 report(msg: "MBB exits via conditional branch/branch but doesn't "
842 "contain any instructions!", MBB);
843 } else if (!MBB->back().isBarrier()) {
844 report(msg: "MBB exits via conditional branch/branch but doesn't end with a "
845 "barrier instruction!", MBB);
846 } else if (!MBB->back().isTerminator()) {
847 report(msg: "MBB exits via conditional branch/branch but the branch "
848 "isn't a terminator instruction!", MBB);
849 }
850 if (Cond.empty()) {
851 report(msg: "MBB exits via conditional branch/branch but there's no "
852 "condition!", MBB);
853 }
854 } else {
855 report(msg: "analyzeBranch returned invalid data!", MBB);
856 }
857
858 // Now check that the successors match up with the answers reported by
859 // analyzeBranch.
860 if (TBB && !MBB->isSuccessor(MBB: TBB))
861 report(msg: "MBB exits via jump or conditional branch, but its target isn't a "
862 "CFG successor!",
863 MBB);
864 if (FBB && !MBB->isSuccessor(MBB: FBB))
865 report(msg: "MBB exits via conditional branch, but its target isn't a CFG "
866 "successor!",
867 MBB);
868
869 // There might be a fallthrough to the next block if there's either no
870 // unconditional true branch, or if there's a condition, and one of the
871 // branches is missing.
872 bool Fallthrough = !TBB || (!Cond.empty() && !FBB);
873
874 // A conditional fallthrough must be an actual CFG successor, not
875 // unreachable. (Conversely, an unconditional fallthrough might not really
876 // be a successor, because the block might end in unreachable.)
877 if (!Cond.empty() && !FBB) {
878 MachineFunction::const_iterator MBBI = std::next(x: MBB->getIterator());
879 if (MBBI == MF->end()) {
880 report(msg: "MBB conditionally falls through out of function!", MBB);
881 } else if (!MBB->isSuccessor(MBB: &*MBBI))
882 report(msg: "MBB exits via conditional branch/fall-through but the CFG "
883 "successors don't match the actual successors!",
884 MBB);
885 }
886
887 // Verify that there aren't any extra un-accounted-for successors.
888 for (const MachineBasicBlock *SuccMBB : MBB->successors()) {
889 // If this successor is one of the branch targets, it's okay.
890 if (SuccMBB == TBB || SuccMBB == FBB)
891 continue;
892 // If we might have a fallthrough, and the successor is the fallthrough
893 // block, that's also ok.
894 if (Fallthrough && SuccMBB == MBB->getNextNode())
895 continue;
896 // Also accept successors which are for exception-handling or might be
897 // inlineasm_br targets.
898 if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
899 continue;
900 report(msg: "MBB has unexpected successors which are not branch targets, "
901 "fallthrough, EHPads, or inlineasm_br targets.",
902 MBB);
903 }
904 }
905
906 regsLive.clear();
907 if (MRI->tracksLiveness()) {
908 for (const auto &LI : MBB->liveins()) {
909 if (!LI.PhysReg.isPhysical()) {
910 report(msg: "MBB live-in list contains non-physical register", MBB);
911 continue;
912 }
913 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: LI.PhysReg));
914 }
915 }
916
917 const MachineFrameInfo &MFI = MF->getFrameInfo();
918 BitVector PR = MFI.getPristineRegs(MF: *MF);
919 for (unsigned I : PR.set_bits())
920 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: I));
921
922 regsKilled.clear();
923 regsDefined.clear();
924
925 if (Indexes)
926 lastIndex = Indexes->getMBBStartIdx(mbb: MBB);
927}
928
929// This function gets called for all bundle headers, including normal
930// stand-alone unbundled instructions.
931void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
932 if (Indexes && Indexes->hasIndex(instr: *MI)) {
933 SlotIndex idx = Indexes->getInstructionIndex(MI: *MI);
934 if (!(idx > lastIndex)) {
935 report(msg: "Instruction index out of order", MI);
936 OS << "Last instruction was at " << lastIndex << '\n';
937 }
938 lastIndex = idx;
939 }
940
941 // Ensure non-terminators don't follow terminators.
942 if (MI->isTerminator()) {
943 if (!FirstTerminator)
944 FirstTerminator = MI;
945 } else if (FirstTerminator) {
946 // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to
947 // precede non-terminators.
948 if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) {
949 report(msg: "Non-terminator instruction after the first terminator", MI);
950 OS << "First terminator was:\t" << *FirstTerminator;
951 }
952 }
953}
954
955// The operands on an INLINEASM instruction must follow a template.
956// Verify that the flag operands make sense.
957void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
958 // The first two operands on INLINEASM are the asm string and global flags.
959 if (MI->getNumOperands() < 2) {
960 report(msg: "Too few operands on inline asm", MI);
961 return;
962 }
963 if (!MI->getOperand(i: 0).isSymbol())
964 report(msg: "Asm string must be an external symbol", MI);
965 if (!MI->getOperand(i: 1).isImm())
966 report(msg: "Asm flags must be an immediate", MI);
967 // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
968 // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,
969 // and Extra_IsConvergent = 32, Extra_MayUnwind = 64.
970 if (!isUInt<7>(x: MI->getOperand(i: 1).getImm()))
971 report(msg: "Unknown asm flags", MO: &MI->getOperand(i: 1), MONum: 1);
972
973 static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");
974
975 unsigned OpNo = InlineAsm::MIOp_FirstOperand;
976 unsigned NumOps;
977 for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
978 const MachineOperand &MO = MI->getOperand(i: OpNo);
979 // There may be implicit ops after the fixed operands.
980 if (!MO.isImm())
981 break;
982 const InlineAsm::Flag F(MO.getImm());
983 NumOps = 1 + F.getNumOperandRegisters();
984 }
985
986 if (OpNo > MI->getNumOperands())
987 report(msg: "Missing operands in last group", MI);
988
989 // An optional MDNode follows the groups.
990 if (OpNo < MI->getNumOperands() && MI->getOperand(i: OpNo).isMetadata())
991 ++OpNo;
992
993 // All trailing operands must be implicit registers.
994 for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
995 const MachineOperand &MO = MI->getOperand(i: OpNo);
996 if (!MO.isReg() || !MO.isImplicit())
997 report(msg: "Expected implicit register after groups", MO: &MO, MONum: OpNo);
998 }
999
1000 if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) {
1001 const MachineBasicBlock *MBB = MI->getParent();
1002
1003 for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands();
1004 i != e; ++i) {
1005 const MachineOperand &MO = MI->getOperand(i);
1006
1007 if (!MO.isMBB())
1008 continue;
1009
1010 // Check the successor & predecessor lists look ok, assume they are
1011 // not. Find the indirect target without going through the successors.
1012 const MachineBasicBlock *IndirectTargetMBB = MO.getMBB();
1013 if (!IndirectTargetMBB) {
1014 report(msg: "INLINEASM_BR indirect target does not exist", MO: &MO, MONum: i);
1015 break;
1016 }
1017
1018 if (!MBB->isSuccessor(MBB: IndirectTargetMBB))
1019 report(msg: "INLINEASM_BR indirect target missing from successor list", MO: &MO,
1020 MONum: i);
1021
1022 if (!IndirectTargetMBB->isPredecessor(MBB))
1023 report(msg: "INLINEASM_BR indirect target predecessor list missing parent",
1024 MO: &MO, MONum: i);
1025 }
1026 }
1027}
1028
1029bool MachineVerifier::verifyAllRegOpsScalar(const MachineInstr &MI,
1030 const MachineRegisterInfo &MRI) {
1031 if (none_of(Range: MI.explicit_operands(), P: [&MRI](const MachineOperand &Op) {
1032 if (!Op.isReg())
1033 return false;
1034 const auto Reg = Op.getReg();
1035 if (Reg.isPhysical())
1036 return false;
1037 return !MRI.getType(Reg).isScalar();
1038 }))
1039 return true;
1040 report(msg: "All register operands must have scalar types", MI: &MI);
1041 return false;
1042}
1043
1044/// Check that types are consistent when two operands need to have the same
1045/// number of vector elements.
1046/// \return true if the types are valid.
1047bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,
1048 const MachineInstr *MI) {
1049 if (Ty0.isVector() != Ty1.isVector()) {
1050 report(msg: "operand types must be all-vector or all-scalar", MI);
1051 // Generally we try to report as many issues as possible at once, but in
1052 // this case it's not clear what should we be comparing the size of the
1053 // scalar with: the size of the whole vector or its lane. Instead of
1054 // making an arbitrary choice and emitting not so helpful message, let's
1055 // avoid the extra noise and stop here.
1056 return false;
1057 }
1058
1059 if (Ty0.isVector() && Ty0.getElementCount() != Ty1.getElementCount()) {
1060 report(msg: "operand types must preserve number of vector elements", MI);
1061 return false;
1062 }
1063
1064 return true;
1065}
1066
1067bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
1068 auto Opcode = MI->getOpcode();
1069 bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
1070 Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
1071 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1072 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1073 AttributeSet Attrs = Intrinsic::getFnAttributes(
1074 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1075 bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
1076 if (NoSideEffects && DeclHasSideEffects) {
1077 report(Msg: Twine(TII->getName(Opcode),
1078 " used with intrinsic that accesses memory"),
1079 MI);
1080 return false;
1081 }
1082 if (!NoSideEffects && !DeclHasSideEffects) {
1083 report(Msg: Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI);
1084 return false;
1085 }
1086 }
1087
1088 return true;
1089}
1090
1091bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
1092 auto Opcode = MI->getOpcode();
1093 bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
1094 Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
1095 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1096 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1097 AttributeSet Attrs = Intrinsic::getFnAttributes(
1098 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1099 bool DeclIsConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
1100 if (NotConvergent && DeclIsConvergent) {
1101 report(Msg: Twine(TII->getName(Opcode), " used with a convergent intrinsic"),
1102 MI);
1103 return false;
1104 }
1105 if (!NotConvergent && !DeclIsConvergent) {
1106 report(
1107 Msg: Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"),
1108 MI);
1109 return false;
1110 }
1111 }
1112
1113 return true;
1114}
1115
1116void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
1117 if (isFunctionSelected)
1118 report(msg: "Unexpected generic instruction in a Selected function", MI);
1119
1120 const MCInstrDesc &MCID = MI->getDesc();
1121 unsigned NumOps = MI->getNumOperands();
1122
1123 // Branches must reference a basic block if they are not indirect
1124 if (MI->isBranch() && !MI->isIndirectBranch()) {
1125 bool HasMBB = false;
1126 for (const MachineOperand &Op : MI->operands()) {
1127 if (Op.isMBB()) {
1128 HasMBB = true;
1129 break;
1130 }
1131 }
1132
1133 if (!HasMBB) {
1134 report(msg: "Branch instruction is missing a basic block operand or "
1135 "isIndirectBranch property",
1136 MI);
1137 }
1138 }
1139
1140 // Check types.
1141 SmallVector<LLT, 4> Types;
1142 for (unsigned I = 0, E = std::min(a: MCID.getNumOperands(), b: NumOps);
1143 I != E; ++I) {
1144 if (!MCID.operands()[I].isGenericType())
1145 continue;
1146 // Generic instructions specify type equality constraints between some of
1147 // their operands. Make sure these are consistent.
1148 size_t TypeIdx = MCID.operands()[I].getGenericTypeIndex();
1149 Types.resize(N: std::max(a: TypeIdx + 1, b: Types.size()));
1150
1151 const MachineOperand *MO = &MI->getOperand(i: I);
1152 if (!MO->isReg()) {
1153 report(msg: "generic instruction must use register operands", MI);
1154 continue;
1155 }
1156
1157 LLT OpTy = MRI->getType(Reg: MO->getReg());
1158 // Don't report a type mismatch if there is no actual mismatch, only a
1159 // type missing, to reduce noise:
1160 if (OpTy.isValid()) {
1161 // Only the first valid type for a type index will be printed: don't
1162 // overwrite it later so it's always clear which type was expected:
1163 if (!Types[TypeIdx].isValid())
1164 Types[TypeIdx] = OpTy;
1165 else if (Types[TypeIdx] != OpTy)
1166 report(msg: "Type mismatch in generic instruction", MO, MONum: I, MOVRegType: OpTy);
1167 } else {
1168 // Generic instructions must have types attached to their operands.
1169 report(msg: "Generic instruction is missing a virtual register type", MO, MONum: I);
1170 }
1171 }
1172
1173 // Generic opcodes must not have physical register operands.
1174 for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
1175 const MachineOperand *MO = &MI->getOperand(i: I);
1176 if (MO->isReg() && MO->getReg().isPhysical())
1177 report(msg: "Generic instruction cannot have physical register", MO, MONum: I);
1178 }
1179
1180 // Avoid out of bounds in checks below. This was already reported earlier.
1181 if (MI->getNumOperands() < MCID.getNumOperands())
1182 return;
1183
1184 StringRef ErrorInfo;
1185 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
1186 report(msg: ErrorInfo.data(), MI);
1187
1188 // Verify properties of various specific instruction types
1189 unsigned Opc = MI->getOpcode();
1190 switch (Opc) {
1191 case TargetOpcode::G_ASSERT_SEXT:
1192 case TargetOpcode::G_ASSERT_ZEXT: {
1193 std::string OpcName =
1194 Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT";
1195 if (!MI->getOperand(i: 2).isImm()) {
1196 report(Msg: Twine(OpcName, " expects an immediate operand #2"), MI);
1197 break;
1198 }
1199
1200 Register Dst = MI->getOperand(i: 0).getReg();
1201 Register Src = MI->getOperand(i: 1).getReg();
1202 LLT SrcTy = MRI->getType(Reg: Src);
1203 int64_t Imm = MI->getOperand(i: 2).getImm();
1204 if (Imm <= 0) {
1205 report(Msg: Twine(OpcName, " size must be >= 1"), MI);
1206 break;
1207 }
1208
1209 if (Imm >= SrcTy.getScalarSizeInBits()) {
1210 report(Msg: Twine(OpcName, " size must be less than source bit width"), MI);
1211 break;
1212 }
1213
1214 const RegisterBank *SrcRB = RBI->getRegBank(Reg: Src, MRI: *MRI, TRI: *TRI);
1215 const RegisterBank *DstRB = RBI->getRegBank(Reg: Dst, MRI: *MRI, TRI: *TRI);
1216
1217 // Allow only the source bank to be set.
1218 if ((SrcRB && DstRB && SrcRB != DstRB) || (DstRB && !SrcRB)) {
1219 report(Msg: Twine(OpcName, " cannot change register bank"), MI);
1220 break;
1221 }
1222
1223 // Don't allow a class change. Do allow member class->regbank.
1224 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: Dst);
1225 if (DstRC && DstRC != MRI->getRegClassOrNull(Reg: Src)) {
1226 report(
1227 Msg: Twine(OpcName, " source and destination register classes must match"),
1228 MI);
1229 break;
1230 }
1231
1232 break;
1233 }
1234
1235 case TargetOpcode::G_CONSTANT:
1236 case TargetOpcode::G_FCONSTANT: {
1237 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1238 if (DstTy.isVector())
1239 report(msg: "Instruction cannot use a vector result type", MI);
1240
1241 if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {
1242 if (!MI->getOperand(i: 1).isCImm()) {
1243 report(msg: "G_CONSTANT operand must be cimm", MI);
1244 break;
1245 }
1246
1247 const ConstantInt *CI = MI->getOperand(i: 1).getCImm();
1248 if (CI->getBitWidth() != DstTy.getSizeInBits())
1249 report(msg: "inconsistent constant size", MI);
1250 } else {
1251 if (!MI->getOperand(i: 1).isFPImm()) {
1252 report(msg: "G_FCONSTANT operand must be fpimm", MI);
1253 break;
1254 }
1255 const ConstantFP *CF = MI->getOperand(i: 1).getFPImm();
1256
1257 if (APFloat::getSizeInBits(Sem: CF->getValueAPF().getSemantics()) !=
1258 DstTy.getSizeInBits()) {
1259 report(msg: "inconsistent constant size", MI);
1260 }
1261 }
1262
1263 break;
1264 }
1265 case TargetOpcode::G_LOAD:
1266 case TargetOpcode::G_STORE:
1267 case TargetOpcode::G_ZEXTLOAD:
1268 case TargetOpcode::G_SEXTLOAD:
1269 case TargetOpcode::G_FPEXTLOAD:
1270 case TargetOpcode::G_FPTRUNCSTORE: {
1271 LLT ValTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1272 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1273 if (!PtrTy.isPointer())
1274 report(msg: "Generic memory instruction must access a pointer", MI);
1275
1276 // Generic loads and stores must have a single MachineMemOperand
1277 // describing that access.
1278 if (!MI->hasOneMemOperand()) {
1279 report(msg: "Generic instruction accessing memory must have one mem operand",
1280 MI);
1281 } else {
1282 const MachineMemOperand &MMO = **MI->memoperands_begin();
1283 if (isa<GExtLoad>(Val: *MI)) {
1284 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1285 RHS: ValTy.getSizeInBits()))
1286 report(msg: "Generic extload must have a narrower memory type", MI);
1287 } else if (isa<GFPTruncStore>(Val: *MI)) {
1288 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1289 RHS: ValTy.getSizeInBits()))
1290 report(msg: "Generic truncstore must have a narrower memory type", MI);
1291 } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
1292 if (TypeSize::isKnownGT(LHS: MMO.getSize().getValue(),
1293 RHS: ValTy.getSizeInBytes()))
1294 report(msg: "load memory size cannot exceed result size", MI);
1295
1296 if (MMO.getRanges()) {
1297 ConstantInt *i =
1298 mdconst::extract<ConstantInt>(MD: MMO.getRanges()->getOperand(I: 0));
1299 const LLT RangeTy = LLT::scalar(SizeInBits: i->getIntegerType()->getBitWidth());
1300 const LLT MemTy = MMO.getMemoryType();
1301 if (MemTy.getScalarType() != RangeTy ||
1302 ValTy.isScalar() != MemTy.isScalar() ||
1303 (ValTy.isVector() &&
1304 ValTy.getNumElements() != MemTy.getNumElements())) {
1305 report(msg: "range is incompatible with the result type", MI);
1306 }
1307 }
1308 } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
1309 if (TypeSize::isKnownLT(LHS: ValTy.getSizeInBytes(),
1310 RHS: MMO.getSize().getValue()))
1311 report(msg: "store memory size cannot exceed value size", MI);
1312 }
1313
1314 const AtomicOrdering Order = MMO.getSuccessOrdering();
1315 if (isa<GAnyStore>(Val: *MI)) {
1316 if (Order == AtomicOrdering::Acquire ||
1317 Order == AtomicOrdering::AcquireRelease)
1318 report(msg: "atomic store cannot use acquire ordering", MI);
1319
1320 } else {
1321 if (Order == AtomicOrdering::Release ||
1322 Order == AtomicOrdering::AcquireRelease)
1323 report(msg: "atomic load cannot use release ordering", MI);
1324 }
1325 }
1326
1327 break;
1328 }
1329 case TargetOpcode::G_PHI: {
1330 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1331 if (!DstTy.isValid() || !all_of(Range: drop_begin(RangeOrContainer: MI->operands()),
1332 P: [this, &DstTy](const MachineOperand &MO) {
1333 if (!MO.isReg())
1334 return true;
1335 LLT Ty = MRI->getType(Reg: MO.getReg());
1336 if (!Ty.isValid() || (Ty != DstTy))
1337 return false;
1338 return true;
1339 }))
1340 report(msg: "Generic Instruction G_PHI has operands with incompatible/missing "
1341 "types",
1342 MI);
1343 break;
1344 }
1345 case TargetOpcode::G_BITCAST: {
1346 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1347 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1348 if (!DstTy.isValid() || !SrcTy.isValid())
1349 break;
1350
1351 if (SrcTy.isPointer() != DstTy.isPointer())
1352 report(msg: "bitcast cannot convert between pointers and other types", MI);
1353
1354 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1355 report(msg: "bitcast sizes must match", MI);
1356
1357 bool SameType = SrcTy.getKind() == DstTy.getKind();
1358 if (SameType && SrcTy.isPointerOrPointerVector())
1359 SameType &= SrcTy.getAddressSpace() == DstTy.getAddressSpace();
1360
1361 SameType &= SrcTy.getScalarSizeInBits() == DstTy.getScalarSizeInBits();
1362
1363 if (SameType && SrcTy.isVector())
1364 SameType &= SrcTy.getElementCount() == DstTy.getElementCount();
1365 if (SameType && SrcTy.isFloatOrFloatVector())
1366 SameType &= SrcTy.getFpSemantics() == DstTy.getFpSemantics();
1367
1368 if (SameType)
1369 report(msg: "bitcast must change the type", MI);
1370
1371 break;
1372 }
1373 case TargetOpcode::G_INTTOPTR:
1374 case TargetOpcode::G_PTRTOINT:
1375 case TargetOpcode::G_ADDRSPACE_CAST: {
1376 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1377 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1378 if (!DstTy.isValid() || !SrcTy.isValid())
1379 break;
1380
1381 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1382
1383 DstTy = DstTy.getScalarType();
1384 SrcTy = SrcTy.getScalarType();
1385
1386 if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {
1387 if (!DstTy.isPointer())
1388 report(msg: "inttoptr result type must be a pointer", MI);
1389 if (SrcTy.isPointer())
1390 report(msg: "inttoptr source type must not be a pointer", MI);
1391 } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {
1392 if (!SrcTy.isPointer())
1393 report(msg: "ptrtoint source type must be a pointer", MI);
1394 if (DstTy.isPointer())
1395 report(msg: "ptrtoint result type must not be a pointer", MI);
1396 } else {
1397 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
1398 if (!SrcTy.isPointer() || !DstTy.isPointer())
1399 report(msg: "addrspacecast types must be pointers", MI);
1400 else {
1401 if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())
1402 report(msg: "addrspacecast must convert different address spaces", MI);
1403 }
1404 }
1405
1406 break;
1407 }
1408 case TargetOpcode::G_PTR_ADD: {
1409 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1410 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1411 LLT OffsetTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1412 if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())
1413 break;
1414
1415 if (!PtrTy.isPointerOrPointerVector())
1416 report(msg: "gep first operand must be a pointer", MI);
1417
1418 if (OffsetTy.isPointerOrPointerVector())
1419 report(msg: "gep offset operand must not be a pointer", MI);
1420
1421 if (PtrTy.isPointerOrPointerVector()) {
1422 const DataLayout &DL = MF->getDataLayout();
1423 unsigned AS = PtrTy.getAddressSpace();
1424 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
1425 if (OffsetTy.getScalarSizeInBits() != IndexSizeInBits) {
1426 report(msg: "gep offset operand must match index size for address space",
1427 MI);
1428 }
1429 }
1430
1431 // TODO: Is the offset allowed to be a scalar with a vector?
1432 break;
1433 }
1434 case TargetOpcode::G_PTRMASK: {
1435 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1436 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1437 LLT MaskTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1438 if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid())
1439 break;
1440
1441 if (!DstTy.isPointerOrPointerVector())
1442 report(msg: "ptrmask result type must be a pointer", MI);
1443
1444 if (!MaskTy.getScalarType().isScalar())
1445 report(msg: "ptrmask mask type must be an integer", MI);
1446
1447 verifyVectorElementMatch(Ty0: DstTy, Ty1: MaskTy, MI);
1448 break;
1449 }
1450 case TargetOpcode::G_SEXT:
1451 case TargetOpcode::G_ZEXT:
1452 case TargetOpcode::G_ANYEXT:
1453 case TargetOpcode::G_TRUNC:
1454 case TargetOpcode::G_TRUNC_SSAT_S:
1455 case TargetOpcode::G_TRUNC_SSAT_U:
1456 case TargetOpcode::G_TRUNC_USAT_U:
1457 case TargetOpcode::G_FPEXT:
1458 case TargetOpcode::G_FPTRUNC: {
1459 // Number of operands and presense of types is already checked (and
1460 // reported in case of any issues), so no need to report them again. As
1461 // we're trying to report as many issues as possible at once, however, the
1462 // instructions aren't guaranteed to have the right number of operands or
1463 // types attached to them at this point
1464 assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
1465 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1466 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1467 if (!DstTy.isValid() || !SrcTy.isValid())
1468 break;
1469
1470 if (DstTy.isPointerOrPointerVector() || SrcTy.isPointerOrPointerVector())
1471 report(msg: "Generic extend/truncate can not operate on pointers", MI);
1472
1473 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1474
1475 unsigned DstSize = DstTy.getScalarSizeInBits();
1476 unsigned SrcSize = SrcTy.getScalarSizeInBits();
1477 switch (MI->getOpcode()) {
1478 default:
1479 if (DstSize <= SrcSize)
1480 report(msg: "Generic extend has destination type no larger than source", MI);
1481 break;
1482 case TargetOpcode::G_TRUNC:
1483 case TargetOpcode::G_TRUNC_SSAT_S:
1484 case TargetOpcode::G_TRUNC_SSAT_U:
1485 case TargetOpcode::G_TRUNC_USAT_U:
1486 case TargetOpcode::G_FPTRUNC:
1487 if (DstSize >= SrcSize)
1488 report(msg: "Generic truncate has destination type no smaller than source",
1489 MI);
1490 break;
1491 }
1492 break;
1493 }
1494 case TargetOpcode::G_SELECT: {
1495 LLT SelTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1496 LLT CondTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1497 if (!SelTy.isValid() || !CondTy.isValid())
1498 break;
1499
1500 // Scalar condition select on a vector is valid.
1501 if (CondTy.isVector())
1502 verifyVectorElementMatch(Ty0: SelTy, Ty1: CondTy, MI);
1503 break;
1504 }
1505 case TargetOpcode::G_MERGE_VALUES: {
1506 // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,
1507 // e.g. s2N = MERGE sN, sN
1508 // Merging multiple scalars into a vector is not allowed, should use
1509 // G_BUILD_VECTOR for that.
1510 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1511 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1512 if (DstTy.isVector() || SrcTy.isVector())
1513 report(msg: "G_MERGE_VALUES cannot operate on vectors", MI);
1514
1515 const unsigned NumOps = MI->getNumOperands();
1516 if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1))
1517 report(msg: "G_MERGE_VALUES result size is inconsistent", MI);
1518
1519 for (unsigned I = 2; I != NumOps; ++I) {
1520 if (MRI->getType(Reg: MI->getOperand(i: I).getReg()) != SrcTy)
1521 report(msg: "G_MERGE_VALUES source types do not match", MI);
1522 }
1523
1524 break;
1525 }
1526 case TargetOpcode::G_UNMERGE_VALUES: {
1527 unsigned NumDsts = MI->getNumOperands() - 1;
1528 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1529 for (unsigned i = 1; i < NumDsts; ++i) {
1530 if (MRI->getType(Reg: MI->getOperand(i).getReg()) != DstTy) {
1531 report(msg: "G_UNMERGE_VALUES destination types do not match", MI);
1532 break;
1533 }
1534 }
1535
1536 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: NumDsts).getReg());
1537 if (DstTy.isVector()) {
1538 // This case is the converse of G_CONCAT_VECTORS.
1539 if (!SrcTy.isVector() ||
1540 (SrcTy.getScalarType() != DstTy.getScalarType() &&
1541 !SrcTy.isPointerVector()) ||
1542 SrcTy.isScalableVector() != DstTy.isScalableVector() ||
1543 SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1544 report(msg: "G_UNMERGE_VALUES source operand does not match vector "
1545 "destination operands",
1546 MI);
1547 } else if (SrcTy.isVector()) {
1548 // This case is the converse of G_BUILD_VECTOR, but relaxed to allow
1549 // mismatched types as long as the total size matches:
1550 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<4 x s32>)
1551 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1552 report(msg: "G_UNMERGE_VALUES vector source operand does not match scalar "
1553 "destination operands",
1554 MI);
1555 } else {
1556 // This case is the converse of G_MERGE_VALUES.
1557 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits()) {
1558 report(msg: "G_UNMERGE_VALUES scalar source operand does not match scalar "
1559 "destination operands",
1560 MI);
1561 }
1562 }
1563 break;
1564 }
1565 case TargetOpcode::G_BUILD_VECTOR: {
1566 // Source types must be scalars, dest type a vector. Total size of scalars
1567 // must match the dest vector size.
1568 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1569 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1570 if (!DstTy.isVector() || SrcEltTy.isVector()) {
1571 report(msg: "G_BUILD_VECTOR must produce a vector from scalar operands", MI);
1572 break;
1573 }
1574
1575 if (DstTy.getElementType() != SrcEltTy)
1576 report(msg: "G_BUILD_VECTOR result element type must match source type", MI);
1577
1578 if (DstTy.getNumElements() != MI->getNumOperands() - 1)
1579 report(msg: "G_BUILD_VECTOR must have an operand for each element", MI);
1580
1581 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1582 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1583 report(msg: "G_BUILD_VECTOR source operand types are not homogeneous", MI);
1584
1585 break;
1586 }
1587 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1588 // Source types must be scalars, dest type a vector. Scalar types must be
1589 // larger than the dest vector elt type, as this is a truncating operation.
1590 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1591 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1592 if (!DstTy.isVector() || SrcEltTy.isVector())
1593 report(msg: "G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands",
1594 MI);
1595 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1596 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1597 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not homogeneous",
1598 MI);
1599 if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits())
1600 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not larger than "
1601 "dest elt type",
1602 MI);
1603 break;
1604 }
1605 case TargetOpcode::G_CONCAT_VECTORS: {
1606 // Source types should be vectors, and total size should match the dest
1607 // vector size.
1608 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1609 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1610 if (!DstTy.isVector() || !SrcTy.isVector())
1611 report(msg: "G_CONCAT_VECTOR requires vector source and destination operands",
1612 MI);
1613
1614 if (MI->getNumOperands() < 3)
1615 report(msg: "G_CONCAT_VECTOR requires at least 2 source operands", MI);
1616
1617 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1618 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1619 report(msg: "G_CONCAT_VECTOR source operand types are not homogeneous", MI);
1620 if (DstTy.getElementCount() !=
1621 SrcTy.getElementCount() * (MI->getNumOperands() - 1))
1622 report(msg: "G_CONCAT_VECTOR num dest and source elements should match", MI);
1623 break;
1624 }
1625 case TargetOpcode::G_ICMP:
1626 case TargetOpcode::G_FCMP: {
1627 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1628 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1629
1630 if ((DstTy.isVector() != SrcTy.isVector()) ||
1631 (DstTy.isVector() &&
1632 DstTy.getElementCount() != SrcTy.getElementCount()))
1633 report(msg: "Generic vector icmp/fcmp must preserve number of lanes", MI);
1634
1635 break;
1636 }
1637 case TargetOpcode::G_SCMP:
1638 case TargetOpcode::G_UCMP: {
1639 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1640 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1641
1642 if (SrcTy.isPointerOrPointerVector()) {
1643 report(msg: "Generic scmp/ucmp does not support pointers as operands", MI);
1644 break;
1645 }
1646
1647 if (DstTy.isPointerOrPointerVector()) {
1648 report(msg: "Generic scmp/ucmp does not support pointers as a result", MI);
1649 break;
1650 }
1651
1652 if (DstTy.getScalarSizeInBits() < 2) {
1653 report(msg: "Result type must be at least 2 bits wide", MI);
1654 break;
1655 }
1656
1657 if ((DstTy.isVector() != SrcTy.isVector()) ||
1658 (DstTy.isVector() &&
1659 DstTy.getElementCount() != SrcTy.getElementCount())) {
1660 report(msg: "Generic vector scmp/ucmp must preserve number of lanes", MI);
1661 break;
1662 }
1663
1664 break;
1665 }
1666 case TargetOpcode::G_EXTRACT: {
1667 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1668 if (!SrcOp.isReg()) {
1669 report(msg: "extract source must be a register", MI);
1670 break;
1671 }
1672
1673 const MachineOperand &OffsetOp = MI->getOperand(i: 2);
1674 if (!OffsetOp.isImm()) {
1675 report(msg: "extract offset must be a constant", MI);
1676 break;
1677 }
1678
1679 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1680 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1681 if (SrcSize == DstSize)
1682 report(msg: "extract source must be larger than result", MI);
1683
1684 if (DstSize + OffsetOp.getImm() > SrcSize)
1685 report(msg: "extract reads past end of register", MI);
1686 break;
1687 }
1688 case TargetOpcode::G_INSERT: {
1689 const MachineOperand &SrcOp = MI->getOperand(i: 2);
1690 if (!SrcOp.isReg()) {
1691 report(msg: "insert source must be a register", MI);
1692 break;
1693 }
1694
1695 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
1696 if (!OffsetOp.isImm()) {
1697 report(msg: "insert offset must be a constant", MI);
1698 break;
1699 }
1700
1701 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1702 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1703
1704 if (DstSize <= SrcSize)
1705 report(msg: "inserted size must be smaller than total register", MI);
1706
1707 if (SrcSize + OffsetOp.getImm() > DstSize)
1708 report(msg: "insert writes past end of register", MI);
1709
1710 break;
1711 }
1712 case TargetOpcode::G_JUMP_TABLE: {
1713 if (!MI->getOperand(i: 1).isJTI())
1714 report(msg: "G_JUMP_TABLE source operand must be a jump table index", MI);
1715 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1716 if (!DstTy.isPointer())
1717 report(msg: "G_JUMP_TABLE dest operand must have a pointer type", MI);
1718 break;
1719 }
1720 case TargetOpcode::G_BRJT: {
1721 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
1722 report(msg: "G_BRJT src operand 0 must be a pointer type", MI);
1723
1724 if (!MI->getOperand(i: 1).isJTI())
1725 report(msg: "G_BRJT src operand 1 must be a jump table index", MI);
1726
1727 const auto &IdxOp = MI->getOperand(i: 2);
1728 if (!IdxOp.isReg() || MRI->getType(Reg: IdxOp.getReg()).isPointer())
1729 report(msg: "G_BRJT src operand 2 must be a scalar reg type", MI);
1730 break;
1731 }
1732 case TargetOpcode::G_INTRINSIC:
1733 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1734 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1735 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
1736 // TODO: Should verify number of def and use operands, but the current
1737 // interface requires passing in IR types for mangling.
1738 const MachineOperand &IntrIDOp = MI->getOperand(i: MI->getNumExplicitDefs());
1739 if (!IntrIDOp.isIntrinsicID()) {
1740 report(msg: "G_INTRINSIC first src operand must be an intrinsic ID", MI);
1741 break;
1742 }
1743
1744 if (!verifyGIntrinsicSideEffects(MI))
1745 break;
1746 if (!verifyGIntrinsicConvergence(MI))
1747 break;
1748
1749 break;
1750 }
1751 case TargetOpcode::G_SEXT_INREG: {
1752 if (!MI->getOperand(i: 2).isImm()) {
1753 report(msg: "G_SEXT_INREG expects an immediate operand #2", MI);
1754 break;
1755 }
1756
1757 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1758 int64_t Imm = MI->getOperand(i: 2).getImm();
1759 if (Imm <= 0)
1760 report(msg: "G_SEXT_INREG size must be >= 1", MI);
1761 if (Imm >= SrcTy.getScalarSizeInBits())
1762 report(msg: "G_SEXT_INREG size must be less than source bit width", MI);
1763 break;
1764 }
1765 case TargetOpcode::G_BSWAP: {
1766 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1767 if (DstTy.getScalarSizeInBits() % 16 != 0)
1768 report(msg: "G_BSWAP size must be a multiple of 16 bits", MI);
1769 break;
1770 }
1771 case TargetOpcode::G_VSCALE: {
1772 if (!MI->getOperand(i: 1).isCImm()) {
1773 report(msg: "G_VSCALE operand must be cimm", MI);
1774 break;
1775 }
1776 if (MI->getOperand(i: 1).getCImm()->isZero()) {
1777 report(msg: "G_VSCALE immediate cannot be zero", MI);
1778 break;
1779 }
1780 break;
1781 }
1782 case TargetOpcode::G_STEP_VECTOR: {
1783 if (!MI->getOperand(i: 1).isCImm()) {
1784 report(msg: "operand must be cimm", MI);
1785 break;
1786 }
1787
1788 if (!MI->getOperand(i: 1).getCImm()->getValue().isStrictlyPositive()) {
1789 report(msg: "step must be > 0", MI);
1790 break;
1791 }
1792
1793 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1794 if (!DstTy.isScalableVector()) {
1795 report(msg: "Destination type must be a scalable vector", MI);
1796 break;
1797 }
1798
1799 // <vscale x 2 x p0>
1800 if (!DstTy.getElementType().isScalar()) {
1801 report(msg: "Destination element type must be scalar", MI);
1802 break;
1803 }
1804
1805 if (MI->getOperand(i: 1).getCImm()->getBitWidth() !=
1806 DstTy.getElementType().getScalarSizeInBits()) {
1807 report(msg: "step bitwidth differs from result type element bitwidth", MI);
1808 break;
1809 }
1810 break;
1811 }
1812 case TargetOpcode::G_INSERT_SUBVECTOR: {
1813 const MachineOperand &Src0Op = MI->getOperand(i: 1);
1814 if (!Src0Op.isReg()) {
1815 report(msg: "G_INSERT_SUBVECTOR first source must be a register", MI);
1816 break;
1817 }
1818
1819 const MachineOperand &Src1Op = MI->getOperand(i: 2);
1820 if (!Src1Op.isReg()) {
1821 report(msg: "G_INSERT_SUBVECTOR second source must be a register", MI);
1822 break;
1823 }
1824
1825 const MachineOperand &IndexOp = MI->getOperand(i: 3);
1826 if (!IndexOp.isImm()) {
1827 report(msg: "G_INSERT_SUBVECTOR index must be an immediate", MI);
1828 break;
1829 }
1830
1831 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1832 LLT Src1Ty = MRI->getType(Reg: Src1Op.getReg());
1833
1834 if (!DstTy.isVector()) {
1835 report(msg: "Destination type must be a vector", MI);
1836 break;
1837 }
1838
1839 if (!Src1Ty.isVector()) {
1840 report(msg: "Second source must be a vector", MI);
1841 break;
1842 }
1843
1844 if (DstTy.getElementType() != Src1Ty.getElementType()) {
1845 report(msg: "Element type of vectors must be the same", MI);
1846 break;
1847 }
1848
1849 if (!DstTy.isScalable() && Src1Ty.isScalable()) {
1850 report(msg: "Cannot insert a scalable vector into a fixed length vector", MI);
1851 break;
1852 }
1853
1854 bool IsMixedFixedIntoScalable =
1855 DstTy.isScalableVector() && Src1Ty.isFixedVector();
1856
1857 if (!IsMixedFixedIntoScalable &&
1858 ElementCount::isKnownGT(LHS: Src1Ty.getElementCount(),
1859 RHS: DstTy.getElementCount())) {
1860 report(msg: "Second source must be smaller than destination vector", MI);
1861 break;
1862 }
1863
1864 uint64_t Idx = IndexOp.getImm();
1865 uint64_t Src1MinLen = Src1Ty.getElementCount().getKnownMinValue();
1866 if (IndexOp.getImm() % Src1MinLen != 0) {
1867 report(msg: "Index must be a multiple of the second source vector's "
1868 "minimum vector length",
1869 MI);
1870 break;
1871 }
1872
1873 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1874 if (Idx >= DstMinLen ||
1875 (!IsMixedFixedIntoScalable && Idx + Src1MinLen > DstMinLen)) {
1876 report(msg: "Subvector type and index must not cause insert to overrun the "
1877 "vector being inserted into",
1878 MI);
1879 break;
1880 }
1881
1882 break;
1883 }
1884 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1885 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1886 if (!SrcOp.isReg()) {
1887 report(msg: "G_EXTRACT_SUBVECTOR first source must be a register", MI);
1888 break;
1889 }
1890
1891 const MachineOperand &IndexOp = MI->getOperand(i: 2);
1892 if (!IndexOp.isImm()) {
1893 report(msg: "G_EXTRACT_SUBVECTOR index must be an immediate", MI);
1894 break;
1895 }
1896
1897 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1898 LLT SrcTy = MRI->getType(Reg: SrcOp.getReg());
1899
1900 if (!DstTy.isVector()) {
1901 report(msg: "Destination type must be a vector", MI);
1902 break;
1903 }
1904
1905 if (!SrcTy.isVector()) {
1906 report(msg: "Source must be a vector", MI);
1907 break;
1908 }
1909
1910 if (DstTy.getElementType() != SrcTy.getElementType()) {
1911 report(msg: "Element type of vectors must be the same", MI);
1912 break;
1913 }
1914
1915 if (DstTy.isScalable() && !SrcTy.isScalable()) {
1916 report(msg: "Cannot extract a scalable vector from a fixed length vector", MI);
1917 break;
1918 }
1919
1920 if (ElementCount::isKnownGT(LHS: DstTy.getElementCount(),
1921 RHS: SrcTy.getElementCount())) {
1922 report(msg: "Destination vector must be smaller than source vector", MI);
1923 break;
1924 }
1925
1926 uint64_t Idx = IndexOp.getImm();
1927 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1928 if (Idx % DstMinLen != 0) {
1929 report(msg: "Index must be a multiple of the destination vector's minimum "
1930 "vector length",
1931 MI);
1932 break;
1933 }
1934
1935 bool IsMixedFixedFromScalable =
1936 DstTy.isFixedVector() && SrcTy.isScalableVector();
1937 uint64_t SrcMinLen = SrcTy.getElementCount().getKnownMinValue();
1938 if (Idx >= SrcMinLen ||
1939 (!IsMixedFixedFromScalable && Idx + DstMinLen > SrcMinLen)) {
1940 report(msg: "Destination type and index must not cause extract to overrun the "
1941 "source vector",
1942 MI);
1943 break;
1944 }
1945
1946 break;
1947 }
1948 case TargetOpcode::G_SHUFFLE_VECTOR: {
1949 const MachineOperand &MaskOp = MI->getOperand(i: 3);
1950 if (!MaskOp.isShuffleMask()) {
1951 report(msg: "Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);
1952 break;
1953 }
1954
1955 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1956 LLT Src0Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1957 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1958
1959 if (Src0Ty != Src1Ty)
1960 report(msg: "Source operands must be the same type", MI);
1961
1962 if (Src0Ty.getScalarType() != DstTy.getScalarType()) {
1963 report(msg: "G_SHUFFLE_VECTOR cannot change element type", MI);
1964 break;
1965 }
1966 if (!Src0Ty.isVector()) {
1967 report(msg: "G_SHUFFLE_VECTOR must have vector src", MI);
1968 break;
1969 }
1970 if (!DstTy.isVector()) {
1971 report(msg: "G_SHUFFLE_VECTOR must have vector dst", MI);
1972 break;
1973 }
1974
1975 // Don't check that all operands are vector because scalars are used in
1976 // place of 1 element vectors.
1977 int SrcNumElts = Src0Ty.getNumElements();
1978 int DstNumElts = DstTy.getNumElements();
1979
1980 ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();
1981
1982 if (static_cast<int>(MaskIdxes.size()) != DstNumElts)
1983 report(msg: "Wrong result type for shufflemask", MI);
1984
1985 for (int Idx : MaskIdxes) {
1986 if (Idx < 0)
1987 continue;
1988
1989 if (Idx >= 2 * SrcNumElts)
1990 report(msg: "Out of bounds shuffle index", MI);
1991 }
1992
1993 break;
1994 }
1995
1996 case TargetOpcode::G_SPLAT_VECTOR: {
1997 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1998 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1999
2000 if (!DstTy.isVector()) {
2001 report(msg: "Destination type must be a vector", MI);
2002 break;
2003 }
2004
2005 if (!SrcTy.isScalar() && !SrcTy.isPointer()) {
2006 report(msg: "Source type must be a scalar or pointer", MI);
2007 break;
2008 }
2009
2010 if (TypeSize::isKnownGT(LHS: DstTy.getElementType().getSizeInBits(),
2011 RHS: SrcTy.getSizeInBits())) {
2012 report(msg: "Element type of the destination must be the same size or smaller "
2013 "than the source type",
2014 MI);
2015 break;
2016 }
2017
2018 break;
2019 }
2020 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2021 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2022 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2023 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2024
2025 if (!DstTy.isScalar() && !DstTy.isPointer()) {
2026 report(msg: "Destination type must be a scalar or pointer", MI);
2027 break;
2028 }
2029
2030 if (!SrcTy.isVector()) {
2031 report(msg: "First source must be a vector", MI);
2032 break;
2033 }
2034
2035 auto TLI = MF->getSubtarget().getTargetLowering();
2036 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2037 report(msg: "Index type must match VectorIdxTy", MI);
2038 break;
2039 }
2040
2041 break;
2042 }
2043 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2044 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2045 LLT VecTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2046 LLT ScaTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2047 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 3).getReg());
2048
2049 if (!DstTy.isVector()) {
2050 report(msg: "Destination type must be a vector", MI);
2051 break;
2052 }
2053
2054 if (VecTy != DstTy) {
2055 report(msg: "Destination type and vector type must match", MI);
2056 break;
2057 }
2058
2059 if (!ScaTy.isScalar() && !ScaTy.isPointer()) {
2060 report(msg: "Inserted element must be a scalar or pointer", MI);
2061 break;
2062 }
2063
2064 auto TLI = MF->getSubtarget().getTargetLowering();
2065 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2066 report(msg: "Index type must match VectorIdxTy", MI);
2067 break;
2068 }
2069
2070 break;
2071 }
2072 case TargetOpcode::G_DYN_STACKALLOC: {
2073 const MachineOperand &DstOp = MI->getOperand(i: 0);
2074 const MachineOperand &AllocOp = MI->getOperand(i: 1);
2075 const MachineOperand &AlignOp = MI->getOperand(i: 2);
2076
2077 if (!DstOp.isReg() || !MRI->getType(Reg: DstOp.getReg()).isPointer()) {
2078 report(msg: "dst operand 0 must be a pointer type", MI);
2079 break;
2080 }
2081
2082 if (!AllocOp.isReg() || !MRI->getType(Reg: AllocOp.getReg()).isScalar()) {
2083 report(msg: "src operand 1 must be a scalar reg type", MI);
2084 break;
2085 }
2086
2087 if (!AlignOp.isImm()) {
2088 report(msg: "src operand 2 must be an immediate type", MI);
2089 break;
2090 }
2091 break;
2092 }
2093 case TargetOpcode::G_MEMCPY_INLINE:
2094 case TargetOpcode::G_MEMCPY:
2095 case TargetOpcode::G_MEMMOVE: {
2096 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2097 if (MMOs.size() != 2) {
2098 report(msg: "memcpy/memmove must have 2 memory operands", MI);
2099 break;
2100 }
2101
2102 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) ||
2103 (MMOs[1]->isStore() || !MMOs[1]->isLoad())) {
2104 report(msg: "wrong memory operand types", MI);
2105 break;
2106 }
2107
2108 if (MMOs[0]->getSize() != MMOs[1]->getSize())
2109 report(msg: "inconsistent memory operand sizes", MI);
2110
2111 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2112 LLT SrcPtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2113
2114 if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) {
2115 report(msg: "memory instruction operand must be a pointer", MI);
2116 break;
2117 }
2118
2119 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2120 report(msg: "inconsistent store address space", MI);
2121 if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())
2122 report(msg: "inconsistent load address space", MI);
2123
2124 if (Opc != TargetOpcode::G_MEMCPY_INLINE)
2125 if (!MI->getOperand(i: 3).isImm() || (MI->getOperand(i: 3).getImm() & ~1LL))
2126 report(msg: "'tail' flag (operand 3) must be an immediate 0 or 1", MI);
2127
2128 break;
2129 }
2130 case TargetOpcode::G_BZERO:
2131 case TargetOpcode::G_MEMSET:
2132 case TargetOpcode::G_MEMSET_INLINE: {
2133 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2134 std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset"
2135 : Opc == TargetOpcode::G_MEMSET_INLINE ? "memset_inline"
2136 : "bzero";
2137 if (MMOs.size() != 1) {
2138 report(Msg: Twine(Name, " must have 1 memory operand"), MI);
2139 break;
2140 }
2141
2142 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
2143 report(Msg: Twine(Name, " memory operand must be a store"), MI);
2144 break;
2145 }
2146
2147 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2148 if (!DstPtrTy.isPointer()) {
2149 report(Msg: Twine(Name, " operand must be a pointer"), MI);
2150 break;
2151 }
2152
2153 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2154 report(Msg: "inconsistent " + Twine(Name, " address space"), MI);
2155
2156 if (Opc != TargetOpcode::G_MEMSET_INLINE) {
2157 if (!MI->getOperand(i: MI->getNumOperands() - 1).isImm() ||
2158 (MI->getOperand(i: MI->getNumOperands() - 1).getImm() & ~1LL))
2159 report(msg: "'tail' flag (last operand) must be an immediate 0 or 1", MI);
2160 }
2161
2162 break;
2163 }
2164 case TargetOpcode::G_UBSANTRAP: {
2165 const MachineOperand &KindOp = MI->getOperand(i: 0);
2166 if (!MI->getOperand(i: 0).isImm()) {
2167 report(msg: "Crash kind must be an immediate", MO: &KindOp, MONum: 0);
2168 break;
2169 }
2170 int64_t Kind = MI->getOperand(i: 0).getImm();
2171 if (!isInt<8>(x: Kind))
2172 report(msg: "Crash kind must be 8 bit wide", MO: &KindOp, MONum: 0);
2173 break;
2174 }
2175 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
2176 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
2177 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2178 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2179 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2180 if (!DstTy.isScalar())
2181 report(msg: "Vector reduction requires a scalar destination type", MI);
2182 if (!Src1Ty.isScalar())
2183 report(msg: "Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI);
2184 if (!Src2Ty.isVector())
2185 report(msg: "Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);
2186 break;
2187 }
2188 case TargetOpcode::G_VECREDUCE_FADD:
2189 case TargetOpcode::G_VECREDUCE_FMUL:
2190 case TargetOpcode::G_VECREDUCE_FMAX:
2191 case TargetOpcode::G_VECREDUCE_FMIN:
2192 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
2193 case TargetOpcode::G_VECREDUCE_FMINIMUM:
2194 case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
2195 case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
2196 case TargetOpcode::G_VECREDUCE_ADD:
2197 case TargetOpcode::G_VECREDUCE_MUL:
2198 case TargetOpcode::G_VECREDUCE_AND:
2199 case TargetOpcode::G_VECREDUCE_OR:
2200 case TargetOpcode::G_VECREDUCE_XOR:
2201 case TargetOpcode::G_VECREDUCE_SMAX:
2202 case TargetOpcode::G_VECREDUCE_SMIN:
2203 case TargetOpcode::G_VECREDUCE_UMAX:
2204 case TargetOpcode::G_VECREDUCE_UMIN: {
2205 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2206 if (!DstTy.isScalar())
2207 report(msg: "Vector reduction requires a scalar destination type", MI);
2208 break;
2209 }
2210
2211 case TargetOpcode::G_SBFX:
2212 case TargetOpcode::G_UBFX: {
2213 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2214 if (DstTy.isVector()) {
2215 report(msg: "Bitfield extraction is not supported on vectors", MI);
2216 break;
2217 }
2218 break;
2219 }
2220 case TargetOpcode::G_SHL:
2221 case TargetOpcode::G_LSHR:
2222 case TargetOpcode::G_ASHR:
2223 case TargetOpcode::G_ROTR:
2224 case TargetOpcode::G_ROTL: {
2225 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2226 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2227 if (Src1Ty.isVector() != Src2Ty.isVector()) {
2228 report(msg: "Shifts and rotates require operands to be either all scalars or "
2229 "all vectors",
2230 MI);
2231 break;
2232 }
2233 break;
2234 }
2235 case TargetOpcode::G_LLROUND:
2236 case TargetOpcode::G_LROUND: {
2237 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2238 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2239 if (!DstTy.isValid() || !SrcTy.isValid())
2240 break;
2241 if (SrcTy.isPointer() || DstTy.isPointer()) {
2242 StringRef Op = SrcTy.isPointer() ? "Source" : "Destination";
2243 report(Msg: Twine(Op, " operand must not be a pointer type"), MI);
2244 } else if (SrcTy.isScalar()) {
2245 verifyAllRegOpsScalar(MI: *MI, MRI: *MRI);
2246 break;
2247 } else if (SrcTy.isVector()) {
2248 verifyVectorElementMatch(Ty0: SrcTy, Ty1: DstTy, MI);
2249 break;
2250 }
2251 break;
2252 }
2253 case TargetOpcode::G_IS_FPCLASS: {
2254 LLT DestTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2255 LLT DestEltTy = DestTy.getScalarType();
2256 if (!DestEltTy.isScalar()) {
2257 report(msg: "Destination must be a scalar or vector of scalars", MI);
2258 break;
2259 }
2260 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2261 LLT SrcEltTy = SrcTy.getScalarType();
2262 if (!SrcEltTy.isScalar()) {
2263 report(msg: "Source must be a scalar or vector of scalars", MI);
2264 break;
2265 }
2266 if (!verifyVectorElementMatch(Ty0: DestTy, Ty1: SrcTy, MI))
2267 break;
2268 const MachineOperand &TestMO = MI->getOperand(i: 2);
2269 if (!TestMO.isImm()) {
2270 report(msg: "floating-point class set (operand 2) must be an immediate", MI);
2271 break;
2272 }
2273 int64_t Test = TestMO.getImm();
2274 if (Test < 0 || Test > fcAllFlags) {
2275 report(msg: "Incorrect floating-point class set (operand 2)", MI);
2276 break;
2277 }
2278 break;
2279 }
2280 case TargetOpcode::G_PREFETCH: {
2281 const MachineOperand &AddrOp = MI->getOperand(i: 0);
2282 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer()) {
2283 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 0);
2284 break;
2285 }
2286 const MachineOperand &RWOp = MI->getOperand(i: 1);
2287 if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {
2288 report(msg: "rw operand must be an immediate 0-1", MO: &RWOp, MONum: 1);
2289 break;
2290 }
2291 const MachineOperand &LocalityOp = MI->getOperand(i: 2);
2292 if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {
2293 report(msg: "locality operand must be an immediate 0-3", MO: &LocalityOp, MONum: 2);
2294 break;
2295 }
2296 const MachineOperand &CacheTypeOp = MI->getOperand(i: 3);
2297 if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {
2298 report(msg: "cache type operand must be an immediate 0-1", MO: &CacheTypeOp, MONum: 3);
2299 break;
2300 }
2301 break;
2302 }
2303 case TargetOpcode::G_ASSERT_ALIGN: {
2304 if (MI->getOperand(i: 2).getImm() < 1)
2305 report(msg: "alignment immediate must be >= 1", MI);
2306 break;
2307 }
2308 case TargetOpcode::G_CONSTANT_POOL: {
2309 if (!MI->getOperand(i: 1).isCPI())
2310 report(msg: "Src operand 1 must be a constant pool index", MI);
2311 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
2312 report(msg: "Dst operand 0 must be a pointer", MI);
2313 break;
2314 }
2315 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {
2316 const MachineOperand &AddrOp = MI->getOperand(i: 1);
2317 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer())
2318 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 1);
2319 break;
2320 }
2321 case TargetOpcode::G_SMIN:
2322 case TargetOpcode::G_SMAX:
2323 case TargetOpcode::G_UMIN:
2324 case TargetOpcode::G_UMAX: {
2325 const LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2326 if (DstTy.isPointerOrPointerVector())
2327 report(msg: "Generic smin/smax/umin/umax does not support pointer operands",
2328 MI);
2329 break;
2330 }
2331 default:
2332 break;
2333 }
2334}
2335
2336void MachineVerifier::visitMachineInstrAfter(const MachineInstr *MI) {
2337 // A dead def is contradictory if the same instruction has a live def of the
2338 // same register. Overlapping dead and live defs of different registers
2339 // describe a partially used def.
2340 if (!LivePhysDefs.empty()) {
2341 for (unsigned MONum : DeadPhysDefOps) {
2342 const MachineOperand &MO = MI->getOperand(i: MONum);
2343 if (is_contained(Range&: LivePhysDefs, Element: MO.getReg().asMCReg())) {
2344 report(msg: "Dead def has a live def of the same register on the same "
2345 "instruction",
2346 MO: &MO, MONum);
2347 }
2348 }
2349 }
2350
2351 LivePhysDefs.clear();
2352 DeadPhysDefOps.clear();
2353}
2354
2355void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
2356 const MCInstrDesc &MCID = MI->getDesc();
2357 if (MI->getNumOperands() < MCID.getNumOperands()) {
2358 report(msg: "Too few operands", MI);
2359 OS << MCID.getNumOperands() << " operands expected, but "
2360 << MI->getNumOperands() << " given.\n";
2361 }
2362
2363 if (MI->getFlag(Flag: MachineInstr::NoConvergent) && !MCID.isConvergent())
2364 report(msg: "NoConvergent flag expected only on convergent instructions.", MI);
2365
2366 if (MI->isPHI()) {
2367 if (MF->getProperties().hasNoPHIs())
2368 report(msg: "Found PHI instruction with NoPHIs property set", MI);
2369
2370 if (FirstNonPHI)
2371 report(msg: "Found PHI instruction after non-PHI", MI);
2372 } else if (FirstNonPHI == nullptr)
2373 FirstNonPHI = MI;
2374
2375 // Check the tied operands.
2376 if (MI->isInlineAsm())
2377 verifyInlineAsm(MI);
2378
2379 // Check that unspillable terminators define a reg and have at most one use.
2380 if (TII->isUnspillableTerminator(MI)) {
2381 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
2382 report(msg: "Unspillable Terminator does not define a reg", MI);
2383 Register Def = MI->getOperand(i: 0).getReg();
2384 if (Def.isVirtual() && hasPHIs(MF: *MF) &&
2385 std::distance(first: MRI->use_nodbg_begin(RegNo: Def), last: MRI->use_nodbg_end()) > 1)
2386 report(msg: "Unspillable Terminator expected to have at most one use!", MI);
2387 }
2388
2389 // A fully-formed DBG_VALUE must have a location. Ignore partially formed
2390 // DBG_VALUEs: these are convenient to use in tests, but should never get
2391 // generated.
2392 if (MI->isDebugValue() && MI->getNumOperands() == 4)
2393 if (!MI->getDebugLoc())
2394 report(msg: "Missing DebugLoc for debug instruction", MI);
2395
2396 // Meta instructions should never be the subject of debug value tracking,
2397 // they don't create a value in the output program at all.
2398 if (MI->isMetaInstruction() && MI->peekDebugInstrNum())
2399 report(msg: "Metadata instruction should not have a value tracking number", MI);
2400
2401 // Check the MachineMemOperands for basic consistency.
2402 for (MachineMemOperand *Op : MI->memoperands()) {
2403 if (Op->isLoad() && !MI->mayLoad())
2404 report(msg: "Missing mayLoad flag", MI);
2405 if (Op->isStore() && !MI->mayStore())
2406 report(msg: "Missing mayStore flag", MI);
2407 }
2408
2409 // Debug values must not have a slot index.
2410 // Other instructions must have one, unless they are inside a bundle.
2411 if (LiveInts) {
2412 bool mapped = !LiveInts->isNotInMIMap(Instr: *MI);
2413 if (MI->isDebugOrPseudoInstr()) {
2414 if (mapped)
2415 report(msg: "Debug instruction has a slot index", MI);
2416 } else if (MI->isInsideBundle()) {
2417 if (mapped)
2418 report(msg: "Instruction inside bundle has a slot index", MI);
2419 } else {
2420 if (!mapped)
2421 report(msg: "Missing slot index", MI);
2422 }
2423 }
2424
2425 unsigned Opc = MCID.getOpcode();
2426 if (isPreISelGenericOpcode(Opcode: Opc) || isPreISelGenericOptimizationHint(Opcode: Opc)) {
2427 verifyPreISelGenericInstruction(MI);
2428 return;
2429 }
2430
2431 StringRef ErrorInfo;
2432 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
2433 report(msg: ErrorInfo.data(), MI);
2434
2435 // Verify properties of various specific instruction types
2436 switch (MI->getOpcode()) {
2437 case TargetOpcode::COPY: {
2438 const MachineOperand &DstOp = MI->getOperand(i: 0);
2439 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2440 const Register SrcReg = SrcOp.getReg();
2441 const Register DstReg = DstOp.getReg();
2442
2443 LLT DstTy = MRI->getType(Reg: DstReg);
2444 LLT SrcTy = MRI->getType(Reg: SrcReg);
2445 if (SrcTy.isValid() && DstTy.isValid()) {
2446 // If both types are valid, check that the types are the same.
2447 if (SrcTy != DstTy) {
2448 report(msg: "Copy Instruction is illegal with mismatching types", MI);
2449 OS << "Def = " << DstTy << ", Src = " << SrcTy << '\n';
2450 }
2451
2452 break;
2453 }
2454
2455 if (!SrcTy.isValid() && !DstTy.isValid())
2456 break;
2457
2458 // If we have only one valid type, this is likely a copy between a virtual
2459 // and physical register.
2460 TypeSize SrcSize = TypeSize::getZero();
2461 TypeSize DstSize = TypeSize::getZero();
2462 if (SrcReg.isPhysical() && DstTy.isValid()) {
2463 if (!hasPhysRegClassForType(TRI: *TRI, Reg: SrcReg, Ty: DstTy))
2464 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2465 } else {
2466 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2467 }
2468
2469 if (DstReg.isPhysical() && SrcTy.isValid()) {
2470 if (!hasPhysRegClassForType(TRI: *TRI, Reg: DstReg, Ty: SrcTy))
2471 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2472 } else {
2473 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2474 }
2475
2476 // The next two checks allow COPY between physical and virtual registers,
2477 // when the virtual register has a scalable size and the physical register
2478 // has a fixed size. These checks allow COPY between *potentially*
2479 // mismatched sizes. However, once RegisterBankSelection occurs,
2480 // MachineVerifier should be able to resolve a fixed size for the scalable
2481 // vector, and at that point this function will know for sure whether the
2482 // sizes are mismatched and correctly report a size mismatch.
2483 if (SrcReg.isPhysical() && DstReg.isVirtual() && DstSize.isScalable() &&
2484 !SrcSize.isScalable())
2485 break;
2486 if (SrcReg.isVirtual() && DstReg.isPhysical() && SrcSize.isScalable() &&
2487 !DstSize.isScalable())
2488 break;
2489
2490 if (SrcSize.isNonZero() && DstSize.isNonZero() && SrcSize != DstSize) {
2491 if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {
2492 report(msg: "Copy Instruction is illegal with mismatching sizes", MI);
2493 OS << "Def Size = " << DstSize << ", Src Size = " << SrcSize << '\n';
2494 }
2495 }
2496 break;
2497 }
2498 case TargetOpcode::COPY_LANEMASK: {
2499 const MachineOperand &DstOp = MI->getOperand(i: 0);
2500 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2501 const MachineOperand &LaneMaskOp = MI->getOperand(i: 2);
2502 const Register SrcReg = SrcOp.getReg();
2503 const LaneBitmask LaneMask = LaneMaskOp.getLaneMask();
2504 LaneBitmask SrcMaxLaneMask = LaneBitmask::getAll();
2505
2506 if (DstOp.getSubReg())
2507 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &DstOp, MONum: 0);
2508
2509 if (SrcOp.getSubReg())
2510 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &SrcOp, MONum: 1);
2511
2512 if (LaneMask.none())
2513 report(msg: "COPY_LANEMASK must read at least one lane", MI);
2514
2515 if (SrcReg.isPhysical()) {
2516 const TargetRegisterClass *SrcRC = TRI->getMinimalPhysRegClass(Reg: SrcReg);
2517 if (SrcRC)
2518 SrcMaxLaneMask = SrcRC->getLaneMask();
2519 } else {
2520 SrcMaxLaneMask = MRI->getMaxLaneMaskForVReg(Reg: SrcReg);
2521 }
2522
2523 // COPY_LANEMASK should be used only for partial copy. For full
2524 // copy, one should strictly use the COPY instruction.
2525 if (SrcMaxLaneMask == LaneMask)
2526 report(msg: "COPY_LANEMASK cannot be used to do full copy", MI);
2527
2528 // If LaneMask is greater than the SrcMaxLaneMask, it implies
2529 // COPY_LANEMASK is attempting to read from the lanes that
2530 // don't exists in the source register.
2531 if (SrcMaxLaneMask < LaneMask)
2532 report(msg: "COPY_LANEMASK attempts to read from the lanes that "
2533 "don't exist in the source register",
2534 MI);
2535
2536 break;
2537 }
2538 case TargetOpcode::STATEPOINT: {
2539 StatepointOpers SO(MI);
2540 if (!MI->getOperand(i: SO.getIDPos()).isImm() ||
2541 !MI->getOperand(i: SO.getNBytesPos()).isImm() ||
2542 !MI->getOperand(i: SO.getNCallArgsPos()).isImm()) {
2543 report(msg: "meta operands to STATEPOINT not constant!", MI);
2544 break;
2545 }
2546
2547 auto VerifyStackMapConstant = [&](unsigned Offset) {
2548 if (Offset >= MI->getNumOperands()) {
2549 report(msg: "stack map constant to STATEPOINT is out of range!", MI);
2550 return;
2551 }
2552 if (!MI->getOperand(i: Offset - 1).isImm() ||
2553 MI->getOperand(i: Offset - 1).getImm() != StackMaps::ConstantOp ||
2554 !MI->getOperand(i: Offset).isImm())
2555 report(msg: "stack map constant to STATEPOINT not well formed!", MI);
2556 };
2557 VerifyStackMapConstant(SO.getCCIdx());
2558 VerifyStackMapConstant(SO.getFlagsIdx());
2559 VerifyStackMapConstant(SO.getNumDeoptArgsIdx());
2560 VerifyStackMapConstant(SO.getNumGCPtrIdx());
2561 VerifyStackMapConstant(SO.getNumAllocaIdx());
2562 VerifyStackMapConstant(SO.getNumGcMapEntriesIdx());
2563
2564 // Verify that all explicit statepoint defs are tied to gc operands as
2565 // they are expected to be a relocation of gc operands.
2566 unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx();
2567 unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2;
2568 for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) {
2569 unsigned UseOpIdx;
2570 if (!MI->isRegTiedToUseOperand(DefOpIdx: Idx, UseOpIdx: &UseOpIdx)) {
2571 report(msg: "STATEPOINT defs expected to be tied", MI);
2572 break;
2573 }
2574 if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) {
2575 report(msg: "STATEPOINT def tied to non-gc operand", MI);
2576 break;
2577 }
2578 }
2579
2580 // TODO: verify we have properly encoded deopt arguments
2581 } break;
2582 case TargetOpcode::INSERT_SUBREG: {
2583 unsigned InsertedSize;
2584 if (unsigned SubIdx = MI->getOperand(i: 2).getSubReg())
2585 InsertedSize = TRI->getSubRegIdxSize(Idx: SubIdx);
2586 else
2587 InsertedSize = TRI->getRegSizeInBits(Reg: MI->getOperand(i: 2).getReg(), MRI: *MRI);
2588 unsigned SubRegSize = TRI->getSubRegIdxSize(Idx: MI->getOperand(i: 3).getImm());
2589 if (SubRegSize < InsertedSize) {
2590 report(msg: "INSERT_SUBREG expected inserted value to have equal or lesser "
2591 "size than the subreg it was inserted into", MI);
2592 break;
2593 }
2594 } break;
2595 case TargetOpcode::REG_SEQUENCE: {
2596 unsigned NumOps = MI->getNumOperands();
2597 if (!(NumOps & 1)) {
2598 report(msg: "Invalid number of operands for REG_SEQUENCE", MI);
2599 break;
2600 }
2601
2602 for (unsigned I = 1; I != NumOps; I += 2) {
2603 const MachineOperand &RegOp = MI->getOperand(i: I);
2604 const MachineOperand &SubRegOp = MI->getOperand(i: I + 1);
2605
2606 if (!RegOp.isReg())
2607 report(msg: "Invalid register operand for REG_SEQUENCE", MO: &RegOp, MONum: I);
2608
2609 if (!SubRegOp.isImm() || SubRegOp.getImm() == 0 ||
2610 SubRegOp.getImm() >= TRI->getNumSubRegIndices()) {
2611 report(msg: "Invalid subregister index operand for REG_SEQUENCE",
2612 MO: &SubRegOp, MONum: I + 1);
2613 }
2614 }
2615
2616 Register DstReg = MI->getOperand(i: 0).getReg();
2617 if (DstReg.isPhysical())
2618 report(msg: "REG_SEQUENCE does not support physical register results", MI);
2619
2620 if (MI->getOperand(i: 0).getSubReg())
2621 report(msg: "Invalid subreg result for REG_SEQUENCE", MI);
2622
2623 break;
2624 }
2625 }
2626}
2627
2628void
2629MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
2630 const MachineInstr *MI = MO->getParent();
2631 const MCInstrDesc &MCID = MI->getDesc();
2632 unsigned NumDefs = MCID.getNumDefs();
2633 if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)
2634 NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;
2635
2636 // The first MCID.NumDefs operands must be explicit register defines
2637 if (MONum < NumDefs) {
2638 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2639 if (!MO->isReg())
2640 report(msg: "Explicit definition must be a register", MO, MONum);
2641 else if (!MO->isDef() && !MCOI.isOptionalDef())
2642 report(msg: "Explicit definition marked as use", MO, MONum);
2643 else if (MO->isImplicit())
2644 report(msg: "Explicit definition marked as implicit", MO, MONum);
2645 } else if (MONum < MCID.getNumOperands()) {
2646 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2647 // Don't check if it's the last operand in a variadic instruction. See,
2648 // e.g., LDM_RET in the arm back end. Check non-variadic operands only.
2649 bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1;
2650 if (!IsOptional) {
2651 if (MO->isReg()) {
2652 if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs())
2653 report(msg: "Explicit operand marked as def", MO, MONum);
2654 if (MO->isImplicit())
2655 report(msg: "Explicit operand marked as implicit", MO, MONum);
2656 }
2657
2658 // Check that an instruction has register operands only as expected.
2659 if (MCOI.OperandType == MCOI::OPERAND_REGISTER &&
2660 !MO->isReg() && !MO->isFI())
2661 report(msg: "Expected a register operand.", MO, MONum);
2662 if (MO->isReg()) {
2663 if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE ||
2664 (MCOI.OperandType == MCOI::OPERAND_PCREL &&
2665 !TII->isPCRelRegisterOperandLegal(MI: *MI, OpIdx: MONum)))
2666 report(msg: "Expected a non-register operand.", MO, MONum);
2667 }
2668 }
2669
2670 int TiedTo = MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::TIED_TO);
2671 if (TiedTo != -1) {
2672 if (!MO->isReg())
2673 report(msg: "Tied use must be a register", MO, MONum);
2674 else if (!MO->isTied())
2675 report(msg: "Operand should be tied", MO, MONum);
2676 else if (unsigned(TiedTo) != MI->findTiedOperandIdx(OpIdx: MONum))
2677 report(msg: "Tied def doesn't match MCInstrDesc", MO, MONum);
2678 else if (MO->getReg().isPhysical()) {
2679 const MachineOperand &MOTied = MI->getOperand(i: TiedTo);
2680 if (!MOTied.isReg())
2681 report(msg: "Tied counterpart must be a register", MO: &MOTied, MONum: TiedTo);
2682 else if (MOTied.getReg().isPhysical() &&
2683 MO->getReg() != MOTied.getReg())
2684 report(msg: "Tied physical registers must match.", MO: &MOTied, MONum: TiedTo);
2685 }
2686 } else if (MO->isReg() && MO->isTied())
2687 report(msg: "Explicit operand should not be tied", MO, MONum);
2688 } else if (!MI->isVariadic()) {
2689 // ARM adds %reg0 operands to indicate predicates. We'll allow that.
2690 if (!MO->isValidExcessOperand())
2691 report(msg: "Extra explicit operand on non-variadic instruction", MO, MONum);
2692 }
2693
2694 // Verify earlyClobber def operand
2695 if (MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::EARLY_CLOBBER) != -1) {
2696 if (!MO->isReg())
2697 report(msg: "Early clobber must be a register", MI);
2698 if (!MO->isEarlyClobber())
2699 report(msg: "Missing earlyClobber flag", MI);
2700 }
2701
2702 switch (MO->getType()) {
2703 case MachineOperand::MO_Register: {
2704 // Verify debug flag on debug instructions. Check this first because reg0
2705 // indicates an undefined debug value.
2706 if (MI->isDebugInstr() && MO->isUse()) {
2707 if (!MO->isDebug())
2708 report(msg: "Register operand must be marked debug", MO, MONum);
2709 } else if (MO->isDebug()) {
2710 report(msg: "Register operand must not be marked debug", MO, MONum);
2711 }
2712
2713 const Register Reg = MO->getReg();
2714 if (!Reg)
2715 return;
2716 if (MRI->tracksLiveness() && !MI->isDebugInstr())
2717 checkLiveness(MO, MONum);
2718
2719 if (MO->isDef() && MO->isUndef() && !MO->getSubReg() &&
2720 MO->getReg().isVirtual()) // TODO: Apply to physregs too
2721 report(msg: "Undef virtual register def operands require a subregister", MO, MONum);
2722
2723 // Verify the consistency of tied operands.
2724 if (MO->isTied()) {
2725 unsigned OtherIdx = MI->findTiedOperandIdx(OpIdx: MONum);
2726 const MachineOperand &OtherMO = MI->getOperand(i: OtherIdx);
2727 if (!OtherMO.isReg())
2728 report(msg: "Must be tied to a register", MO, MONum);
2729 if (!OtherMO.isTied())
2730 report(msg: "Missing tie flags on tied operand", MO, MONum);
2731 if (MI->findTiedOperandIdx(OpIdx: OtherIdx) != MONum)
2732 report(msg: "Inconsistent tie links", MO, MONum);
2733
2734 // See IsUndef in MachineOperand.h.
2735 if (MO->isUse() && MO->isUndef() && Reg.isVirtual() &&
2736 OtherMO.getReg() != Reg &&
2737 any_of(Range: MI->all_uses(), P: [&](const MachineOperand &Other) {
2738 return &Other != MO && Other.isUndef() && Other.getReg() == Reg &&
2739 Other.getSubReg() == MO->getSubReg();
2740 }))
2741 report(msg: "Tied undef use shares a virtual register with another read", MO,
2742 MONum);
2743
2744 if (MONum < MCID.getNumDefs()) {
2745 if (OtherIdx < MCID.getNumOperands()) {
2746 if (-1 == MCID.getOperandConstraint(OpNum: OtherIdx, Constraint: MCOI::TIED_TO))
2747 report(msg: "Explicit def tied to explicit use without tie constraint",
2748 MO, MONum);
2749 } else {
2750 if (!OtherMO.isImplicit())
2751 report(msg: "Explicit def should be tied to implicit use", MO, MONum);
2752 }
2753 }
2754 }
2755
2756 // Verify two-address constraints after the twoaddressinstruction pass.
2757 // Both twoaddressinstruction pass and phi-node-elimination pass call
2758 // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification
2759 // after twoaddressinstruction pass not after phi-node-elimination pass. So
2760 // we shouldn't use the IsSSA as the condition, we should based on
2761 // TiedOpsRewritten property to verify two-address constraints, this
2762 // property will be set in twoaddressinstruction pass.
2763 unsigned DefIdx;
2764 if (MF->getProperties().hasTiedOpsRewritten() && MO->isUse() &&
2765 MI->isRegTiedToDefOperand(UseOpIdx: MONum, DefOpIdx: &DefIdx) &&
2766 Reg != MI->getOperand(i: DefIdx).getReg())
2767 report(msg: "Two-address instruction operands must be identical", MO, MONum);
2768
2769 // Check register classes.
2770 unsigned SubIdx = MO->getSubReg();
2771
2772 if (Reg.isPhysical()) {
2773 if (SubIdx) {
2774 report(msg: "Illegal subregister index for physical register", MO, MONum);
2775 return;
2776 }
2777 if (MONum < MCID.getNumOperands()) {
2778 if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum)) {
2779 if (!DRC->contains(Reg)) {
2780 report(msg: "Illegal physical register for instruction", MO, MONum);
2781 OS << printReg(Reg, TRI) << " is not a "
2782 << TRI->getRegClassName(Class: DRC) << " register.\n";
2783 }
2784 }
2785 }
2786 if (MO->isRenamable()) {
2787 if (MRI->isReserved(PhysReg: Reg)) {
2788 report(msg: "isRenamable set on reserved register", MO, MONum);
2789 return;
2790 }
2791 }
2792 } else {
2793 // Virtual register.
2794 const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);
2795 if (!RC) {
2796 // This is a generic virtual register.
2797
2798 // Do not allow undef uses for generic virtual registers. This ensures
2799 // getVRegDef can never fail and return null on a generic register.
2800 //
2801 // FIXME: This restriction should probably be broadened to all SSA
2802 // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still
2803 // run on the SSA function just before phi elimination.
2804 if (MO->isUndef())
2805 report(msg: "Generic virtual register use cannot be undef", MO, MONum);
2806
2807 // Debug value instruction is permitted to use undefined vregs.
2808 // This is a performance measure to skip the overhead of immediately
2809 // pruning unused debug operands. The final undef substitution occurs
2810 // when debug values are allocated in LDVImpl::handleDebugValue, so
2811 // these verifications always apply after this pass.
2812 if (isFunctionTracksDebugUserValues || !MO->isUse() ||
2813 !MI->isDebugValue() || !MRI->def_empty(RegNo: Reg)) {
2814 // If we're post-Select, we can't have gvregs anymore.
2815 if (isFunctionSelected) {
2816 report(msg: "Generic virtual register invalid in a Selected function",
2817 MO, MONum);
2818 return;
2819 }
2820
2821 // The gvreg must have a type and it must not have a SubIdx.
2822 LLT Ty = MRI->getType(Reg);
2823 if (!Ty.isValid()) {
2824 report(msg: "Generic virtual register must have a valid type", MO,
2825 MONum);
2826 return;
2827 }
2828
2829 const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
2830 const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo();
2831
2832 // If we're post-RegBankSelect, the gvreg must have a bank.
2833 if (!RegBank && isFunctionRegBankSelected) {
2834 report(msg: "Generic virtual register must have a bank in a "
2835 "RegBankSelected function",
2836 MO, MONum);
2837 return;
2838 }
2839
2840 // Make sure the register fits into its register bank if any.
2841 if (RegBank && Ty.isValid() && !Ty.isScalableVector() &&
2842 RBI->getMaximumSize(RegBankID: RegBank->getID()) < Ty.getSizeInBits()) {
2843 report(msg: "Register bank is too small for virtual register", MO,
2844 MONum);
2845 OS << "Register bank " << RegBank->getName() << " too small("
2846 << RBI->getMaximumSize(RegBankID: RegBank->getID()) << ") to fit "
2847 << Ty.getSizeInBits() << "-bits\n";
2848 return;
2849 }
2850 }
2851
2852 if (SubIdx) {
2853 report(msg: "Generic virtual register does not allow subregister index", MO,
2854 MONum);
2855 return;
2856 }
2857
2858 // If this is a target specific instruction and this operand
2859 // has register class constraint, the virtual register must
2860 // comply to it.
2861 if (!isPreISelGenericOpcode(Opcode: MCID.getOpcode()) &&
2862 MONum < MCID.getNumOperands() && TII->getRegClass(MCID, OpNum: MONum)) {
2863 report(msg: "Virtual register does not match instruction constraint", MO,
2864 MONum);
2865 OS << "Expect register class "
2866 << TRI->getRegClassName(Class: TII->getRegClass(MCID, OpNum: MONum))
2867 << " but got nothing\n";
2868 return;
2869 }
2870
2871 break;
2872 }
2873 // Validate that SubIdx can be applied to the virtual register.
2874 if (!TRI->isSubRegValidForRegClass(RC, Idx: SubIdx)) {
2875 report(msg: "Invalid subregister index for virtual register", MO, MONum);
2876 OS << "Register class " << TRI->getRegClassName(Class: RC)
2877 << " does not support subreg index "
2878 << TRI->getSubRegIndexName(SubIdx) << '\n';
2879 return;
2880 }
2881 if (MONum >= MCID.getNumOperands())
2882 break;
2883 const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum);
2884 if (!DRC)
2885 break;
2886
2887 // If SubIdx is used, verify that RC with SubIdx can be used for an
2888 // operand of class DRC. This is valid if for every register in RC, the
2889 // register obtained by applying SubIdx to it is in DRC.
2890 if (SubIdx && TRI->getMatchingSuperRegClass(A: RC, B: DRC, Idx: SubIdx) != RC) {
2891 report(msg: "Illegal virtual register for instruction", MO, MONum);
2892 OS << TRI->getRegClassName(Class: RC) << "." << TRI->getSubRegIndexName(SubIdx)
2893 << " cannot be used for " << TRI->getRegClassName(Class: DRC)
2894 << " operands.";
2895 }
2896
2897 // If no SubIdx is used, verify that RC is a sub-class of DRC.
2898 if (!SubIdx && !RC->hasSuperClassEq(RC: DRC)) {
2899 report(msg: "Illegal virtual register for instruction", MO, MONum);
2900 OS << "Expected a " << TRI->getRegClassName(Class: DRC)
2901 << " register, but got a " << TRI->getRegClassName(Class: RC)
2902 << " register\n";
2903 }
2904 }
2905 break;
2906 }
2907
2908 case MachineOperand::MO_RegisterMask:
2909 regMasks.push_back(Elt: MO->getRegMask());
2910 break;
2911
2912 case MachineOperand::MO_MachineBasicBlock:
2913 if (MI->isPHI() && !MO->getMBB()->isSuccessor(MBB: MI->getParent()))
2914 report(msg: "PHI operand is not in the CFG", MO, MONum);
2915 break;
2916
2917 case MachineOperand::MO_FrameIndex:
2918 if (LiveStks && LiveStks->hasInterval(Slot: MO->getIndex()) &&
2919 LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
2920 int FI = MO->getIndex();
2921 LiveInterval &LI = LiveStks->getInterval(Slot: FI);
2922 SlotIndex Idx = LiveInts->getInstructionIndex(Instr: *MI);
2923
2924 bool MayStore = MI->mayStore();
2925 bool MayLoad = MI->mayLoad();
2926 // For a memory-to-memory move, we need to check if the frame
2927 // index is used for storing or loading, by inspecting the
2928 // memory operands.
2929 if (MayStore && MayLoad) {
2930 for (const MachineMemOperand *MMO : MI->memoperands()) {
2931 const auto *Value = dyn_cast_if_present<FixedStackPseudoSourceValue>(
2932 Val: MMO->getPseudoValue());
2933 if (!Value || Value->getFrameIndex() != FI)
2934 continue;
2935
2936 if (MMO->isStore())
2937 MayLoad = false;
2938 else
2939 MayStore = false;
2940 break;
2941 }
2942 if (MayLoad == MayStore)
2943 report(msg: "Missing fixed stack memoperand.", MI);
2944 }
2945 if (MayLoad && !LI.liveAt(index: Idx.getRegSlot(EC: true))) {
2946 report(msg: "Instruction loads from dead spill slot", MO, MONum);
2947 OS << "Live stack: " << LI << '\n';
2948 }
2949 if (MayStore && !LI.liveAt(index: Idx.getRegSlot())) {
2950 report(msg: "Instruction stores to dead spill slot", MO, MONum);
2951 OS << "Live stack: " << LI << '\n';
2952 }
2953 }
2954 break;
2955
2956 case MachineOperand::MO_CFIIndex:
2957 if (MO->getCFIIndex() >= MF->getFrameInstructions().size())
2958 report(msg: "CFI instruction has invalid index", MO, MONum);
2959 break;
2960
2961 default:
2962 break;
2963 }
2964}
2965
2966void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
2967 unsigned MONum, SlotIndex UseIdx,
2968 const LiveRange &LR,
2969 VirtRegOrUnit VRegOrUnit,
2970 LaneBitmask LaneMask) {
2971 const MachineInstr *MI = MO->getParent();
2972
2973 if (!LR.verify()) {
2974 report(msg: "invalid live range", MO, MONum);
2975 report_context_liverange(LR);
2976 report_context_vreg_regunit(VRegOrUnit);
2977 report_context(Pos: UseIdx);
2978 return;
2979 }
2980
2981 LiveQueryResult LRQ = LR.Query(Idx: UseIdx);
2982 bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut());
2983 // Check if we have a segment at the use, note however that we only need one
2984 // live subregister range, the others may be dead.
2985 if (!HasValue && LaneMask.none()) {
2986 report(msg: "No live segment at use", MO, MONum);
2987 report_context_liverange(LR);
2988 report_context_vreg_regunit(VRegOrUnit);
2989 report_context(Pos: UseIdx);
2990 }
2991 if (MO->isKill() && !LRQ.isKill()) {
2992 report(msg: "Live range continues after kill flag", MO, MONum);
2993 report_context_liverange(LR);
2994 report_context_vreg_regunit(VRegOrUnit);
2995 if (LaneMask.any())
2996 report_context_lanemask(LaneMask);
2997 report_context(Pos: UseIdx);
2998 }
2999}
3000
3001void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
3002 unsigned MONum, SlotIndex DefIdx,
3003 const LiveRange &LR,
3004 VirtRegOrUnit VRegOrUnit,
3005 bool SubRangeCheck,
3006 LaneBitmask LaneMask) {
3007 if (!LR.verify()) {
3008 report(msg: "invalid live range", MO, MONum);
3009 report_context_liverange(LR);
3010 report_context_vreg_regunit(VRegOrUnit);
3011 if (LaneMask.any())
3012 report_context_lanemask(LaneMask);
3013 report_context(Pos: DefIdx);
3014 }
3015
3016 if (const VNInfo *VNI = LR.getVNInfoAt(Idx: DefIdx)) {
3017 // The LR can correspond to the whole reg and its def slot is not obliged
3018 // to be the same as the MO' def slot. E.g. when we check here "normal"
3019 // subreg MO but there is other EC subreg MO in the same instruction so the
3020 // whole reg has EC def slot and differs from the currently checked MO' def
3021 // slot. For example:
3022 // %0 [16e,32r:0) 0@16e L..3 [16e,32r:0) 0@16e L..C [16r,32r:0) 0@16r
3023 // Check that there is an early-clobber def of the same superregister
3024 // somewhere is performed in visitMachineFunctionAfter()
3025 if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) ||
3026 !SlotIndex::isSameInstr(A: VNI->def, B: DefIdx) ||
3027 (VNI->def != DefIdx &&
3028 (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) {
3029 report(msg: "Inconsistent valno->def", MO, MONum);
3030 report_context_liverange(LR);
3031 report_context_vreg_regunit(VRegOrUnit);
3032 if (LaneMask.any())
3033 report_context_lanemask(LaneMask);
3034 report_context(VNI: *VNI);
3035 report_context(Pos: DefIdx);
3036 }
3037 } else {
3038 report(msg: "No live segment at def", MO, MONum);
3039 report_context_liverange(LR);
3040 report_context_vreg_regunit(VRegOrUnit);
3041 if (LaneMask.any())
3042 report_context_lanemask(LaneMask);
3043 report_context(Pos: DefIdx);
3044 }
3045 // Check that, if the dead def flag is present, LiveInts agree.
3046 if (MO->isDead()) {
3047 LiveQueryResult LRQ = LR.Query(Idx: DefIdx);
3048 if (!LRQ.isDeadDef()) {
3049 assert(VRegOrUnit.isVirtualReg() && "Expecting a virtual register.");
3050 // A dead subreg def only tells us that the specific subreg is dead. There
3051 // could be other non-dead defs of other subregs, or we could have other
3052 // parts of the register being live through the instruction. So unless we
3053 // are checking liveness for a subrange it is ok for the live range to
3054 // continue, given that we have a dead def of a subregister.
3055 if (SubRangeCheck || MO->getSubReg() == 0) {
3056 report(msg: "Live range continues after dead def flag", MO, MONum);
3057 report_context_liverange(LR);
3058 report_context_vreg_regunit(VRegOrUnit);
3059 if (LaneMask.any())
3060 report_context_lanemask(LaneMask);
3061 }
3062 }
3063 }
3064}
3065
3066void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
3067 const MachineInstr *MI = MO->getParent();
3068 const Register Reg = MO->getReg();
3069 const unsigned SubRegIdx = MO->getSubReg();
3070
3071 const LiveInterval *LI = nullptr;
3072 if (LiveInts && Reg.isVirtual()) {
3073 if (LiveInts->hasInterval(Reg)) {
3074 LI = &LiveInts->getInterval(Reg);
3075 if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&
3076 !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(VReg: Reg))
3077 report(msg: "Live interval for subreg operand has no subranges", MO, MONum);
3078 } else {
3079 report(msg: "Virtual register has no live interval", MO, MONum);
3080 }
3081 }
3082
3083 // Both use and def operands can read a register.
3084 if (MO->readsReg()) {
3085 if (MO->isKill())
3086 addRegWithSubRegs(RV&: regsKilled, Reg);
3087
3088 // Check LiveInts liveness and kill.
3089 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3090 SlotIndex UseIdx;
3091 if (MI->isPHI()) {
3092 // PHI use occurs on the edge, so check for live out here instead.
3093 UseIdx = LiveInts->getMBBEndIdx(
3094 mbb: MI->getOperand(i: MONum + 1).getMBB()).getPrevSlot();
3095 } else {
3096 UseIdx = LiveInts->getInstructionIndex(Instr: *MI);
3097 }
3098 // Check the cached regunit intervals.
3099 if (Reg.isPhysical() && !isReserved(Reg)) {
3100 for (MCRegUnit Unit : TRI->regunits(Reg: Reg.asMCReg())) {
3101 if (MRI->isReservedRegUnit(Unit))
3102 continue;
3103 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3104 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LR, VRegOrUnit: VirtRegOrUnit(Unit));
3105 }
3106 }
3107
3108 if (Reg.isVirtual()) {
3109 // This is a virtual register interval.
3110 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3111
3112 if (LI->hasSubRanges() && !MO->isDef()) {
3113 LaneBitmask MOMask = SubRegIdx != 0
3114 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3115 : MRI->getMaxLaneMaskForVReg(Reg);
3116 LaneBitmask LiveInMask;
3117 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3118 if ((MOMask & SR.LaneMask).none())
3119 continue;
3120 checkLivenessAtUse(MO, MONum, UseIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg),
3121 LaneMask: SR.LaneMask);
3122 LiveQueryResult LRQ = SR.Query(Idx: UseIdx);
3123 if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))
3124 LiveInMask |= SR.LaneMask;
3125 }
3126 // At least parts of the register has to be live at the use.
3127 if ((LiveInMask & MOMask).none()) {
3128 report(msg: "No live subrange at use", MO, MONum);
3129 report_context(LI: *LI);
3130 report_context(Pos: UseIdx);
3131 }
3132 // For PHIs all lanes should be live
3133 if (MI->isPHI() && LiveInMask != MOMask) {
3134 report(msg: "Not all lanes of PHI source live at use", MO, MONum);
3135 report_context(LI: *LI);
3136 report_context(Pos: UseIdx);
3137 }
3138 }
3139 }
3140 }
3141
3142 // Use of a dead register.
3143 if (!regsLive.count(V: Reg)) {
3144 if (Reg.isPhysical()) {
3145 // Reserved registers may be used even when 'dead'.
3146 bool Bad = !isReserved(Reg);
3147 // We are fine if just any subregister has a defined value.
3148 if (Bad) {
3149
3150 for (const MCPhysReg &SubReg : TRI->subregs(Reg)) {
3151 if (regsLive.count(V: SubReg)) {
3152 Bad = false;
3153 break;
3154 }
3155 }
3156 }
3157 // If there is an additional implicit-use of a super register we stop
3158 // here. By definition we are fine if the super register is not
3159 // (completely) dead, if the complete super register is dead we will
3160 // get a report for its operand.
3161 if (Bad) {
3162 for (const MachineOperand &MOP : MI->uses()) {
3163 if (!MOP.isReg() || !MOP.isImplicit())
3164 continue;
3165
3166 if (!MOP.getReg().isPhysical())
3167 continue;
3168
3169 if (MOP.getReg() != Reg &&
3170 all_of(Range: TRI->regunits(Reg), P: [&](const MCRegUnit RegUnit) {
3171 return llvm::is_contained(Range: TRI->regunits(Reg: MOP.getReg()),
3172 Element: RegUnit);
3173 }))
3174 Bad = false;
3175 }
3176 }
3177 if (Bad)
3178 report(msg: "Using an undefined physical register", MO, MONum);
3179 } else if (MRI->def_empty(RegNo: Reg)) {
3180 report(msg: "Reading virtual register without a def", MO, MONum);
3181 } else {
3182 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3183 // We don't know which virtual registers are live in, so only complain
3184 // if vreg was killed in this MBB. Otherwise keep track of vregs that
3185 // must be live in. PHI instructions are handled separately.
3186 if (MInfo.regsKilled.count(V: Reg))
3187 report(msg: "Using a killed virtual register", MO, MONum);
3188 else if (!MI->isPHI())
3189 MInfo.vregsLiveIn.insert(KV: std::make_pair(x: Reg, y&: MI));
3190 }
3191 }
3192 }
3193
3194 if (MO->isDef()) {
3195 // Register defined.
3196 // TODO: verify that earlyclobber ops are not used.
3197 if (MO->isDead())
3198 addRegWithSubRegs(RV&: regsDead, Reg);
3199 else
3200 addRegWithSubRegs(RV&: regsDefined, Reg);
3201
3202 if (Reg.isPhysical()) {
3203 if (MO->isDead())
3204 DeadPhysDefOps.push_back(Elt: MONum);
3205 else
3206 LivePhysDefs.push_back(Elt: Reg.asMCReg());
3207 }
3208
3209 // Verify SSA form.
3210 if (MRI->isSSA() && Reg.isVirtual()) {
3211 if (!MRI->hasOneDef(RegNo: Reg))
3212 report(msg: "Multiple virtual register defs in SSA form", MO, MONum);
3213 if (MO->getSubReg())
3214 report(msg: "Subreg def in SSA form", MO, MONum);
3215 }
3216
3217 // Check LiveInts for a live segment, but only for virtual registers.
3218 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3219 SlotIndex DefIdx = LiveInts->getInstructionIndex(Instr: *MI);
3220 DefIdx = DefIdx.getRegSlot(EC: MO->isEarlyClobber());
3221
3222 if (Reg.isVirtual()) {
3223 checkLivenessAtDef(MO, MONum, DefIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3224
3225 if (LI->hasSubRanges()) {
3226 LaneBitmask MOMask = SubRegIdx != 0
3227 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3228 : MRI->getMaxLaneMaskForVReg(Reg);
3229 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3230 if ((SR.LaneMask & MOMask).none())
3231 continue;
3232 checkLivenessAtDef(MO, MONum, DefIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg), SubRangeCheck: true,
3233 LaneMask: SR.LaneMask);
3234 }
3235 }
3236 }
3237 }
3238 }
3239}
3240
3241// This function gets called after visiting all instructions in a bundle. The
3242// argument points to the bundle header.
3243// Normal stand-alone instructions are also considered 'bundles', and this
3244// function is called for all of them.
3245void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {
3246 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3247 set_union(S1&: MInfo.regsKilled, S2: regsKilled);
3248 set_subtract(S1&: regsLive, S2: regsKilled); regsKilled.clear();
3249 // Kill any masked registers.
3250 while (!regMasks.empty()) {
3251 const uint32_t *Mask = regMasks.pop_back_val();
3252 for (Register Reg : regsLive)
3253 if (Reg.isPhysical() &&
3254 MachineOperand::clobbersPhysReg(RegMask: Mask, PhysReg: Reg.asMCReg()))
3255 regsDead.push_back(Elt: Reg);
3256 }
3257 set_subtract(S1&: regsLive, S2: regsDead); regsDead.clear();
3258 set_union(S1&: regsLive, S2: regsDefined); regsDefined.clear();
3259}
3260
3261void
3262MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {
3263 MBBInfoMap[MBB].regsLiveOut = regsLive;
3264 regsLive.clear();
3265
3266 if (Indexes) {
3267 SlotIndex stop = Indexes->getMBBEndIdx(mbb: MBB);
3268 if (!(stop > lastIndex)) {
3269 report(msg: "Block ends before last instruction index", MBB);
3270 OS << "Block ends at " << stop << " last instruction was at " << lastIndex
3271 << '\n';
3272 }
3273 lastIndex = stop;
3274 }
3275}
3276
3277namespace {
3278// This implements a set of registers that serves as a filter: can filter other
3279// sets by passing through elements not in the filter and blocking those that
3280// are. Any filter implicitly includes the full set of physical registers upon
3281// creation, thus filtering them all out. The filter itself as a set only grows,
3282// and needs to be as efficient as possible.
3283struct VRegFilter {
3284 // Add elements to the filter itself. \pre Input set \p FromRegSet must have
3285 // no duplicates. Both virtual and physical registers are fine.
3286 template <typename RegSetT> void add(const RegSetT &FromRegSet) {
3287 SmallVector<Register, 0> VRegsBuffer;
3288 filterAndAdd(FromRegSet, VRegsBuffer);
3289 }
3290 // Filter \p FromRegSet through the filter and append passed elements into \p
3291 // ToVRegs. All elements appended are then added to the filter itself.
3292 // \returns true if anything changed.
3293 template <typename RegSetT>
3294 bool filterAndAdd(const RegSetT &FromRegSet,
3295 SmallVectorImpl<Register> &ToVRegs) {
3296 unsigned SparseUniverse = Sparse.size();
3297 unsigned NewSparseUniverse = SparseUniverse;
3298 unsigned NewDenseSize = Dense.size();
3299 size_t Begin = ToVRegs.size();
3300 for (Register Reg : FromRegSet) {
3301 if (!Reg.isVirtual())
3302 continue;
3303 unsigned Index = Reg.virtRegIndex();
3304 if (Index < SparseUniverseMax) {
3305 if (Index < SparseUniverse && Sparse.test(Idx: Index))
3306 continue;
3307 NewSparseUniverse = std::max(a: NewSparseUniverse, b: Index + 1);
3308 } else {
3309 if (Dense.count(V: Reg))
3310 continue;
3311 ++NewDenseSize;
3312 }
3313 ToVRegs.push_back(Elt: Reg);
3314 }
3315 size_t End = ToVRegs.size();
3316 if (Begin == End)
3317 return false;
3318 // Reserving space in sets once performs better than doing so continuously
3319 // and pays easily for double look-ups (even in Dense with SparseUniverseMax
3320 // tuned all the way down) and double iteration (the second one is over a
3321 // SmallVector, which is a lot cheaper compared to DenseSet or BitVector).
3322 Sparse.resize(N: NewSparseUniverse);
3323 Dense.reserve(Size: NewDenseSize);
3324 for (unsigned I = Begin; I < End; ++I) {
3325 Register Reg = ToVRegs[I];
3326 unsigned Index = Reg.virtRegIndex();
3327 if (Index < SparseUniverseMax)
3328 Sparse.set(Index);
3329 else
3330 Dense.insert(V: Reg);
3331 }
3332 return true;
3333 }
3334
3335private:
3336 static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8;
3337 // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyond
3338 // are tracked by Dense. The only purpose of the threshold and the Dense set
3339 // is to have a reasonably growing memory usage in pathological cases (large
3340 // number of very sparse VRegFilter instances live at the same time). In
3341 // practice even in the worst-by-execution time cases having all elements
3342 // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more
3343 // space efficient than if tracked by Dense. The threshold is set to keep the
3344 // worst-case memory usage within 2x of figures determined empirically for
3345 // "all Dense" scenario in such worst-by-execution-time cases.
3346 BitVector Sparse;
3347 DenseSet<Register> Dense;
3348};
3349
3350// Implements both a transfer function and a (binary, in-place) join operator
3351// for a dataflow over register sets with set union join and filtering transfer
3352// (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.
3353// Maintains out_b as its state, allowing for O(n) iteration over it at any
3354// time, where n is the size of the set (as opposed to O(U) where U is the
3355// universe). filter_b implicitly contains all physical registers at all times.
3356class FilteringVRegSet {
3357 VRegFilter Filter;
3358 SmallVector<Register, 0> VRegs;
3359
3360public:
3361 // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.
3362 // Both virtual and physical registers are fine.
3363 template <typename RegSetT> void addToFilter(const RegSetT &RS) {
3364 Filter.add(RS);
3365 }
3366 // Passes \p RS through the filter_b (transfer function) and adds what's left
3367 // to itself (out_b).
3368 template <typename RegSetT> bool add(const RegSetT &RS) {
3369 // Double-duty the Filter: to maintain VRegs a set (and the join operation
3370 // a set union) just add everything being added here to the Filter as well.
3371 return Filter.filterAndAdd(RS, VRegs);
3372 }
3373 using const_iterator = decltype(VRegs)::const_iterator;
3374 const_iterator begin() const { return VRegs.begin(); }
3375 const_iterator end() const { return VRegs.end(); }
3376 size_t size() const { return VRegs.size(); }
3377};
3378} // namespace
3379
3380// Calculate the largest possible vregsPassed sets. These are the registers that
3381// can pass through an MBB live, but may not be live every time. It is assumed
3382// that all vregsPassed sets are empty before the call.
3383void MachineVerifier::calcRegsPassed() {
3384 if (MF->empty())
3385 // ReversePostOrderTraversal doesn't handle empty functions.
3386 return;
3387
3388 for (const MachineBasicBlock *MB :
3389 ReversePostOrderTraversal<const MachineFunction *>(MF)) {
3390 FilteringVRegSet VRegs;
3391 BBInfo &Info = MBBInfoMap[MB];
3392 assert(Info.reachable);
3393
3394 VRegs.addToFilter(RS: Info.regsKilled);
3395 VRegs.addToFilter(RS: Info.regsLiveOut);
3396 for (const MachineBasicBlock *Pred : MB->predecessors()) {
3397 const BBInfo &PredInfo = MBBInfoMap[Pred];
3398 if (!PredInfo.reachable)
3399 continue;
3400
3401 VRegs.add(RS: PredInfo.regsLiveOut);
3402 VRegs.add(RS: PredInfo.vregsPassed);
3403 }
3404 Info.vregsPassed.reserve(Size: VRegs.size());
3405 Info.vregsPassed.insert_range(R&: VRegs);
3406 }
3407}
3408
3409// Calculate the set of virtual registers that must be passed through each basic
3410// block in order to satisfy the requirements of successor blocks. This is very
3411// similar to calcRegsPassed, only backwards.
3412void MachineVerifier::calcRegsRequired() {
3413 // First push live-in regs to predecessors' vregsRequired.
3414 SmallPtrSet<const MachineBasicBlock*, 8> todo;
3415 for (const auto &MBB : *MF) {
3416 BBInfo &MInfo = MBBInfoMap[&MBB];
3417 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3418 BBInfo &PInfo = MBBInfoMap[Pred];
3419 if (PInfo.addRequired(RM: MInfo.vregsLiveIn))
3420 todo.insert(Ptr: Pred);
3421 }
3422
3423 // Handle the PHI node.
3424 for (const MachineInstr &MI : MBB.phis()) {
3425 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3426 // Skip those Operands which are undef regs or not regs.
3427 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())
3428 continue;
3429
3430 // Get register and predecessor for one PHI edge.
3431 Register Reg = MI.getOperand(i).getReg();
3432 const MachineBasicBlock *Pred = MI.getOperand(i: i + 1).getMBB();
3433
3434 BBInfo &PInfo = MBBInfoMap[Pred];
3435 if (PInfo.addRequired(Reg))
3436 todo.insert(Ptr: Pred);
3437 }
3438 }
3439 }
3440
3441 // Iteratively push vregsRequired to predecessors. This will converge to the
3442 // same final state regardless of DenseSet iteration order.
3443 while (!todo.empty()) {
3444 const MachineBasicBlock *MBB = *todo.begin();
3445 todo.erase(Ptr: MBB);
3446 BBInfo &MInfo = MBBInfoMap[MBB];
3447 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
3448 if (Pred == MBB)
3449 continue;
3450 BBInfo &SInfo = MBBInfoMap[Pred];
3451 if (SInfo.addRequired(RS: MInfo.vregsRequired))
3452 todo.insert(Ptr: Pred);
3453 }
3454 }
3455}
3456
3457// Check PHI instructions at the beginning of MBB. It is assumed that
3458// calcRegsPassed has been run so BBInfo::isLiveOut is valid.
3459void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {
3460 BBInfo &MInfo = MBBInfoMap[&MBB];
3461
3462 SmallPtrSet<const MachineBasicBlock*, 8> seen;
3463 for (const MachineInstr &Phi : MBB) {
3464 if (!Phi.isPHI())
3465 break;
3466 seen.clear();
3467
3468 const MachineOperand &MODef = Phi.getOperand(i: 0);
3469 if (!MODef.isReg() || !MODef.isDef()) {
3470 report(msg: "Expected first PHI operand to be a register def", MO: &MODef, MONum: 0);
3471 continue;
3472 }
3473 if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||
3474 MODef.isEarlyClobber() || MODef.isDebug())
3475 report(msg: "Unexpected flag on PHI operand", MO: &MODef, MONum: 0);
3476 Register DefReg = MODef.getReg();
3477 if (!DefReg.isVirtual())
3478 report(msg: "Expected first PHI operand to be a virtual register", MO: &MODef, MONum: 0);
3479
3480 for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {
3481 const MachineOperand &MO0 = Phi.getOperand(i: I);
3482 if (!MO0.isReg()) {
3483 report(msg: "Expected PHI operand to be a register", MO: &MO0, MONum: I);
3484 continue;
3485 }
3486 if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||
3487 MO0.isDebug() || MO0.isTied())
3488 report(msg: "Unexpected flag on PHI operand", MO: &MO0, MONum: I);
3489
3490 const MachineOperand &MO1 = Phi.getOperand(i: I + 1);
3491 if (!MO1.isMBB()) {
3492 report(msg: "Expected PHI operand to be a basic block", MO: &MO1, MONum: I + 1);
3493 continue;
3494 }
3495
3496 const MachineBasicBlock &Pre = *MO1.getMBB();
3497 if (!Pre.isSuccessor(MBB: &MBB)) {
3498 report(msg: "PHI input is not a predecessor block", MO: &MO1, MONum: I + 1);
3499 continue;
3500 }
3501
3502 if (MInfo.reachable) {
3503 seen.insert(Ptr: &Pre);
3504 BBInfo &PrInfo = MBBInfoMap[&Pre];
3505 if (!MO0.isUndef() && PrInfo.reachable &&
3506 !PrInfo.isLiveOut(Reg: MO0.getReg()))
3507 report(msg: "PHI operand is not live-out from predecessor", MO: &MO0, MONum: I);
3508 }
3509 }
3510
3511 // Did we see all predecessors?
3512 if (MInfo.reachable) {
3513 for (MachineBasicBlock *Pred : MBB.predecessors()) {
3514 if (!seen.count(Ptr: Pred)) {
3515 report(msg: "Missing PHI operand", MI: &Phi);
3516 OS << printMBBReference(MBB: *Pred)
3517 << " is a predecessor according to the CFG.\n";
3518 }
3519 }
3520 }
3521 }
3522}
3523
3524static void
3525verifyConvergenceControl(const MachineFunction &MF, MachineDominatorTree &DT,
3526 std::function<void(const Twine &Message)> FailureCB,
3527 raw_ostream &OS) {
3528 MachineConvergenceVerifier CV;
3529 CV.initialize(OS: &OS, FailureCB, F: MF);
3530
3531 for (const auto &MBB : MF) {
3532 CV.visit(BB: MBB);
3533 for (const auto &MI : MBB.instrs())
3534 CV.visit(I: MI);
3535 }
3536
3537 if (CV.sawTokens()) {
3538 DT.recalculate(Func&: const_cast<MachineFunction &>(MF));
3539 CV.verify(DT);
3540 }
3541}
3542
3543void MachineVerifier::visitMachineFunctionAfter() {
3544 auto FailureCB = [this](const Twine &Message) {
3545 report(msg: Message.str().c_str(), MF);
3546 };
3547 verifyConvergenceControl(MF: *MF, DT, FailureCB, OS);
3548
3549 calcRegsPassed();
3550
3551 for (const MachineBasicBlock &MBB : *MF)
3552 checkPHIOps(MBB);
3553
3554 // Now check liveness info if available
3555 calcRegsRequired();
3556
3557 // Check for killed virtual registers that should be live out.
3558 for (const auto &MBB : *MF) {
3559 BBInfo &MInfo = MBBInfoMap[&MBB];
3560 for (Register VReg : MInfo.vregsRequired)
3561 if (MInfo.regsKilled.count(V: VReg)) {
3562 report(msg: "Virtual register killed in block, but needed live out.", MBB: &MBB);
3563 OS << "Virtual register " << printReg(Reg: VReg)
3564 << " is used after the block.\n";
3565 }
3566 }
3567
3568 if (!MF->empty()) {
3569 BBInfo &MInfo = MBBInfoMap[&MF->front()];
3570 for (Register VReg : MInfo.vregsRequired) {
3571 report(msg: "Virtual register defs don't dominate all uses.", MF);
3572 report_context_vreg(VReg);
3573 }
3574 }
3575
3576 if (LiveInts)
3577 verifyLiveIntervals();
3578
3579 // Check live-in list of each MBB. If a register is live into MBB, check
3580 // that the register is in regsLiveOut of each predecessor block. Since
3581 // this must come from a definition in the predecessor or its live-in
3582 // list, this will catch a live-through case where the predecessor does not
3583 // have the register in its live-in list. This currently only checks
3584 // registers that have no aliases, are not allocatable and are not
3585 // reserved, which could mean a condition code register for instance.
3586 if (MRI->tracksLiveness())
3587 for (const auto &MBB : *MF)
3588 for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) {
3589 MCRegister LiveInReg = P.PhysReg;
3590 bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid();
3591 if (hasAliases || isAllocatable(Reg: LiveInReg) || isReserved(Reg: LiveInReg))
3592 continue;
3593 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3594 BBInfo &PInfo = MBBInfoMap[Pred];
3595 if (!PInfo.regsLiveOut.count(V: LiveInReg)) {
3596 report(msg: "Live in register not found to be live out from predecessor.",
3597 MBB: &MBB);
3598 OS << TRI->getName(RegNo: LiveInReg) << " not found to be live out from "
3599 << printMBBReference(MBB: *Pred) << '\n';
3600 }
3601 }
3602 }
3603
3604 for (auto CSInfo : MF->getCallSitesInfo())
3605 if (!CSInfo.first->isCall())
3606 report(msg: "Call site info referencing instruction that is not call", MF);
3607
3608 // If there's debug-info, check that we don't have any duplicate value
3609 // tracking numbers.
3610 if (MF->getFunction().getSubprogram()) {
3611 DenseSet<unsigned> SeenNumbers;
3612 for (const auto &MBB : *MF) {
3613 for (const auto &MI : MBB) {
3614 if (auto Num = MI.peekDebugInstrNum()) {
3615 auto Result = SeenNumbers.insert(V: (unsigned)Num);
3616 if (!Result.second)
3617 report(msg: "Instruction has a duplicated value tracking number", MI: &MI);
3618 }
3619 }
3620 }
3621 }
3622}
3623
3624void MachineVerifier::verifyLiveIntervals() {
3625 assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
3626 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3627 Register Reg = Register::index2VirtReg(Index: I);
3628
3629 // Spilling and splitting may leave unused registers around. Skip them.
3630 if (MRI->reg_nodbg_empty(RegNo: Reg))
3631 continue;
3632
3633 if (!LiveInts->hasInterval(Reg)) {
3634 report(msg: "Missing live interval for virtual register", MF);
3635 OS << printReg(Reg, TRI) << " still has defs or uses\n";
3636 continue;
3637 }
3638
3639 const LiveInterval &LI = LiveInts->getInterval(Reg);
3640 assert(Reg == LI.reg() && "Invalid reg to interval mapping");
3641 verifyLiveInterval(LI);
3642 }
3643
3644 // Verify all the cached regunit intervals.
3645 for (MCRegUnit Unit : TRI->regunits())
3646 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3647 verifyLiveRange(*LR, VirtRegOrUnit(Unit));
3648}
3649
3650void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
3651 const VNInfo *VNI,
3652 VirtRegOrUnit VRegOrUnit,
3653 LaneBitmask LaneMask) {
3654 if (VNI->isUnused())
3655 return;
3656
3657 const VNInfo *DefVNI = LR.getVNInfoAt(Idx: VNI->def);
3658
3659 if (!DefVNI) {
3660 report(msg: "Value not live at VNInfo def and not marked unused", MF);
3661 report_context(LR, VRegOrUnit, LaneMask);
3662 report_context(VNI: *VNI);
3663 return;
3664 }
3665
3666 if (DefVNI != VNI) {
3667 report(msg: "Live segment at def has different VNInfo", MF);
3668 report_context(LR, VRegOrUnit, LaneMask);
3669 report_context(VNI: *VNI);
3670 return;
3671 }
3672
3673 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: VNI->def);
3674 if (!MBB) {
3675 report(msg: "Invalid VNInfo definition index", MF);
3676 report_context(LR, VRegOrUnit, LaneMask);
3677 report_context(VNI: *VNI);
3678 return;
3679 }
3680
3681 if (VNI->isPHIDef()) {
3682 if (VNI->def != LiveInts->getMBBStartIdx(mbb: MBB)) {
3683 report(msg: "PHIDef VNInfo is not defined at MBB start", MBB);
3684 report_context(LR, VRegOrUnit, LaneMask);
3685 report_context(VNI: *VNI);
3686 }
3687 return;
3688 }
3689
3690 // Non-PHI def.
3691 const MachineInstr *MI = LiveInts->getInstructionFromIndex(index: VNI->def);
3692 if (!MI) {
3693 report(msg: "No instruction at VNInfo def index", MBB);
3694 report_context(LR, VRegOrUnit, LaneMask);
3695 report_context(VNI: *VNI);
3696 return;
3697 }
3698
3699 bool hasDef = false;
3700 bool isEarlyClobber = false;
3701 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3702 if (!MOI->isReg() || !MOI->isDef())
3703 continue;
3704 if (VRegOrUnit.isVirtualReg()) {
3705 if (MOI->getReg() != VRegOrUnit.asVirtualReg())
3706 continue;
3707 } else {
3708 if (!MOI->getReg().isPhysical() ||
3709 !TRI->hasRegUnit(Reg: MOI->getReg(), RegUnit: VRegOrUnit.asMCRegUnit()))
3710 continue;
3711 }
3712 if (LaneMask.any() &&
3713 (TRI->getSubRegIndexLaneMask(SubIdx: MOI->getSubReg()) & LaneMask).none())
3714 continue;
3715 hasDef = true;
3716 if (MOI->isEarlyClobber())
3717 isEarlyClobber = true;
3718 }
3719
3720 if (!hasDef) {
3721 report(msg: "Defining instruction does not modify register", MI);
3722 report_context(LR, VRegOrUnit, LaneMask);
3723 report_context(VNI: *VNI);
3724 }
3725
3726 // Early clobber defs begin at USE slots, but other defs must begin at
3727 // DEF slots.
3728 if (isEarlyClobber) {
3729 if (!VNI->def.isEarlyClobber()) {
3730 report(msg: "Early clobber def must be at an early-clobber slot", MBB);
3731 report_context(LR, VRegOrUnit, LaneMask);
3732 report_context(VNI: *VNI);
3733 }
3734 } else if (!VNI->def.isRegister()) {
3735 report(msg: "Non-PHI, non-early clobber def must be at a register slot", MBB);
3736 report_context(LR, VRegOrUnit, LaneMask);
3737 report_context(VNI: *VNI);
3738 }
3739}
3740
3741void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
3742 const LiveRange::const_iterator I,
3743 VirtRegOrUnit VRegOrUnit,
3744 LaneBitmask LaneMask) {
3745 const LiveRange::Segment &S = *I;
3746 const VNInfo *VNI = S.valno;
3747 assert(VNI && "Live segment has no valno");
3748
3749 if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(ValNo: VNI->id)) {
3750 report(msg: "Foreign valno in live segment", MF);
3751 report_context(LR, VRegOrUnit, LaneMask);
3752 report_context(S);
3753 report_context(VNI: *VNI);
3754 }
3755
3756 if (VNI->isUnused()) {
3757 report(msg: "Live segment valno is marked unused", MF);
3758 report_context(LR, VRegOrUnit, LaneMask);
3759 report_context(S);
3760 }
3761
3762 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: S.start);
3763 if (!MBB) {
3764 report(msg: "Bad start of live segment, no basic block", MF);
3765 report_context(LR, VRegOrUnit, LaneMask);
3766 report_context(S);
3767 return;
3768 }
3769 SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(mbb: MBB);
3770 if (S.start != MBBStartIdx && S.start != VNI->def) {
3771 report(msg: "Live segment must begin at MBB entry or valno def", MBB);
3772 report_context(LR, VRegOrUnit, LaneMask);
3773 report_context(S);
3774 }
3775
3776 const MachineBasicBlock *EndMBB =
3777 LiveInts->getMBBFromIndex(index: S.end.getPrevSlot());
3778 if (!EndMBB) {
3779 report(msg: "Bad end of live segment, no basic block", MF);
3780 report_context(LR, VRegOrUnit, LaneMask);
3781 report_context(S);
3782 return;
3783 }
3784
3785 // Checks for non-live-out segments.
3786 if (S.end != LiveInts->getMBBEndIdx(mbb: EndMBB)) {
3787 // RegUnit intervals are allowed dead phis.
3788 if (!VRegOrUnit.isVirtualReg() && VNI->isPHIDef() && S.start == VNI->def &&
3789 S.end == VNI->def.getDeadSlot())
3790 return;
3791
3792 // The live segment is ending inside EndMBB
3793 const MachineInstr *MI =
3794 LiveInts->getInstructionFromIndex(index: S.end.getPrevSlot());
3795 if (!MI) {
3796 report(msg: "Live segment doesn't end at a valid instruction", MBB: EndMBB);
3797 report_context(LR, VRegOrUnit, LaneMask);
3798 report_context(S);
3799 return;
3800 }
3801
3802 // The block slot must refer to a basic block boundary.
3803 if (S.end.isBlock()) {
3804 report(msg: "Live segment ends at B slot of an instruction", MBB: EndMBB);
3805 report_context(LR, VRegOrUnit, LaneMask);
3806 report_context(S);
3807 }
3808
3809 if (S.end.isDead()) {
3810 // Segment ends on the dead slot.
3811 // That means there must be a dead def.
3812 if (!SlotIndex::isSameInstr(A: S.start, B: S.end)) {
3813 report(msg: "Live segment ending at dead slot spans instructions", MBB: EndMBB);
3814 report_context(LR, VRegOrUnit, LaneMask);
3815 report_context(S);
3816 }
3817 }
3818
3819 // After tied operands are rewritten, a live segment can only end at an
3820 // early-clobber slot if it is being redefined by an early-clobber def.
3821 // TODO: Before tied operands are rewritten, a live segment can only end at
3822 // an early-clobber slot if the last use is tied to an early-clobber def.
3823 if (MF->getProperties().hasTiedOpsRewritten() && S.end.isEarlyClobber()) {
3824 if (I + 1 == LR.end() || (I + 1)->start != S.end) {
3825 report(msg: "Live segment ending at early clobber slot must be "
3826 "redefined by an EC def in the same instruction",
3827 MBB: EndMBB);
3828 report_context(LR, VRegOrUnit, LaneMask);
3829 report_context(S);
3830 }
3831 }
3832
3833 // The following checks only apply to virtual registers. Physreg liveness
3834 // is too weird to check.
3835 if (VRegOrUnit.isVirtualReg()) {
3836 // A live segment can end with either a redefinition, a kill flag on a
3837 // use, or a dead flag on a def.
3838 bool hasRead = false;
3839 bool hasSubRegDef = false;
3840 bool hasDeadDef = false;
3841 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3842 if (!MOI->isReg() || MOI->getReg() != VRegOrUnit.asVirtualReg())
3843 continue;
3844 unsigned Sub = MOI->getSubReg();
3845 LaneBitmask SLM =
3846 Sub != 0 ? TRI->getSubRegIndexLaneMask(SubIdx: Sub) : LaneBitmask::getAll();
3847 if (MOI->isDef()) {
3848 if (Sub != 0) {
3849 hasSubRegDef = true;
3850 // An operand %0:sub0 reads %0:sub1..n. Invert the lane
3851 // mask for subregister defs. Read-undef defs will be handled by
3852 // readsReg below.
3853 SLM = ~SLM;
3854 }
3855 if (MOI->isDead())
3856 hasDeadDef = true;
3857 }
3858 if (LaneMask.any() && (LaneMask & SLM).none())
3859 continue;
3860 if (MOI->readsReg())
3861 hasRead = true;
3862 }
3863 if (S.end.isDead()) {
3864 // Make sure that the corresponding machine operand for a "dead" live
3865 // range has the dead flag. We cannot perform this check for subregister
3866 // liveranges as partially dead values are allowed.
3867 if (LaneMask.none() && !hasDeadDef) {
3868 report(
3869 msg: "Instruction ending live segment on dead slot has no dead flag",
3870 MI);
3871 report_context(LR, VRegOrUnit, LaneMask);
3872 report_context(S);
3873 }
3874 } else {
3875 if (!hasRead) {
3876 // When tracking subregister liveness, the main range must start new
3877 // values on partial register writes, even if there is no read.
3878 if (!MRI->shouldTrackSubRegLiveness(VReg: VRegOrUnit.asVirtualReg()) ||
3879 LaneMask.any() || !hasSubRegDef) {
3880 report(msg: "Instruction ending live segment doesn't read the register",
3881 MI);
3882 report_context(LR, VRegOrUnit, LaneMask);
3883 report_context(S);
3884 }
3885 }
3886 }
3887 }
3888 }
3889
3890 // Now check all the basic blocks in this live segment.
3891 MachineFunction::const_iterator MFI = MBB->getIterator();
3892 // Is this live segment the beginning of a non-PHIDef VN?
3893 if (S.start == VNI->def && !VNI->isPHIDef()) {
3894 // Not live-in to any blocks.
3895 if (MBB == EndMBB)
3896 return;
3897 // Skip this block.
3898 ++MFI;
3899 }
3900
3901 SmallVector<SlotIndex, 4> Undefs;
3902 if (LaneMask.any()) {
3903 LiveInterval &OwnerLI = LiveInts->getInterval(Reg: VRegOrUnit.asVirtualReg());
3904 OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, MRI: *MRI, Indexes: *Indexes);
3905 }
3906
3907 while (true) {
3908 assert(LiveInts->isLiveInToMBB(LR, &*MFI));
3909 // We don't know how to track physregs into a landing pad.
3910 if (!VRegOrUnit.isVirtualReg() && MFI->isEHPad()) {
3911 if (&*MFI == EndMBB)
3912 break;
3913 ++MFI;
3914 continue;
3915 }
3916
3917 // Is VNI a PHI-def in the current block?
3918 bool IsPHI = VNI->isPHIDef() &&
3919 VNI->def == LiveInts->getMBBStartIdx(mbb: &*MFI);
3920
3921 // Check that VNI is live-out of all predecessors.
3922 for (const MachineBasicBlock *Pred : MFI->predecessors()) {
3923 SlotIndex PEnd = LiveInts->getMBBEndIdx(mbb: Pred);
3924 // Predecessor of landing pad live-out on last call.
3925 if (MFI->isEHPad()) {
3926 for (const MachineInstr &MI : llvm::reverse(C: *Pred)) {
3927 if (MI.isCall()) {
3928 PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex();
3929 break;
3930 }
3931 }
3932 }
3933 const VNInfo *PVNI = LR.getVNInfoBefore(Idx: PEnd);
3934
3935 // All predecessors must have a live-out value. However for a phi
3936 // instruction with subregister intervals
3937 // only one of the subregisters (not necessarily the current one) needs to
3938 // be defined.
3939 if (!PVNI && (LaneMask.none() || !IsPHI)) {
3940 if (LiveRangeCalc::isJointlyDominated(MBB: Pred, Defs: Undefs, Indexes: *Indexes))
3941 continue;
3942 report(msg: "Register not marked live out of predecessor", MBB: Pred);
3943 report_context(LR, VRegOrUnit, LaneMask);
3944 report_context(VNI: *VNI);
3945 OS << " live into " << printMBBReference(MBB: *MFI) << '@'
3946 << LiveInts->getMBBStartIdx(mbb: &*MFI) << ", not live before " << PEnd
3947 << '\n';
3948 continue;
3949 }
3950
3951 // Only PHI-defs can take different predecessor values.
3952 if (!IsPHI && PVNI != VNI) {
3953 report(msg: "Different value live out of predecessor", MBB: Pred);
3954 report_context(LR, VRegOrUnit, LaneMask);
3955 OS << "Valno #" << PVNI->id << " live out of "
3956 << printMBBReference(MBB: *Pred) << '@' << PEnd << "\nValno #" << VNI->id
3957 << " live into " << printMBBReference(MBB: *MFI) << '@'
3958 << LiveInts->getMBBStartIdx(mbb: &*MFI) << '\n';
3959 }
3960 }
3961 if (&*MFI == EndMBB)
3962 break;
3963 ++MFI;
3964 }
3965}
3966
3967void MachineVerifier::verifyLiveRange(const LiveRange &LR,
3968 VirtRegOrUnit VRegOrUnit,
3969 LaneBitmask LaneMask) {
3970 for (const VNInfo *VNI : LR.valnos)
3971 verifyLiveRangeValue(LR, VNI, VRegOrUnit, LaneMask);
3972
3973 for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
3974 verifyLiveRangeSegment(LR, I, VRegOrUnit, LaneMask);
3975}
3976
3977void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
3978 Register Reg = LI.reg();
3979 assert(Reg.isVirtual());
3980 verifyLiveRange(LR: LI, VRegOrUnit: VirtRegOrUnit(Reg));
3981
3982 if (LI.hasSubRanges()) {
3983 LaneBitmask Mask;
3984 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
3985 for (const LiveInterval::SubRange &SR : LI.subranges()) {
3986 if ((Mask & SR.LaneMask).any()) {
3987 report(msg: "Lane masks of sub ranges overlap in live interval", MF);
3988 report_context(LI);
3989 }
3990 if ((SR.LaneMask & ~MaxMask).any()) {
3991 report(msg: "Subrange lanemask is invalid", MF);
3992 report_context(LI);
3993 }
3994 if (SR.empty()) {
3995 report(msg: "Subrange must not be empty", MF);
3996 report_context(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
3997 }
3998 Mask |= SR.LaneMask;
3999 verifyLiveRange(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
4000 if (!LI.covers(Other: SR)) {
4001 report(msg: "A Subrange is not covered by the main range", MF);
4002 report_context(LI);
4003 }
4004 }
4005 }
4006
4007 // Check the LI only has one connected component.
4008 ConnectedVNInfoEqClasses ConEQ(*LiveInts);
4009 unsigned NumComp = ConEQ.Classify(LR: LI);
4010 if (NumComp > 1) {
4011 report(msg: "Multiple connected components in live interval", MF);
4012 report_context(LI);
4013 for (unsigned comp = 0; comp != NumComp; ++comp) {
4014 OS << comp << ": valnos";
4015 for (const VNInfo *I : LI.valnos)
4016 if (comp == ConEQ.getEqClass(VNI: I))
4017 OS << ' ' << I->id;
4018 OS << '\n';
4019 }
4020 }
4021}
4022
4023namespace {
4024
4025 // FrameSetup and FrameDestroy can have zero adjustment, so using a single
4026 // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the
4027 // value is zero.
4028 // We use a bool plus an integer to capture the stack state.
4029struct StackStateOfBB {
4030 StackStateOfBB() = default;
4031 StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup)
4032 : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),
4033 ExitIsSetup(ExitSetup) {}
4034
4035 // Can be negative, which means we are setting up a frame.
4036 int EntryValue = 0;
4037 int ExitValue = 0;
4038 bool EntryIsSetup = false;
4039 bool ExitIsSetup = false;
4040};
4041
4042} // end anonymous namespace
4043
4044/// Make sure on every path through the CFG, a FrameSetup <n> is always followed
4045/// by a FrameDestroy <n>, stack adjustments are identical on all
4046/// CFG edges to a merge point, and frame is destroyed at end of a return block.
4047void MachineVerifier::verifyStackFrame() {
4048 unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
4049 unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
4050 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
4051 return;
4052
4053 SmallVector<StackStateOfBB, 8> SPState;
4054 SPState.resize(N: MF->getNumBlockIDs());
4055 df_iterator_default_set<const MachineBasicBlock*> Reachable;
4056
4057 // Visit the MBBs in DFS order.
4058 for (df_ext_iterator<const MachineFunction *,
4059 df_iterator_default_set<const MachineBasicBlock *>>
4060 DFI = df_ext_begin(G: MF, S&: Reachable), DFE = df_ext_end(G: MF, S&: Reachable);
4061 DFI != DFE; ++DFI) {
4062 const MachineBasicBlock *MBB = *DFI;
4063
4064 StackStateOfBB BBState;
4065 // Check the exit state of the DFS stack predecessor.
4066 if (DFI.getPathLength() >= 2) {
4067 const MachineBasicBlock *StackPred = DFI.getPath(n: DFI.getPathLength() - 2);
4068 assert(Reachable.count(StackPred) &&
4069 "DFS stack predecessor is already visited.\n");
4070 BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;
4071 BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;
4072 BBState.ExitValue = BBState.EntryValue;
4073 BBState.ExitIsSetup = BBState.EntryIsSetup;
4074 }
4075
4076 if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) {
4077 report(msg: "Call frame size on entry does not match value computed from "
4078 "predecessor",
4079 MBB);
4080 OS << "Call frame size on entry " << MBB->getCallFrameSize()
4081 << " does not match value computed from predecessor "
4082 << -BBState.EntryValue << '\n';
4083 }
4084
4085 // Update stack state by checking contents of MBB.
4086 for (const auto &I : *MBB) {
4087 if (I.getOpcode() == FrameSetupOpcode) {
4088 if (BBState.ExitIsSetup)
4089 report(msg: "FrameSetup is after another FrameSetup", MI: &I);
4090 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4091 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4092 "instruction.", MI: &I);
4093 BBState.ExitValue -= TII->getFrameTotalSize(I);
4094 BBState.ExitIsSetup = true;
4095 }
4096
4097 if (I.getOpcode() == FrameDestroyOpcode) {
4098 int Size = TII->getFrameTotalSize(I);
4099 if (!BBState.ExitIsSetup)
4100 report(msg: "FrameDestroy is not after a FrameSetup", MI: &I);
4101 int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :
4102 BBState.ExitValue;
4103 if (BBState.ExitIsSetup && AbsSPAdj != Size) {
4104 report(msg: "FrameDestroy <n> is after FrameSetup <m>", MI: &I);
4105 OS << "FrameDestroy <" << Size << "> is after FrameSetup <"
4106 << AbsSPAdj << ">.\n";
4107 }
4108 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4109 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4110 "instruction.", MI: &I);
4111 BBState.ExitValue += Size;
4112 BBState.ExitIsSetup = false;
4113 }
4114 }
4115 SPState[MBB->getNumber()] = BBState;
4116
4117 // Make sure the exit state of any predecessor is consistent with the entry
4118 // state.
4119 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
4120 if (Reachable.count(Ptr: Pred) &&
4121 (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue ||
4122 SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {
4123 report(msg: "The exit stack state of a predecessor is inconsistent.", MBB);
4124 OS << "Predecessor " << printMBBReference(MBB: *Pred) << " has exit state ("
4125 << SPState[Pred->getNumber()].ExitValue << ", "
4126 << SPState[Pred->getNumber()].ExitIsSetup << "), while "
4127 << printMBBReference(MBB: *MBB) << " has entry state ("
4128 << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";
4129 }
4130 }
4131
4132 // Make sure the entry state of any successor is consistent with the exit
4133 // state.
4134 for (const MachineBasicBlock *Succ : MBB->successors()) {
4135 if (Reachable.count(Ptr: Succ) &&
4136 (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue ||
4137 SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {
4138 report(msg: "The entry stack state of a successor is inconsistent.", MBB);
4139 OS << "Successor " << printMBBReference(MBB: *Succ) << " has entry state ("
4140 << SPState[Succ->getNumber()].EntryValue << ", "
4141 << SPState[Succ->getNumber()].EntryIsSetup << "), while "
4142 << printMBBReference(MBB: *MBB) << " has exit state ("
4143 << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";
4144 }
4145 }
4146
4147 // Make sure a basic block with return ends with zero stack adjustment.
4148 if (!MBB->empty() && MBB->back().isReturn()) {
4149 if (BBState.ExitIsSetup)
4150 report(msg: "A return block ends with a FrameSetup.", MBB);
4151 if (BBState.ExitValue)
4152 report(msg: "A return block ends with a nonzero stack adjustment.", MBB);
4153 }
4154 }
4155}
4156
4157void MachineVerifier::verifyStackProtector() {
4158 const MachineFrameInfo &MFI = MF->getFrameInfo();
4159 if (!MFI.hasStackProtectorIndex())
4160 return;
4161 // Only applicable when the offsets of frame objects have been determined,
4162 // which is indicated by a non-zero stack size.
4163 if (!MFI.getStackSize())
4164 return;
4165 const TargetFrameLowering &TFI = *MF->getSubtarget().getFrameLowering();
4166 bool StackGrowsDown =
4167 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
4168 unsigned FI = MFI.getStackProtectorIndex();
4169 int64_t SPStart = MFI.getObjectOffset(ObjectIdx: FI);
4170 int64_t SPEnd = SPStart + MFI.getObjectSize(ObjectIdx: FI);
4171 for (unsigned I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
4172 if (I == FI)
4173 continue;
4174 if (MFI.isDeadObjectIndex(ObjectIdx: I))
4175 continue;
4176 // FIXME: Skip non-default stack objects, as some targets may place them
4177 // above the stack protector. This is a workaround for the fact that
4178 // backends such as AArch64 may place SVE stack objects *above* the stack
4179 // protector.
4180 if (MFI.getStackID(ObjectIdx: I) != TargetStackID::Default)
4181 continue;
4182 // Skip variable-sized objects because they do not have a fixed offset.
4183 if (MFI.isVariableSizedObjectIndex(ObjectIdx: I))
4184 continue;
4185 // FIXME: Skip spill slots which may be allocated above the stack protector.
4186 // Ideally this would only skip callee-saved registers, but we don't have
4187 // that information here. For example, spill-slots used for scavenging are
4188 // not described in CalleeSavedInfo.
4189 if (MFI.isSpillSlotObjectIndex(ObjectIdx: I))
4190 continue;
4191 int64_t ObjStart = MFI.getObjectOffset(ObjectIdx: I);
4192 int64_t ObjEnd = ObjStart + MFI.getObjectSize(ObjectIdx: I);
4193 if (SPStart < ObjEnd && ObjStart < SPEnd) {
4194 report(msg: "Stack protector overlaps with another stack object", MF);
4195 break;
4196 }
4197 if ((StackGrowsDown && SPStart <= ObjStart) ||
4198 (!StackGrowsDown && SPStart >= ObjStart)) {
4199 report(msg: "Stack protector is not the top-most object on the stack", MF);
4200 break;
4201 }
4202 }
4203}
4204