1//===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Pass to verify generated machine code. The following is checked:
10//
11// Operand counts: All explicit operands must be present.
12//
13// Register classes: All physical and virtual register operands must be
14// compatible with the register class required by the instruction descriptor.
15//
16// Register live intervals: Registers must be defined only once, and must be
17// defined before use.
18//
19// The machine code verifier is enabled with the command-line option
20// -verify-machineinstrs.
21//===----------------------------------------------------------------------===//
22
23#include "llvm/CodeGen/MachineVerifier.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/ADT/DenseSet.h"
27#include "llvm/ADT/DepthFirstIterator.h"
28#include "llvm/ADT/PostOrderIterator.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SetOperations.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/StringRef.h"
34#include "llvm/ADT/Twine.h"
35#include "llvm/CodeGen/CodeGenCommonISel.h"
36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
37#include "llvm/CodeGen/LiveInterval.h"
38#include "llvm/CodeGen/LiveIntervals.h"
39#include "llvm/CodeGen/LiveRangeCalc.h"
40#include "llvm/CodeGen/LiveStacks.h"
41#include "llvm/CodeGen/LiveVariables.h"
42#include "llvm/CodeGen/MachineBasicBlock.h"
43#include "llvm/CodeGen/MachineConvergenceVerifier.h"
44#include "llvm/CodeGen/MachineDominators.h"
45#include "llvm/CodeGen/MachineFrameInfo.h"
46#include "llvm/CodeGen/MachineFunction.h"
47#include "llvm/CodeGen/MachineFunctionPass.h"
48#include "llvm/CodeGen/MachineInstr.h"
49#include "llvm/CodeGen/MachineInstrBundle.h"
50#include "llvm/CodeGen/MachineMemOperand.h"
51#include "llvm/CodeGen/MachineOperand.h"
52#include "llvm/CodeGen/MachineRegisterInfo.h"
53#include "llvm/CodeGen/PseudoSourceValue.h"
54#include "llvm/CodeGen/RegisterBank.h"
55#include "llvm/CodeGen/RegisterBankInfo.h"
56#include "llvm/CodeGen/SlotIndexes.h"
57#include "llvm/CodeGen/StackMaps.h"
58#include "llvm/CodeGen/TargetInstrInfo.h"
59#include "llvm/CodeGen/TargetLowering.h"
60#include "llvm/CodeGen/TargetOpcodes.h"
61#include "llvm/CodeGen/TargetRegisterInfo.h"
62#include "llvm/CodeGen/TargetSubtargetInfo.h"
63#include "llvm/CodeGenTypes/LowLevelType.h"
64#include "llvm/IR/BasicBlock.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/EHPersonalities.h"
67#include "llvm/IR/Function.h"
68#include "llvm/IR/InlineAsm.h"
69#include "llvm/IR/Instructions.h"
70#include "llvm/InitializePasses.h"
71#include "llvm/MC/LaneBitmask.h"
72#include "llvm/MC/MCAsmInfo.h"
73#include "llvm/MC/MCDwarf.h"
74#include "llvm/MC/MCInstrDesc.h"
75#include "llvm/MC/MCRegisterInfo.h"
76#include "llvm/MC/MCTargetOptions.h"
77#include "llvm/Pass.h"
78#include "llvm/Support/Casting.h"
79#include "llvm/Support/ErrorHandling.h"
80#include "llvm/Support/ManagedStatic.h"
81#include "llvm/Support/MathExtras.h"
82#include "llvm/Support/ModRef.h"
83#include "llvm/Support/Mutex.h"
84#include "llvm/Support/raw_ostream.h"
85#include "llvm/Target/TargetMachine.h"
86#include <algorithm>
87#include <cassert>
88#include <cstddef>
89#include <cstdint>
90#include <iterator>
91#include <string>
92#include <utility>
93
94using namespace llvm;
95
96namespace {
97
98/// Used the by the ReportedErrors class to guarantee only one error is reported
99/// at one time.
100static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock;
101
102struct MachineVerifier {
103 MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b,
104 raw_ostream *OS, bool AbortOnError = true)
105 : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b),
106 ReportedErrs(AbortOnError) {}
107
108 MachineVerifier(Pass *pass, const char *b, raw_ostream *OS,
109 bool AbortOnError = true)
110 : PASS(pass), OS(OS ? *OS : nulls()), Banner(b),
111 ReportedErrs(AbortOnError) {}
112
113 MachineVerifier(const char *b, LiveVariables *LiveVars,
114 LiveIntervals *LiveInts, LiveStacks *LiveStks,
115 SlotIndexes *Indexes, raw_ostream *OS,
116 bool AbortOnError = true)
117 : OS(OS ? *OS : nulls()), Banner(b), LiveVars(LiveVars),
118 LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes),
119 ReportedErrs(AbortOnError) {}
120
121 /// \returns true if no problems were found.
122 bool verify(const MachineFunction &MF);
123
124 MachineFunctionAnalysisManager *MFAM = nullptr;
125 Pass *const PASS = nullptr;
126 raw_ostream &OS;
127 const char *Banner;
128 const MachineFunction *MF = nullptr;
129 const TargetMachine *TM = nullptr;
130 const TargetInstrInfo *TII = nullptr;
131 const TargetRegisterInfo *TRI = nullptr;
132 const MachineRegisterInfo *MRI = nullptr;
133 const RegisterBankInfo *RBI = nullptr;
134
135 // Avoid querying the MachineFunctionProperties for each operand.
136 bool isFunctionRegBankSelected = false;
137 bool isFunctionSelected = false;
138 bool isFunctionTracksDebugUserValues = false;
139
140 using RegVector = SmallVector<Register, 16>;
141 using RegMaskVector = SmallVector<const uint32_t *, 4>;
142 using RegSet = DenseSet<Register>;
143 using RegMap = DenseMap<Register, const MachineInstr *>;
144 using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;
145
146 const MachineInstr *FirstNonPHI = nullptr;
147 const MachineInstr *FirstTerminator = nullptr;
148 BlockSet FunctionBlocks;
149
150 BitVector regsReserved;
151 RegSet regsLive;
152 RegVector regsDefined, regsDead, regsKilled;
153 RegMaskVector regMasks;
154
155 SlotIndex lastIndex;
156
157 // Add Reg and any sub-registers to RV
158 void addRegWithSubRegs(RegVector &RV, Register Reg) {
159 RV.push_back(Elt: Reg);
160 if (Reg.isPhysical())
161 append_range(C&: RV, R: TRI->subregs(Reg: Reg.asMCReg()));
162 }
163
164 struct BBInfo {
165 // Is this MBB reachable from the MF entry point?
166 bool reachable = false;
167
168 // Vregs that must be live in because they are used without being
169 // defined. Map value is the user. vregsLiveIn doesn't include regs
170 // that only are used by PHI nodes.
171 RegMap vregsLiveIn;
172
173 // Regs killed in MBB. They may be defined again, and will then be in both
174 // regsKilled and regsLiveOut.
175 RegSet regsKilled;
176
177 // Regs defined in MBB and live out. Note that vregs passing through may
178 // be live out without being mentioned here.
179 RegSet regsLiveOut;
180
181 // Vregs that pass through MBB untouched. This set is disjoint from
182 // regsKilled and regsLiveOut.
183 RegSet vregsPassed;
184
185 // Vregs that must pass through MBB because they are needed by a successor
186 // block. This set is disjoint from regsLiveOut.
187 RegSet vregsRequired;
188
189 // Set versions of block's predecessor and successor lists.
190 BlockSet Preds, Succs;
191
192 BBInfo() = default;
193
194 // Add register to vregsRequired if it belongs there. Return true if
195 // anything changed.
196 bool addRequired(Register Reg) {
197 if (!Reg.isVirtual())
198 return false;
199 if (regsLiveOut.count(V: Reg))
200 return false;
201 return vregsRequired.insert(V: Reg).second;
202 }
203
204 // Same for a full set.
205 bool addRequired(const RegSet &RS) {
206 bool Changed = false;
207 for (Register Reg : RS)
208 Changed |= addRequired(Reg);
209 return Changed;
210 }
211
212 // Same for a full map.
213 bool addRequired(const RegMap &RM) {
214 bool Changed = false;
215 for (const auto &I : RM)
216 Changed |= addRequired(Reg: I.first);
217 return Changed;
218 }
219
220 // Live-out registers are either in regsLiveOut or vregsPassed.
221 bool isLiveOut(Register Reg) const {
222 return regsLiveOut.count(V: Reg) || vregsPassed.count(V: Reg);
223 }
224 };
225
226 // Extra register info per MBB.
227 DenseMap<const MachineBasicBlock *, BBInfo> MBBInfoMap;
228
229 bool isReserved(Register Reg) {
230 return Reg.id() < regsReserved.size() && regsReserved.test(Idx: Reg.id());
231 }
232
233 bool isAllocatable(Register Reg) const {
234 return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(RegNo: Reg) &&
235 !regsReserved.test(Idx: Reg.id());
236 }
237
238 // Analysis information if available
239 LiveVariables *LiveVars = nullptr;
240 LiveIntervals *LiveInts = nullptr;
241 LiveStacks *LiveStks = nullptr;
242 SlotIndexes *Indexes = nullptr;
243
244 /// A class to track the number of reported error and to guarantee that only
245 /// one error is reported at one time.
246 class ReportedErrors {
247 unsigned NumReported = 0;
248 bool AbortOnError;
249
250 public:
251 /// \param AbortOnError -- If set, abort after printing the first error.
252 ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {}
253
254 ~ReportedErrors() {
255 if (!hasError())
256 return;
257 if (AbortOnError)
258 report_fatal_error(reason: "Found " + Twine(NumReported) +
259 " machine code errors.");
260 // Since we haven't aborted, release the lock to allow other threads to
261 // report errors.
262 ReportedErrorsLock->unlock();
263 }
264
265 /// Increment the number of reported errors.
266 /// \returns true if this is the first reported error.
267 bool increment() {
268 // If this is the first error this thread has encountered, grab the lock
269 // to prevent other threads from reporting errors at the same time.
270 // Otherwise we assume we already have the lock.
271 if (!hasError())
272 ReportedErrorsLock->lock();
273 ++NumReported;
274 return NumReported == 1;
275 }
276
277 /// \returns true if an error was reported.
278 bool hasError() { return NumReported; }
279 };
280 ReportedErrors ReportedErrs;
281
282 // This is calculated only when trying to verify convergence control tokens.
283 // Similar to the LLVM IR verifier, we calculate this locally instead of
284 // relying on the pass manager.
285 MachineDominatorTree DT;
286
287 void visitMachineFunctionBefore();
288 void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
289 void visitMachineBundleBefore(const MachineInstr *MI);
290
291 /// Verify that all of \p MI's virtual register operands are scalars.
292 /// \returns True if all virtual register operands are scalar. False
293 /// otherwise.
294 bool verifyAllRegOpsScalar(const MachineInstr &MI,
295 const MachineRegisterInfo &MRI);
296 bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);
297
298 bool verifyGIntrinsicSideEffects(const MachineInstr *MI);
299 bool verifyGIntrinsicConvergence(const MachineInstr *MI);
300 void verifyPreISelGenericInstruction(const MachineInstr *MI);
301
302 void visitMachineInstrBefore(const MachineInstr *MI);
303 void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
304 void visitMachineBundleAfter(const MachineInstr *MI);
305 void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);
306 void visitMachineFunctionAfter();
307
308 void report(const char *msg, const MachineFunction *MF);
309 void report(const char *msg, const MachineBasicBlock *MBB);
310 void report(const char *msg, const MachineInstr *MI);
311 void report(const char *msg, const MachineOperand *MO, unsigned MONum,
312 LLT MOVRegType = LLT{});
313 void report(const Twine &Msg, const MachineInstr *MI);
314
315 void report_context(const LiveInterval &LI) const;
316 void report_context(const LiveRange &LR, VirtRegOrUnit VRegOrUnit,
317 LaneBitmask LaneMask) const;
318 void report_context(const LiveRange::Segment &S) const;
319 void report_context(const VNInfo &VNI) const;
320 void report_context(SlotIndex Pos) const;
321 void report_context(MCPhysReg PhysReg) const;
322 void report_context_liverange(const LiveRange &LR) const;
323 void report_context_lanemask(LaneBitmask LaneMask) const;
324 void report_context_vreg(Register VReg) const;
325 void report_context_vreg_regunit(VirtRegOrUnit VRegOrUnit) const;
326
327 void verifyInlineAsm(const MachineInstr *MI);
328
329 void checkLiveness(const MachineOperand *MO, unsigned MONum);
330 void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
331 SlotIndex UseIdx, const LiveRange &LR,
332 VirtRegOrUnit VRegOrUnit,
333 LaneBitmask LaneMask = LaneBitmask::getNone());
334 void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
335 SlotIndex DefIdx, const LiveRange &LR,
336 VirtRegOrUnit VRegOrUnit, bool SubRangeCheck = false,
337 LaneBitmask LaneMask = LaneBitmask::getNone());
338
339 void markReachable(const MachineBasicBlock *MBB);
340 void calcRegsPassed();
341 void checkPHIOps(const MachineBasicBlock &MBB);
342
343 void calcRegsRequired();
344 void verifyLiveVariables();
345 void verifyLiveIntervals();
346 void verifyLiveInterval(const LiveInterval &);
347 void verifyLiveRangeValue(const LiveRange &, const VNInfo *, VirtRegOrUnit,
348 LaneBitmask);
349 void verifyLiveRangeSegment(const LiveRange &,
350 const LiveRange::const_iterator I, VirtRegOrUnit,
351 LaneBitmask);
352 void verifyLiveRange(const LiveRange &, VirtRegOrUnit,
353 LaneBitmask LaneMask = LaneBitmask::getNone());
354
355 void verifyStackFrame();
356 /// Check that the stack protector is the top-most object in the stack.
357 void verifyStackProtector();
358
359 void verifySlotIndexes() const;
360 void verifyProperties(const MachineFunction &MF);
361};
362
363struct MachineVerifierLegacyPass : public MachineFunctionPass {
364 static char ID; // Pass ID, replacement for typeid
365
366 const std::string Banner;
367
368 MachineVerifierLegacyPass(std::string banner = std::string())
369 : MachineFunctionPass(ID), Banner(std::move(banner)) {
370 initializeMachineVerifierLegacyPassPass(*PassRegistry::getPassRegistry());
371 }
372
373 void getAnalysisUsage(AnalysisUsage &AU) const override {
374 AU.addUsedIfAvailable<LiveStacksWrapperLegacy>();
375 AU.addUsedIfAvailable<LiveVariablesWrapperPass>();
376 AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
377 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
378 AU.setPreservesAll();
379 MachineFunctionPass::getAnalysisUsage(AU);
380 }
381
382 bool runOnMachineFunction(MachineFunction &MF) override {
383 // Skip functions that have known verification problems.
384 // FIXME: Remove this mechanism when all problematic passes have been
385 // fixed.
386 if (MF.getProperties().hasFailsVerification())
387 return false;
388
389 MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);
390 return false;
391 }
392};
393
394} // end anonymous namespace
395
396PreservedAnalyses
397MachineVerifierPass::run(MachineFunction &MF,
398 MachineFunctionAnalysisManager &MFAM) {
399 // Skip functions that have known verification problems.
400 // FIXME: Remove this mechanism when all problematic passes have been
401 // fixed.
402 if (MF.getProperties().hasFailsVerification())
403 return PreservedAnalyses::all();
404 MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);
405 return PreservedAnalyses::all();
406}
407
408char MachineVerifierLegacyPass::ID = 0;
409
410INITIALIZE_PASS(MachineVerifierLegacyPass, "machineverifier",
411 "Verify generated machine code", false, false)
412
413FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {
414 return new MachineVerifierLegacyPass(Banner);
415}
416
417void llvm::verifyMachineFunction(const std::string &Banner,
418 const MachineFunction &MF) {
419 // TODO: Use MFAM after porting below analyses.
420 // LiveVariables *LiveVars;
421 // LiveIntervals *LiveInts;
422 // LiveStacks *LiveStks;
423 // SlotIndexes *Indexes;
424 MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);
425}
426
427bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,
428 bool AbortOnError) const {
429 return MachineVerifier(p, Banner, OS, AbortOnError).verify(MF: *this);
430}
431
432bool MachineFunction::verify(MachineFunctionAnalysisManager &MFAM,
433 const char *Banner, raw_ostream *OS,
434 bool AbortOnError) const {
435 return MachineVerifier(MFAM, Banner, OS, AbortOnError).verify(MF: *this);
436}
437
438bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
439 const char *Banner, raw_ostream *OS,
440 bool AbortOnError) const {
441 return MachineVerifier(Banner, /*LiveVars=*/nullptr, LiveInts,
442 /*LiveStks=*/nullptr, Indexes, OS, AbortOnError)
443 .verify(MF: *this);
444}
445
446void MachineVerifier::verifySlotIndexes() const {
447 if (Indexes == nullptr)
448 return;
449
450 // Ensure the IdxMBB list is sorted by slot indexes.
451 SlotIndex Last;
452 for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),
453 E = Indexes->MBBIndexEnd(); I != E; ++I) {
454 assert(!Last.isValid() || I->first > Last);
455 Last = I->first;
456 }
457}
458
459void MachineVerifier::verifyProperties(const MachineFunction &MF) {
460 // If a pass has introduced virtual registers without clearing the
461 // NoVRegs property (or set it without allocating the vregs)
462 // then report an error.
463 if (MF.getProperties().hasNoVRegs() && MRI->getNumVirtRegs())
464 report(msg: "Function has NoVRegs property but there are VReg operands", MF: &MF);
465}
466
467bool MachineVerifier::verify(const MachineFunction &MF) {
468 this->MF = &MF;
469 TM = &MF.getTarget();
470 TII = MF.getSubtarget().getInstrInfo();
471 TRI = MF.getSubtarget().getRegisterInfo();
472 RBI = MF.getSubtarget().getRegBankInfo();
473 MRI = &MF.getRegInfo();
474
475 const MachineFunctionProperties &Props = MF.getProperties();
476 const bool isFunctionFailedISel = Props.hasFailedISel();
477
478 // If we're mid-GlobalISel and we already triggered the fallback path then
479 // it's expected that the MIR is somewhat broken but that's ok since we'll
480 // reset it and clear the FailedISel attribute in ResetMachineFunctions.
481 if (isFunctionFailedISel)
482 return true;
483
484 isFunctionRegBankSelected = Props.hasRegBankSelected();
485 isFunctionSelected = Props.hasSelected();
486 isFunctionTracksDebugUserValues = Props.hasTracksDebugUserValues();
487
488 if (PASS) {
489 auto *LISWrapper = PASS->getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
490 LiveInts = LISWrapper ? &LISWrapper->getLIS() : nullptr;
491 // We don't want to verify LiveVariables if LiveIntervals is available.
492 auto *LVWrapper = PASS->getAnalysisIfAvailable<LiveVariablesWrapperPass>();
493 if (!LiveInts)
494 LiveVars = LVWrapper ? &LVWrapper->getLV() : nullptr;
495 auto *LSWrapper = PASS->getAnalysisIfAvailable<LiveStacksWrapperLegacy>();
496 LiveStks = LSWrapper ? &LSWrapper->getLS() : nullptr;
497 auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>();
498 Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
499 }
500 if (MFAM) {
501 MachineFunction &Func = const_cast<MachineFunction &>(MF);
502 LiveInts = MFAM->getCachedResult<LiveIntervalsAnalysis>(IR&: Func);
503 if (!LiveInts)
504 LiveVars = MFAM->getCachedResult<LiveVariablesAnalysis>(IR&: Func);
505 // TODO: LiveStks = MFAM->getCachedResult<LiveStacksAnalysis>(Func);
506 Indexes = MFAM->getCachedResult<SlotIndexesAnalysis>(IR&: Func);
507 }
508
509 verifySlotIndexes();
510
511 verifyProperties(MF);
512
513 visitMachineFunctionBefore();
514 for (const MachineBasicBlock &MBB : MF) {
515 visitMachineBasicBlockBefore(MBB: &MBB);
516 // Keep track of the current bundle header.
517 const MachineInstr *CurBundle = nullptr;
518 // Do we expect the next instruction to be part of the same bundle?
519 bool InBundle = false;
520
521 for (const MachineInstr &MI : MBB.instrs()) {
522 if (MI.getParent() != &MBB) {
523 report(msg: "Bad instruction parent pointer", MBB: &MBB);
524 OS << "Instruction: " << MI;
525 continue;
526 }
527
528 // Check for consistent bundle flags.
529 if (InBundle && !MI.isBundledWithPred())
530 report(msg: "Missing BundledPred flag, "
531 "BundledSucc was set on predecessor",
532 MI: &MI);
533 if (!InBundle && MI.isBundledWithPred())
534 report(msg: "BundledPred flag is set, "
535 "but BundledSucc not set on predecessor",
536 MI: &MI);
537
538 // Is this a bundle header?
539 if (!MI.isInsideBundle()) {
540 if (CurBundle)
541 visitMachineBundleAfter(MI: CurBundle);
542 CurBundle = &MI;
543 visitMachineBundleBefore(MI: CurBundle);
544 } else if (!CurBundle)
545 report(msg: "No bundle header", MI: &MI);
546 visitMachineInstrBefore(MI: &MI);
547 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
548 const MachineOperand &Op = MI.getOperand(i: I);
549 if (Op.getParent() != &MI) {
550 // Make sure to use correct addOperand / removeOperand / ChangeTo
551 // functions when replacing operands of a MachineInstr.
552 report(msg: "Instruction has operand with wrong parent set", MI: &MI);
553 }
554
555 visitMachineOperand(MO: &Op, MONum: I);
556 }
557
558 // Was this the last bundled instruction?
559 InBundle = MI.isBundledWithSucc();
560 }
561 if (CurBundle)
562 visitMachineBundleAfter(MI: CurBundle);
563 if (InBundle)
564 report(msg: "BundledSucc flag set on last instruction in block", MI: &MBB.back());
565 visitMachineBasicBlockAfter(MBB: &MBB);
566 }
567 visitMachineFunctionAfter();
568
569 // Clean up.
570 regsLive.clear();
571 regsDefined.clear();
572 regsDead.clear();
573 regsKilled.clear();
574 regMasks.clear();
575 MBBInfoMap.clear();
576
577 return !ReportedErrs.hasError();
578}
579
580void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
581 assert(MF);
582 OS << '\n';
583 if (ReportedErrs.increment()) {
584 if (Banner)
585 OS << "# " << Banner << '\n';
586
587 if (LiveInts != nullptr)
588 LiveInts->print(O&: OS);
589 else
590 MF->print(OS, Indexes);
591 }
592
593 OS << "*** Bad machine code: " << msg << " ***\n"
594 << "- function: " << MF->getName() << '\n';
595}
596
597void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
598 assert(MBB);
599 report(msg, MF: MBB->getParent());
600 OS << "- basic block: " << printMBBReference(MBB: *MBB) << ' ' << MBB->getName()
601 << " (" << (const void *)MBB << ')';
602 if (Indexes)
603 OS << " [" << Indexes->getMBBStartIdx(mbb: MBB) << ';'
604 << Indexes->getMBBEndIdx(mbb: MBB) << ')';
605 OS << '\n';
606}
607
608void MachineVerifier::report(const char *msg, const MachineInstr *MI) {
609 assert(MI);
610 report(msg, MBB: MI->getParent());
611 OS << "- instruction: ";
612 if (Indexes && Indexes->hasIndex(instr: *MI))
613 OS << Indexes->getInstructionIndex(MI: *MI) << '\t';
614 MI->print(OS, /*IsStandalone=*/true);
615}
616
617void MachineVerifier::report(const char *msg, const MachineOperand *MO,
618 unsigned MONum, LLT MOVRegType) {
619 assert(MO);
620 report(msg, MI: MO->getParent());
621 OS << "- operand " << MONum << ": ";
622 MO->print(os&: OS, TypeToPrint: MOVRegType, TRI);
623 OS << '\n';
624}
625
626void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) {
627 report(msg: Msg.str().c_str(), MI);
628}
629
630void MachineVerifier::report_context(SlotIndex Pos) const {
631 OS << "- at: " << Pos << '\n';
632}
633
634void MachineVerifier::report_context(const LiveInterval &LI) const {
635 OS << "- interval: " << LI << '\n';
636}
637
638void MachineVerifier::report_context(const LiveRange &LR,
639 VirtRegOrUnit VRegOrUnit,
640 LaneBitmask LaneMask) const {
641 report_context_liverange(LR);
642 report_context_vreg_regunit(VRegOrUnit);
643 if (LaneMask.any())
644 report_context_lanemask(LaneMask);
645}
646
647void MachineVerifier::report_context(const LiveRange::Segment &S) const {
648 OS << "- segment: " << S << '\n';
649}
650
651void MachineVerifier::report_context(const VNInfo &VNI) const {
652 OS << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n";
653}
654
655void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
656 OS << "- liverange: " << LR << '\n';
657}
658
659void MachineVerifier::report_context(MCPhysReg PReg) const {
660 OS << "- p. register: " << printReg(Reg: PReg, TRI) << '\n';
661}
662
663void MachineVerifier::report_context_vreg(Register VReg) const {
664 OS << "- v. register: " << printReg(Reg: VReg, TRI) << '\n';
665}
666
667void MachineVerifier::report_context_vreg_regunit(
668 VirtRegOrUnit VRegOrUnit) const {
669 if (VRegOrUnit.isVirtualReg()) {
670 report_context_vreg(VReg: VRegOrUnit.asVirtualReg());
671 } else {
672 OS << "- regunit: " << printRegUnit(Unit: VRegOrUnit.asMCRegUnit(), TRI)
673 << '\n';
674 }
675}
676
677void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {
678 OS << "- lanemask: " << PrintLaneMask(LaneMask) << '\n';
679}
680
681void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
682 BBInfo &MInfo = MBBInfoMap[MBB];
683 if (!MInfo.reachable) {
684 MInfo.reachable = true;
685 for (const MachineBasicBlock *Succ : MBB->successors())
686 markReachable(MBB: Succ);
687 }
688}
689
690void MachineVerifier::visitMachineFunctionBefore() {
691 lastIndex = SlotIndex();
692 regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()
693 : TRI->getReservedRegs(MF: *MF);
694
695 if (!MF->empty())
696 markReachable(MBB: &MF->front());
697
698 // Build a set of the basic blocks in the function.
699 FunctionBlocks.clear();
700 for (const auto &MBB : *MF) {
701 FunctionBlocks.insert(Ptr: &MBB);
702 BBInfo &MInfo = MBBInfoMap[&MBB];
703
704 MInfo.Preds.insert_range(R: MBB.predecessors());
705 if (MInfo.Preds.size() != MBB.pred_size())
706 report(msg: "MBB has duplicate entries in its predecessor list.", MBB: &MBB);
707
708 MInfo.Succs.insert_range(R: MBB.successors());
709 if (MInfo.Succs.size() != MBB.succ_size())
710 report(msg: "MBB has duplicate entries in its successor list.", MBB: &MBB);
711 }
712
713 // Check that the register use lists are sane.
714 MRI->verifyUseLists();
715
716 if (!MF->empty()) {
717 verifyStackFrame();
718 verifyStackProtector();
719 }
720}
721
722void
723MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
724 FirstTerminator = nullptr;
725 FirstNonPHI = nullptr;
726
727 if (!MF->getProperties().hasNoPHIs() && MRI->tracksLiveness()) {
728 // If this block has allocatable physical registers live-in, check that
729 // it is an entry block or landing pad.
730 for (const auto &LI : MBB->liveins()) {
731 if (isAllocatable(Reg: LI.PhysReg) && !MBB->isEHPad() &&
732 MBB->getIterator() != MBB->getParent()->begin() &&
733 !MBB->isInlineAsmBrIndirectTarget()) {
734 report(msg: "MBB has allocatable live-in, but isn't entry, landing-pad, or "
735 "inlineasm-br-indirect-target.",
736 MBB);
737 report_context(PReg: LI.PhysReg);
738 }
739 }
740 }
741
742 if (MBB->isIRBlockAddressTaken()) {
743 if (!MBB->getAddressTakenIRBlock()->hasAddressTaken())
744 report(msg: "ir-block-address-taken is associated with basic block not used by "
745 "a blockaddress.",
746 MBB);
747 }
748
749 // Count the number of landing pad successors.
750 SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs;
751 for (const auto *succ : MBB->successors()) {
752 if (succ->isEHPad())
753 LandingPadSuccs.insert(Ptr: succ);
754 if (!FunctionBlocks.count(Ptr: succ))
755 report(msg: "MBB has successor that isn't part of the function.", MBB);
756 if (!MBBInfoMap[succ].Preds.count(Ptr: MBB)) {
757 report(msg: "Inconsistent CFG", MBB);
758 OS << "MBB is not in the predecessor list of the successor "
759 << printMBBReference(MBB: *succ) << ".\n";
760 }
761 }
762
763 // Check the predecessor list.
764 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
765 if (!FunctionBlocks.count(Ptr: Pred))
766 report(msg: "MBB has predecessor that isn't part of the function.", MBB);
767 if (!MBBInfoMap[Pred].Succs.count(Ptr: MBB)) {
768 report(msg: "Inconsistent CFG", MBB);
769 OS << "MBB is not in the successor list of the predecessor "
770 << printMBBReference(MBB: *Pred) << ".\n";
771 }
772 }
773
774 const MCAsmInfo *AsmInfo = TM->getMCAsmInfo();
775 const BasicBlock *BB = MBB->getBasicBlock();
776 const Function &F = MF->getFunction();
777 if (LandingPadSuccs.size() > 1 &&
778 !(AsmInfo &&
779 AsmInfo->getExceptionHandlingType() == ExceptionHandling::SjLj &&
780 BB && isa<SwitchInst>(Val: BB->getTerminator())) &&
781 !isScopedEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
782 report(msg: "MBB has more than one landing pad successor", MBB);
783
784 // Call analyzeBranch. If it succeeds, there several more conditions to check.
785 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
786 SmallVector<MachineOperand, 4> Cond;
787 if (!TII->analyzeBranch(MBB&: *const_cast<MachineBasicBlock *>(MBB), TBB, FBB,
788 Cond)) {
789 // Ok, analyzeBranch thinks it knows what's going on with this block. Let's
790 // check whether its answers match up with reality.
791 if (!TBB && !FBB) {
792 // Block falls through to its successor.
793 if (!MBB->empty() && MBB->back().isBarrier() &&
794 !TII->isPredicated(MI: MBB->back())) {
795 report(msg: "MBB exits via unconditional fall-through but ends with a "
796 "barrier instruction!", MBB);
797 }
798 if (!Cond.empty()) {
799 report(msg: "MBB exits via unconditional fall-through but has a condition!",
800 MBB);
801 }
802 } else if (TBB && !FBB && Cond.empty()) {
803 // Block unconditionally branches somewhere.
804 if (MBB->empty()) {
805 report(msg: "MBB exits via unconditional branch but doesn't contain "
806 "any instructions!", MBB);
807 } else if (!MBB->back().isBarrier()) {
808 report(msg: "MBB exits via unconditional branch but doesn't end with a "
809 "barrier instruction!", MBB);
810 } else if (!MBB->back().isTerminator()) {
811 report(msg: "MBB exits via unconditional branch but the branch isn't a "
812 "terminator instruction!", MBB);
813 }
814 } else if (TBB && !FBB && !Cond.empty()) {
815 // Block conditionally branches somewhere, otherwise falls through.
816 if (MBB->empty()) {
817 report(msg: "MBB exits via conditional branch/fall-through but doesn't "
818 "contain any instructions!", MBB);
819 } else if (MBB->back().isBarrier()) {
820 report(msg: "MBB exits via conditional branch/fall-through but ends with a "
821 "barrier instruction!", MBB);
822 } else if (!MBB->back().isTerminator()) {
823 report(msg: "MBB exits via conditional branch/fall-through but the branch "
824 "isn't a terminator instruction!", MBB);
825 }
826 } else if (TBB && FBB) {
827 // Block conditionally branches somewhere, otherwise branches
828 // somewhere else.
829 if (MBB->empty()) {
830 report(msg: "MBB exits via conditional branch/branch but doesn't "
831 "contain any instructions!", MBB);
832 } else if (!MBB->back().isBarrier()) {
833 report(msg: "MBB exits via conditional branch/branch but doesn't end with a "
834 "barrier instruction!", MBB);
835 } else if (!MBB->back().isTerminator()) {
836 report(msg: "MBB exits via conditional branch/branch but the branch "
837 "isn't a terminator instruction!", MBB);
838 }
839 if (Cond.empty()) {
840 report(msg: "MBB exits via conditional branch/branch but there's no "
841 "condition!", MBB);
842 }
843 } else {
844 report(msg: "analyzeBranch returned invalid data!", MBB);
845 }
846
847 // Now check that the successors match up with the answers reported by
848 // analyzeBranch.
849 if (TBB && !MBB->isSuccessor(MBB: TBB))
850 report(msg: "MBB exits via jump or conditional branch, but its target isn't a "
851 "CFG successor!",
852 MBB);
853 if (FBB && !MBB->isSuccessor(MBB: FBB))
854 report(msg: "MBB exits via conditional branch, but its target isn't a CFG "
855 "successor!",
856 MBB);
857
858 // There might be a fallthrough to the next block if there's either no
859 // unconditional true branch, or if there's a condition, and one of the
860 // branches is missing.
861 bool Fallthrough = !TBB || (!Cond.empty() && !FBB);
862
863 // A conditional fallthrough must be an actual CFG successor, not
864 // unreachable. (Conversely, an unconditional fallthrough might not really
865 // be a successor, because the block might end in unreachable.)
866 if (!Cond.empty() && !FBB) {
867 MachineFunction::const_iterator MBBI = std::next(x: MBB->getIterator());
868 if (MBBI == MF->end()) {
869 report(msg: "MBB conditionally falls through out of function!", MBB);
870 } else if (!MBB->isSuccessor(MBB: &*MBBI))
871 report(msg: "MBB exits via conditional branch/fall-through but the CFG "
872 "successors don't match the actual successors!",
873 MBB);
874 }
875
876 // Verify that there aren't any extra un-accounted-for successors.
877 for (const MachineBasicBlock *SuccMBB : MBB->successors()) {
878 // If this successor is one of the branch targets, it's okay.
879 if (SuccMBB == TBB || SuccMBB == FBB)
880 continue;
881 // If we might have a fallthrough, and the successor is the fallthrough
882 // block, that's also ok.
883 if (Fallthrough && SuccMBB == MBB->getNextNode())
884 continue;
885 // Also accept successors which are for exception-handling or might be
886 // inlineasm_br targets.
887 if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
888 continue;
889 report(msg: "MBB has unexpected successors which are not branch targets, "
890 "fallthrough, EHPads, or inlineasm_br targets.",
891 MBB);
892 }
893 }
894
895 regsLive.clear();
896 if (MRI->tracksLiveness()) {
897 for (const auto &LI : MBB->liveins()) {
898 if (!LI.PhysReg.isPhysical()) {
899 report(msg: "MBB live-in list contains non-physical register", MBB);
900 continue;
901 }
902 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: LI.PhysReg));
903 }
904 }
905
906 const MachineFrameInfo &MFI = MF->getFrameInfo();
907 BitVector PR = MFI.getPristineRegs(MF: *MF);
908 for (unsigned I : PR.set_bits())
909 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: I));
910
911 regsKilled.clear();
912 regsDefined.clear();
913
914 if (Indexes)
915 lastIndex = Indexes->getMBBStartIdx(mbb: MBB);
916}
917
918// This function gets called for all bundle headers, including normal
919// stand-alone unbundled instructions.
920void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
921 if (Indexes && Indexes->hasIndex(instr: *MI)) {
922 SlotIndex idx = Indexes->getInstructionIndex(MI: *MI);
923 if (!(idx > lastIndex)) {
924 report(msg: "Instruction index out of order", MI);
925 OS << "Last instruction was at " << lastIndex << '\n';
926 }
927 lastIndex = idx;
928 }
929
930 // Ensure non-terminators don't follow terminators.
931 if (MI->isTerminator()) {
932 if (!FirstTerminator)
933 FirstTerminator = MI;
934 } else if (FirstTerminator) {
935 // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to
936 // precede non-terminators.
937 if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) {
938 report(msg: "Non-terminator instruction after the first terminator", MI);
939 OS << "First terminator was:\t" << *FirstTerminator;
940 }
941 }
942}
943
944// The operands on an INLINEASM instruction must follow a template.
945// Verify that the flag operands make sense.
946void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
947 // The first two operands on INLINEASM are the asm string and global flags.
948 if (MI->getNumOperands() < 2) {
949 report(msg: "Too few operands on inline asm", MI);
950 return;
951 }
952 if (!MI->getOperand(i: 0).isSymbol())
953 report(msg: "Asm string must be an external symbol", MI);
954 if (!MI->getOperand(i: 1).isImm())
955 report(msg: "Asm flags must be an immediate", MI);
956 // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
957 // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,
958 // and Extra_IsConvergent = 32.
959 if (!isUInt<6>(x: MI->getOperand(i: 1).getImm()))
960 report(msg: "Unknown asm flags", MO: &MI->getOperand(i: 1), MONum: 1);
961
962 static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");
963
964 unsigned OpNo = InlineAsm::MIOp_FirstOperand;
965 unsigned NumOps;
966 for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
967 const MachineOperand &MO = MI->getOperand(i: OpNo);
968 // There may be implicit ops after the fixed operands.
969 if (!MO.isImm())
970 break;
971 const InlineAsm::Flag F(MO.getImm());
972 NumOps = 1 + F.getNumOperandRegisters();
973 }
974
975 if (OpNo > MI->getNumOperands())
976 report(msg: "Missing operands in last group", MI);
977
978 // An optional MDNode follows the groups.
979 if (OpNo < MI->getNumOperands() && MI->getOperand(i: OpNo).isMetadata())
980 ++OpNo;
981
982 // All trailing operands must be implicit registers.
983 for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
984 const MachineOperand &MO = MI->getOperand(i: OpNo);
985 if (!MO.isReg() || !MO.isImplicit())
986 report(msg: "Expected implicit register after groups", MO: &MO, MONum: OpNo);
987 }
988
989 if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) {
990 const MachineBasicBlock *MBB = MI->getParent();
991
992 for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands();
993 i != e; ++i) {
994 const MachineOperand &MO = MI->getOperand(i);
995
996 if (!MO.isMBB())
997 continue;
998
999 // Check the successor & predecessor lists look ok, assume they are
1000 // not. Find the indirect target without going through the successors.
1001 const MachineBasicBlock *IndirectTargetMBB = MO.getMBB();
1002 if (!IndirectTargetMBB) {
1003 report(msg: "INLINEASM_BR indirect target does not exist", MO: &MO, MONum: i);
1004 break;
1005 }
1006
1007 if (!MBB->isSuccessor(MBB: IndirectTargetMBB))
1008 report(msg: "INLINEASM_BR indirect target missing from successor list", MO: &MO,
1009 MONum: i);
1010
1011 if (!IndirectTargetMBB->isPredecessor(MBB))
1012 report(msg: "INLINEASM_BR indirect target predecessor list missing parent",
1013 MO: &MO, MONum: i);
1014 }
1015 }
1016}
1017
1018bool MachineVerifier::verifyAllRegOpsScalar(const MachineInstr &MI,
1019 const MachineRegisterInfo &MRI) {
1020 if (none_of(Range: MI.explicit_operands(), P: [&MRI](const MachineOperand &Op) {
1021 if (!Op.isReg())
1022 return false;
1023 const auto Reg = Op.getReg();
1024 if (Reg.isPhysical())
1025 return false;
1026 return !MRI.getType(Reg).isScalar();
1027 }))
1028 return true;
1029 report(msg: "All register operands must have scalar types", MI: &MI);
1030 return false;
1031}
1032
1033/// Check that types are consistent when two operands need to have the same
1034/// number of vector elements.
1035/// \return true if the types are valid.
1036bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,
1037 const MachineInstr *MI) {
1038 if (Ty0.isVector() != Ty1.isVector()) {
1039 report(msg: "operand types must be all-vector or all-scalar", MI);
1040 // Generally we try to report as many issues as possible at once, but in
1041 // this case it's not clear what should we be comparing the size of the
1042 // scalar with: the size of the whole vector or its lane. Instead of
1043 // making an arbitrary choice and emitting not so helpful message, let's
1044 // avoid the extra noise and stop here.
1045 return false;
1046 }
1047
1048 if (Ty0.isVector() && Ty0.getElementCount() != Ty1.getElementCount()) {
1049 report(msg: "operand types must preserve number of vector elements", MI);
1050 return false;
1051 }
1052
1053 return true;
1054}
1055
1056bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
1057 auto Opcode = MI->getOpcode();
1058 bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
1059 Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
1060 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1061 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1062 AttributeSet Attrs = Intrinsic::getFnAttributes(
1063 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1064 bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
1065 if (NoSideEffects && DeclHasSideEffects) {
1066 report(Msg: Twine(TII->getName(Opcode),
1067 " used with intrinsic that accesses memory"),
1068 MI);
1069 return false;
1070 }
1071 if (!NoSideEffects && !DeclHasSideEffects) {
1072 report(Msg: Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI);
1073 return false;
1074 }
1075 }
1076
1077 return true;
1078}
1079
1080bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
1081 auto Opcode = MI->getOpcode();
1082 bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
1083 Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
1084 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1085 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1086 AttributeSet Attrs = Intrinsic::getFnAttributes(
1087 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1088 bool DeclIsConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
1089 if (NotConvergent && DeclIsConvergent) {
1090 report(Msg: Twine(TII->getName(Opcode), " used with a convergent intrinsic"),
1091 MI);
1092 return false;
1093 }
1094 if (!NotConvergent && !DeclIsConvergent) {
1095 report(
1096 Msg: Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"),
1097 MI);
1098 return false;
1099 }
1100 }
1101
1102 return true;
1103}
1104
1105void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
1106 if (isFunctionSelected)
1107 report(msg: "Unexpected generic instruction in a Selected function", MI);
1108
1109 const MCInstrDesc &MCID = MI->getDesc();
1110 unsigned NumOps = MI->getNumOperands();
1111
1112 // Branches must reference a basic block if they are not indirect
1113 if (MI->isBranch() && !MI->isIndirectBranch()) {
1114 bool HasMBB = false;
1115 for (const MachineOperand &Op : MI->operands()) {
1116 if (Op.isMBB()) {
1117 HasMBB = true;
1118 break;
1119 }
1120 }
1121
1122 if (!HasMBB) {
1123 report(msg: "Branch instruction is missing a basic block operand or "
1124 "isIndirectBranch property",
1125 MI);
1126 }
1127 }
1128
1129 // Check types.
1130 SmallVector<LLT, 4> Types;
1131 for (unsigned I = 0, E = std::min(a: MCID.getNumOperands(), b: NumOps);
1132 I != E; ++I) {
1133 if (!MCID.operands()[I].isGenericType())
1134 continue;
1135 // Generic instructions specify type equality constraints between some of
1136 // their operands. Make sure these are consistent.
1137 size_t TypeIdx = MCID.operands()[I].getGenericTypeIndex();
1138 Types.resize(N: std::max(a: TypeIdx + 1, b: Types.size()));
1139
1140 const MachineOperand *MO = &MI->getOperand(i: I);
1141 if (!MO->isReg()) {
1142 report(msg: "generic instruction must use register operands", MI);
1143 continue;
1144 }
1145
1146 LLT OpTy = MRI->getType(Reg: MO->getReg());
1147 // Don't report a type mismatch if there is no actual mismatch, only a
1148 // type missing, to reduce noise:
1149 if (OpTy.isValid()) {
1150 // Only the first valid type for a type index will be printed: don't
1151 // overwrite it later so it's always clear which type was expected:
1152 if (!Types[TypeIdx].isValid())
1153 Types[TypeIdx] = OpTy;
1154 else if (Types[TypeIdx] != OpTy)
1155 report(msg: "Type mismatch in generic instruction", MO, MONum: I, MOVRegType: OpTy);
1156 } else {
1157 // Generic instructions must have types attached to their operands.
1158 report(msg: "Generic instruction is missing a virtual register type", MO, MONum: I);
1159 }
1160 }
1161
1162 // Generic opcodes must not have physical register operands.
1163 for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
1164 const MachineOperand *MO = &MI->getOperand(i: I);
1165 if (MO->isReg() && MO->getReg().isPhysical())
1166 report(msg: "Generic instruction cannot have physical register", MO, MONum: I);
1167 }
1168
1169 // Avoid out of bounds in checks below. This was already reported earlier.
1170 if (MI->getNumOperands() < MCID.getNumOperands())
1171 return;
1172
1173 StringRef ErrorInfo;
1174 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
1175 report(msg: ErrorInfo.data(), MI);
1176
1177 // Verify properties of various specific instruction types
1178 unsigned Opc = MI->getOpcode();
1179 switch (Opc) {
1180 case TargetOpcode::G_ASSERT_SEXT:
1181 case TargetOpcode::G_ASSERT_ZEXT: {
1182 std::string OpcName =
1183 Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT";
1184 if (!MI->getOperand(i: 2).isImm()) {
1185 report(Msg: Twine(OpcName, " expects an immediate operand #2"), MI);
1186 break;
1187 }
1188
1189 Register Dst = MI->getOperand(i: 0).getReg();
1190 Register Src = MI->getOperand(i: 1).getReg();
1191 LLT SrcTy = MRI->getType(Reg: Src);
1192 int64_t Imm = MI->getOperand(i: 2).getImm();
1193 if (Imm <= 0) {
1194 report(Msg: Twine(OpcName, " size must be >= 1"), MI);
1195 break;
1196 }
1197
1198 if (Imm >= SrcTy.getScalarSizeInBits()) {
1199 report(Msg: Twine(OpcName, " size must be less than source bit width"), MI);
1200 break;
1201 }
1202
1203 const RegisterBank *SrcRB = RBI->getRegBank(Reg: Src, MRI: *MRI, TRI: *TRI);
1204 const RegisterBank *DstRB = RBI->getRegBank(Reg: Dst, MRI: *MRI, TRI: *TRI);
1205
1206 // Allow only the source bank to be set.
1207 if ((SrcRB && DstRB && SrcRB != DstRB) || (DstRB && !SrcRB)) {
1208 report(Msg: Twine(OpcName, " cannot change register bank"), MI);
1209 break;
1210 }
1211
1212 // Don't allow a class change. Do allow member class->regbank.
1213 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: Dst);
1214 if (DstRC && DstRC != MRI->getRegClassOrNull(Reg: Src)) {
1215 report(
1216 Msg: Twine(OpcName, " source and destination register classes must match"),
1217 MI);
1218 break;
1219 }
1220
1221 break;
1222 }
1223
1224 case TargetOpcode::G_CONSTANT:
1225 case TargetOpcode::G_FCONSTANT: {
1226 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1227 if (DstTy.isVector())
1228 report(msg: "Instruction cannot use a vector result type", MI);
1229
1230 if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {
1231 if (!MI->getOperand(i: 1).isCImm()) {
1232 report(msg: "G_CONSTANT operand must be cimm", MI);
1233 break;
1234 }
1235
1236 const ConstantInt *CI = MI->getOperand(i: 1).getCImm();
1237 if (CI->getBitWidth() != DstTy.getSizeInBits())
1238 report(msg: "inconsistent constant size", MI);
1239 } else {
1240 if (!MI->getOperand(i: 1).isFPImm()) {
1241 report(msg: "G_FCONSTANT operand must be fpimm", MI);
1242 break;
1243 }
1244 const ConstantFP *CF = MI->getOperand(i: 1).getFPImm();
1245
1246 if (APFloat::getSizeInBits(Sem: CF->getValueAPF().getSemantics()) !=
1247 DstTy.getSizeInBits()) {
1248 report(msg: "inconsistent constant size", MI);
1249 }
1250 }
1251
1252 break;
1253 }
1254 case TargetOpcode::G_LOAD:
1255 case TargetOpcode::G_STORE:
1256 case TargetOpcode::G_ZEXTLOAD:
1257 case TargetOpcode::G_SEXTLOAD: {
1258 LLT ValTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1259 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1260 if (!PtrTy.isPointer())
1261 report(msg: "Generic memory instruction must access a pointer", MI);
1262
1263 // Generic loads and stores must have a single MachineMemOperand
1264 // describing that access.
1265 if (!MI->hasOneMemOperand()) {
1266 report(msg: "Generic instruction accessing memory must have one mem operand",
1267 MI);
1268 } else {
1269 const MachineMemOperand &MMO = **MI->memoperands_begin();
1270 if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD ||
1271 MI->getOpcode() == TargetOpcode::G_SEXTLOAD) {
1272 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1273 RHS: ValTy.getSizeInBits()))
1274 report(msg: "Generic extload must have a narrower memory type", MI);
1275 } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
1276 if (TypeSize::isKnownGT(LHS: MMO.getSize().getValue(),
1277 RHS: ValTy.getSizeInBytes()))
1278 report(msg: "load memory size cannot exceed result size", MI);
1279
1280 if (MMO.getRanges()) {
1281 ConstantInt *i =
1282 mdconst::extract<ConstantInt>(MD: MMO.getRanges()->getOperand(I: 0));
1283 const LLT RangeTy = LLT::scalar(SizeInBits: i->getIntegerType()->getBitWidth());
1284 const LLT MemTy = MMO.getMemoryType();
1285 if (MemTy.getScalarType() != RangeTy ||
1286 ValTy.isScalar() != MemTy.isScalar() ||
1287 (ValTy.isVector() &&
1288 ValTy.getNumElements() != MemTy.getNumElements())) {
1289 report(msg: "range is incompatible with the result type", MI);
1290 }
1291 }
1292 } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
1293 if (TypeSize::isKnownLT(LHS: ValTy.getSizeInBytes(),
1294 RHS: MMO.getSize().getValue()))
1295 report(msg: "store memory size cannot exceed value size", MI);
1296 }
1297
1298 const AtomicOrdering Order = MMO.getSuccessOrdering();
1299 if (Opc == TargetOpcode::G_STORE) {
1300 if (Order == AtomicOrdering::Acquire ||
1301 Order == AtomicOrdering::AcquireRelease)
1302 report(msg: "atomic store cannot use acquire ordering", MI);
1303
1304 } else {
1305 if (Order == AtomicOrdering::Release ||
1306 Order == AtomicOrdering::AcquireRelease)
1307 report(msg: "atomic load cannot use release ordering", MI);
1308 }
1309 }
1310
1311 break;
1312 }
1313 case TargetOpcode::G_PHI: {
1314 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1315 if (!DstTy.isValid() || !all_of(Range: drop_begin(RangeOrContainer: MI->operands()),
1316 P: [this, &DstTy](const MachineOperand &MO) {
1317 if (!MO.isReg())
1318 return true;
1319 LLT Ty = MRI->getType(Reg: MO.getReg());
1320 if (!Ty.isValid() || (Ty != DstTy))
1321 return false;
1322 return true;
1323 }))
1324 report(msg: "Generic Instruction G_PHI has operands with incompatible/missing "
1325 "types",
1326 MI);
1327 break;
1328 }
1329 case TargetOpcode::G_BITCAST: {
1330 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1331 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1332 if (!DstTy.isValid() || !SrcTy.isValid())
1333 break;
1334
1335 if (SrcTy.isPointer() != DstTy.isPointer())
1336 report(msg: "bitcast cannot convert between pointers and other types", MI);
1337
1338 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1339 report(msg: "bitcast sizes must match", MI);
1340
1341 if (SrcTy == DstTy)
1342 report(msg: "bitcast must change the type", MI);
1343
1344 break;
1345 }
1346 case TargetOpcode::G_INTTOPTR:
1347 case TargetOpcode::G_PTRTOINT:
1348 case TargetOpcode::G_ADDRSPACE_CAST: {
1349 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1350 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1351 if (!DstTy.isValid() || !SrcTy.isValid())
1352 break;
1353
1354 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1355
1356 DstTy = DstTy.getScalarType();
1357 SrcTy = SrcTy.getScalarType();
1358
1359 if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {
1360 if (!DstTy.isPointer())
1361 report(msg: "inttoptr result type must be a pointer", MI);
1362 if (SrcTy.isPointer())
1363 report(msg: "inttoptr source type must not be a pointer", MI);
1364 } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {
1365 if (!SrcTy.isPointer())
1366 report(msg: "ptrtoint source type must be a pointer", MI);
1367 if (DstTy.isPointer())
1368 report(msg: "ptrtoint result type must not be a pointer", MI);
1369 } else {
1370 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
1371 if (!SrcTy.isPointer() || !DstTy.isPointer())
1372 report(msg: "addrspacecast types must be pointers", MI);
1373 else {
1374 if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())
1375 report(msg: "addrspacecast must convert different address spaces", MI);
1376 }
1377 }
1378
1379 break;
1380 }
1381 case TargetOpcode::G_PTR_ADD: {
1382 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1383 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1384 LLT OffsetTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1385 if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())
1386 break;
1387
1388 if (!PtrTy.isPointerOrPointerVector())
1389 report(msg: "gep first operand must be a pointer", MI);
1390
1391 if (OffsetTy.isPointerOrPointerVector())
1392 report(msg: "gep offset operand must not be a pointer", MI);
1393
1394 if (PtrTy.isPointerOrPointerVector()) {
1395 const DataLayout &DL = MF->getDataLayout();
1396 unsigned AS = PtrTy.getAddressSpace();
1397 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
1398 if (OffsetTy.getScalarSizeInBits() != IndexSizeInBits) {
1399 report(msg: "gep offset operand must match index size for address space",
1400 MI);
1401 }
1402 }
1403
1404 // TODO: Is the offset allowed to be a scalar with a vector?
1405 break;
1406 }
1407 case TargetOpcode::G_PTRMASK: {
1408 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1409 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1410 LLT MaskTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1411 if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid())
1412 break;
1413
1414 if (!DstTy.isPointerOrPointerVector())
1415 report(msg: "ptrmask result type must be a pointer", MI);
1416
1417 if (!MaskTy.getScalarType().isScalar())
1418 report(msg: "ptrmask mask type must be an integer", MI);
1419
1420 verifyVectorElementMatch(Ty0: DstTy, Ty1: MaskTy, MI);
1421 break;
1422 }
1423 case TargetOpcode::G_SEXT:
1424 case TargetOpcode::G_ZEXT:
1425 case TargetOpcode::G_ANYEXT:
1426 case TargetOpcode::G_TRUNC:
1427 case TargetOpcode::G_FPEXT:
1428 case TargetOpcode::G_FPTRUNC: {
1429 // Number of operands and presense of types is already checked (and
1430 // reported in case of any issues), so no need to report them again. As
1431 // we're trying to report as many issues as possible at once, however, the
1432 // instructions aren't guaranteed to have the right number of operands or
1433 // types attached to them at this point
1434 assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
1435 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1436 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1437 if (!DstTy.isValid() || !SrcTy.isValid())
1438 break;
1439
1440 if (DstTy.isPointerOrPointerVector() || SrcTy.isPointerOrPointerVector())
1441 report(msg: "Generic extend/truncate can not operate on pointers", MI);
1442
1443 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1444
1445 unsigned DstSize = DstTy.getScalarSizeInBits();
1446 unsigned SrcSize = SrcTy.getScalarSizeInBits();
1447 switch (MI->getOpcode()) {
1448 default:
1449 if (DstSize <= SrcSize)
1450 report(msg: "Generic extend has destination type no larger than source", MI);
1451 break;
1452 case TargetOpcode::G_TRUNC:
1453 case TargetOpcode::G_FPTRUNC:
1454 if (DstSize >= SrcSize)
1455 report(msg: "Generic truncate has destination type no smaller than source",
1456 MI);
1457 break;
1458 }
1459 break;
1460 }
1461 case TargetOpcode::G_SELECT: {
1462 LLT SelTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1463 LLT CondTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1464 if (!SelTy.isValid() || !CondTy.isValid())
1465 break;
1466
1467 // Scalar condition select on a vector is valid.
1468 if (CondTy.isVector())
1469 verifyVectorElementMatch(Ty0: SelTy, Ty1: CondTy, MI);
1470 break;
1471 }
1472 case TargetOpcode::G_MERGE_VALUES: {
1473 // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,
1474 // e.g. s2N = MERGE sN, sN
1475 // Merging multiple scalars into a vector is not allowed, should use
1476 // G_BUILD_VECTOR for that.
1477 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1478 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1479 if (DstTy.isVector() || SrcTy.isVector())
1480 report(msg: "G_MERGE_VALUES cannot operate on vectors", MI);
1481
1482 const unsigned NumOps = MI->getNumOperands();
1483 if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1))
1484 report(msg: "G_MERGE_VALUES result size is inconsistent", MI);
1485
1486 for (unsigned I = 2; I != NumOps; ++I) {
1487 if (MRI->getType(Reg: MI->getOperand(i: I).getReg()) != SrcTy)
1488 report(msg: "G_MERGE_VALUES source types do not match", MI);
1489 }
1490
1491 break;
1492 }
1493 case TargetOpcode::G_UNMERGE_VALUES: {
1494 unsigned NumDsts = MI->getNumOperands() - 1;
1495 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1496 for (unsigned i = 1; i < NumDsts; ++i) {
1497 if (MRI->getType(Reg: MI->getOperand(i).getReg()) != DstTy) {
1498 report(msg: "G_UNMERGE_VALUES destination types do not match", MI);
1499 break;
1500 }
1501 }
1502
1503 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: NumDsts).getReg());
1504 if (DstTy.isVector()) {
1505 // This case is the converse of G_CONCAT_VECTORS.
1506 if (!SrcTy.isVector() ||
1507 (SrcTy.getScalarType() != DstTy.getScalarType() &&
1508 !SrcTy.isPointerVector()) ||
1509 SrcTy.isScalableVector() != DstTy.isScalableVector() ||
1510 SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1511 report(msg: "G_UNMERGE_VALUES source operand does not match vector "
1512 "destination operands",
1513 MI);
1514 } else if (SrcTy.isVector()) {
1515 // This case is the converse of G_BUILD_VECTOR, but relaxed to allow
1516 // mismatched types as long as the total size matches:
1517 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<4 x s32>)
1518 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1519 report(msg: "G_UNMERGE_VALUES vector source operand does not match scalar "
1520 "destination operands",
1521 MI);
1522 } else {
1523 // This case is the converse of G_MERGE_VALUES.
1524 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits()) {
1525 report(msg: "G_UNMERGE_VALUES scalar source operand does not match scalar "
1526 "destination operands",
1527 MI);
1528 }
1529 }
1530 break;
1531 }
1532 case TargetOpcode::G_BUILD_VECTOR: {
1533 // Source types must be scalars, dest type a vector. Total size of scalars
1534 // must match the dest vector size.
1535 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1536 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1537 if (!DstTy.isVector() || SrcEltTy.isVector()) {
1538 report(msg: "G_BUILD_VECTOR must produce a vector from scalar operands", MI);
1539 break;
1540 }
1541
1542 if (DstTy.getElementType() != SrcEltTy)
1543 report(msg: "G_BUILD_VECTOR result element type must match source type", MI);
1544
1545 if (DstTy.getNumElements() != MI->getNumOperands() - 1)
1546 report(msg: "G_BUILD_VECTOR must have an operand for each elemement", MI);
1547
1548 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1549 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1550 report(msg: "G_BUILD_VECTOR source operand types are not homogeneous", MI);
1551
1552 break;
1553 }
1554 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1555 // Source types must be scalars, dest type a vector. Scalar types must be
1556 // larger than the dest vector elt type, as this is a truncating operation.
1557 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1558 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1559 if (!DstTy.isVector() || SrcEltTy.isVector())
1560 report(msg: "G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands",
1561 MI);
1562 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1563 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1564 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not homogeneous",
1565 MI);
1566 if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits())
1567 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not larger than "
1568 "dest elt type",
1569 MI);
1570 break;
1571 }
1572 case TargetOpcode::G_CONCAT_VECTORS: {
1573 // Source types should be vectors, and total size should match the dest
1574 // vector size.
1575 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1576 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1577 if (!DstTy.isVector() || !SrcTy.isVector())
1578 report(msg: "G_CONCAT_VECTOR requires vector source and destination operands",
1579 MI);
1580
1581 if (MI->getNumOperands() < 3)
1582 report(msg: "G_CONCAT_VECTOR requires at least 2 source operands", MI);
1583
1584 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1585 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1586 report(msg: "G_CONCAT_VECTOR source operand types are not homogeneous", MI);
1587 if (DstTy.getElementCount() !=
1588 SrcTy.getElementCount() * (MI->getNumOperands() - 1))
1589 report(msg: "G_CONCAT_VECTOR num dest and source elements should match", MI);
1590 break;
1591 }
1592 case TargetOpcode::G_ICMP:
1593 case TargetOpcode::G_FCMP: {
1594 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1595 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1596
1597 if ((DstTy.isVector() != SrcTy.isVector()) ||
1598 (DstTy.isVector() &&
1599 DstTy.getElementCount() != SrcTy.getElementCount()))
1600 report(msg: "Generic vector icmp/fcmp must preserve number of lanes", MI);
1601
1602 break;
1603 }
1604 case TargetOpcode::G_SCMP:
1605 case TargetOpcode::G_UCMP: {
1606 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1607 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1608
1609 if (SrcTy.isPointerOrPointerVector()) {
1610 report(msg: "Generic scmp/ucmp does not support pointers as operands", MI);
1611 break;
1612 }
1613
1614 if (DstTy.isPointerOrPointerVector()) {
1615 report(msg: "Generic scmp/ucmp does not support pointers as a result", MI);
1616 break;
1617 }
1618
1619 if (DstTy.getScalarSizeInBits() < 2) {
1620 report(msg: "Result type must be at least 2 bits wide", MI);
1621 break;
1622 }
1623
1624 if ((DstTy.isVector() != SrcTy.isVector()) ||
1625 (DstTy.isVector() &&
1626 DstTy.getElementCount() != SrcTy.getElementCount())) {
1627 report(msg: "Generic vector scmp/ucmp must preserve number of lanes", MI);
1628 break;
1629 }
1630
1631 break;
1632 }
1633 case TargetOpcode::G_EXTRACT: {
1634 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1635 if (!SrcOp.isReg()) {
1636 report(msg: "extract source must be a register", MI);
1637 break;
1638 }
1639
1640 const MachineOperand &OffsetOp = MI->getOperand(i: 2);
1641 if (!OffsetOp.isImm()) {
1642 report(msg: "extract offset must be a constant", MI);
1643 break;
1644 }
1645
1646 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1647 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1648 if (SrcSize == DstSize)
1649 report(msg: "extract source must be larger than result", MI);
1650
1651 if (DstSize + OffsetOp.getImm() > SrcSize)
1652 report(msg: "extract reads past end of register", MI);
1653 break;
1654 }
1655 case TargetOpcode::G_INSERT: {
1656 const MachineOperand &SrcOp = MI->getOperand(i: 2);
1657 if (!SrcOp.isReg()) {
1658 report(msg: "insert source must be a register", MI);
1659 break;
1660 }
1661
1662 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
1663 if (!OffsetOp.isImm()) {
1664 report(msg: "insert offset must be a constant", MI);
1665 break;
1666 }
1667
1668 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1669 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1670
1671 if (DstSize <= SrcSize)
1672 report(msg: "inserted size must be smaller than total register", MI);
1673
1674 if (SrcSize + OffsetOp.getImm() > DstSize)
1675 report(msg: "insert writes past end of register", MI);
1676
1677 break;
1678 }
1679 case TargetOpcode::G_JUMP_TABLE: {
1680 if (!MI->getOperand(i: 1).isJTI())
1681 report(msg: "G_JUMP_TABLE source operand must be a jump table index", MI);
1682 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1683 if (!DstTy.isPointer())
1684 report(msg: "G_JUMP_TABLE dest operand must have a pointer type", MI);
1685 break;
1686 }
1687 case TargetOpcode::G_BRJT: {
1688 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
1689 report(msg: "G_BRJT src operand 0 must be a pointer type", MI);
1690
1691 if (!MI->getOperand(i: 1).isJTI())
1692 report(msg: "G_BRJT src operand 1 must be a jump table index", MI);
1693
1694 const auto &IdxOp = MI->getOperand(i: 2);
1695 if (!IdxOp.isReg() || MRI->getType(Reg: IdxOp.getReg()).isPointer())
1696 report(msg: "G_BRJT src operand 2 must be a scalar reg type", MI);
1697 break;
1698 }
1699 case TargetOpcode::G_INTRINSIC:
1700 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1701 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1702 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
1703 // TODO: Should verify number of def and use operands, but the current
1704 // interface requires passing in IR types for mangling.
1705 const MachineOperand &IntrIDOp = MI->getOperand(i: MI->getNumExplicitDefs());
1706 if (!IntrIDOp.isIntrinsicID()) {
1707 report(msg: "G_INTRINSIC first src operand must be an intrinsic ID", MI);
1708 break;
1709 }
1710
1711 if (!verifyGIntrinsicSideEffects(MI))
1712 break;
1713 if (!verifyGIntrinsicConvergence(MI))
1714 break;
1715
1716 break;
1717 }
1718 case TargetOpcode::G_SEXT_INREG: {
1719 if (!MI->getOperand(i: 2).isImm()) {
1720 report(msg: "G_SEXT_INREG expects an immediate operand #2", MI);
1721 break;
1722 }
1723
1724 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1725 int64_t Imm = MI->getOperand(i: 2).getImm();
1726 if (Imm <= 0)
1727 report(msg: "G_SEXT_INREG size must be >= 1", MI);
1728 if (Imm >= SrcTy.getScalarSizeInBits())
1729 report(msg: "G_SEXT_INREG size must be less than source bit width", MI);
1730 break;
1731 }
1732 case TargetOpcode::G_BSWAP: {
1733 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1734 if (DstTy.getScalarSizeInBits() % 16 != 0)
1735 report(msg: "G_BSWAP size must be a multiple of 16 bits", MI);
1736 break;
1737 }
1738 case TargetOpcode::G_VSCALE: {
1739 if (!MI->getOperand(i: 1).isCImm()) {
1740 report(msg: "G_VSCALE operand must be cimm", MI);
1741 break;
1742 }
1743 if (MI->getOperand(i: 1).getCImm()->isZero()) {
1744 report(msg: "G_VSCALE immediate cannot be zero", MI);
1745 break;
1746 }
1747 break;
1748 }
1749 case TargetOpcode::G_STEP_VECTOR: {
1750 if (!MI->getOperand(i: 1).isCImm()) {
1751 report(msg: "operand must be cimm", MI);
1752 break;
1753 }
1754
1755 if (!MI->getOperand(i: 1).getCImm()->getValue().isStrictlyPositive()) {
1756 report(msg: "step must be > 0", MI);
1757 break;
1758 }
1759
1760 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1761 if (!DstTy.isScalableVector()) {
1762 report(msg: "Destination type must be a scalable vector", MI);
1763 break;
1764 }
1765
1766 // <vscale x 2 x p0>
1767 if (!DstTy.getElementType().isScalar()) {
1768 report(msg: "Destination element type must be scalar", MI);
1769 break;
1770 }
1771
1772 if (MI->getOperand(i: 1).getCImm()->getBitWidth() !=
1773 DstTy.getElementType().getScalarSizeInBits()) {
1774 report(msg: "step bitwidth differs from result type element bitwidth", MI);
1775 break;
1776 }
1777 break;
1778 }
1779 case TargetOpcode::G_INSERT_SUBVECTOR: {
1780 const MachineOperand &Src0Op = MI->getOperand(i: 1);
1781 if (!Src0Op.isReg()) {
1782 report(msg: "G_INSERT_SUBVECTOR first source must be a register", MI);
1783 break;
1784 }
1785
1786 const MachineOperand &Src1Op = MI->getOperand(i: 2);
1787 if (!Src1Op.isReg()) {
1788 report(msg: "G_INSERT_SUBVECTOR second source must be a register", MI);
1789 break;
1790 }
1791
1792 const MachineOperand &IndexOp = MI->getOperand(i: 3);
1793 if (!IndexOp.isImm()) {
1794 report(msg: "G_INSERT_SUBVECTOR index must be an immediate", MI);
1795 break;
1796 }
1797
1798 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1799 LLT Src1Ty = MRI->getType(Reg: Src1Op.getReg());
1800
1801 if (!DstTy.isVector()) {
1802 report(msg: "Destination type must be a vector", MI);
1803 break;
1804 }
1805
1806 if (!Src1Ty.isVector()) {
1807 report(msg: "Second source must be a vector", MI);
1808 break;
1809 }
1810
1811 if (DstTy.getElementType() != Src1Ty.getElementType()) {
1812 report(msg: "Element type of vectors must be the same", MI);
1813 break;
1814 }
1815
1816 if (Src1Ty.isScalable() != DstTy.isScalable()) {
1817 report(msg: "Vector types must both be fixed or both be scalable", MI);
1818 break;
1819 }
1820
1821 if (ElementCount::isKnownGT(LHS: Src1Ty.getElementCount(),
1822 RHS: DstTy.getElementCount())) {
1823 report(msg: "Second source must be smaller than destination vector", MI);
1824 break;
1825 }
1826
1827 uint64_t Idx = IndexOp.getImm();
1828 uint64_t Src1MinLen = Src1Ty.getElementCount().getKnownMinValue();
1829 if (IndexOp.getImm() % Src1MinLen != 0) {
1830 report(msg: "Index must be a multiple of the second source vector's "
1831 "minimum vector length",
1832 MI);
1833 break;
1834 }
1835
1836 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1837 if (Idx >= DstMinLen || Idx + Src1MinLen > DstMinLen) {
1838 report(msg: "Subvector type and index must not cause insert to overrun the "
1839 "vector being inserted into",
1840 MI);
1841 break;
1842 }
1843
1844 break;
1845 }
1846 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1847 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1848 if (!SrcOp.isReg()) {
1849 report(msg: "G_EXTRACT_SUBVECTOR first source must be a register", MI);
1850 break;
1851 }
1852
1853 const MachineOperand &IndexOp = MI->getOperand(i: 2);
1854 if (!IndexOp.isImm()) {
1855 report(msg: "G_EXTRACT_SUBVECTOR index must be an immediate", MI);
1856 break;
1857 }
1858
1859 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1860 LLT SrcTy = MRI->getType(Reg: SrcOp.getReg());
1861
1862 if (!DstTy.isVector()) {
1863 report(msg: "Destination type must be a vector", MI);
1864 break;
1865 }
1866
1867 if (!SrcTy.isVector()) {
1868 report(msg: "Source must be a vector", MI);
1869 break;
1870 }
1871
1872 if (DstTy.getElementType() != SrcTy.getElementType()) {
1873 report(msg: "Element type of vectors must be the same", MI);
1874 break;
1875 }
1876
1877 if (SrcTy.isScalable() != DstTy.isScalable()) {
1878 report(msg: "Vector types must both be fixed or both be scalable", MI);
1879 break;
1880 }
1881
1882 if (ElementCount::isKnownGT(LHS: DstTy.getElementCount(),
1883 RHS: SrcTy.getElementCount())) {
1884 report(msg: "Destination vector must be smaller than source vector", MI);
1885 break;
1886 }
1887
1888 uint64_t Idx = IndexOp.getImm();
1889 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1890 if (Idx % DstMinLen != 0) {
1891 report(msg: "Index must be a multiple of the destination vector's minimum "
1892 "vector length",
1893 MI);
1894 break;
1895 }
1896
1897 uint64_t SrcMinLen = SrcTy.getElementCount().getKnownMinValue();
1898 if (Idx >= SrcMinLen || Idx + DstMinLen > SrcMinLen) {
1899 report(msg: "Destination type and index must not cause extract to overrun the "
1900 "source vector",
1901 MI);
1902 break;
1903 }
1904
1905 break;
1906 }
1907 case TargetOpcode::G_SHUFFLE_VECTOR: {
1908 const MachineOperand &MaskOp = MI->getOperand(i: 3);
1909 if (!MaskOp.isShuffleMask()) {
1910 report(msg: "Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);
1911 break;
1912 }
1913
1914 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1915 LLT Src0Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1916 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1917
1918 if (Src0Ty != Src1Ty)
1919 report(msg: "Source operands must be the same type", MI);
1920
1921 if (Src0Ty.getScalarType() != DstTy.getScalarType())
1922 report(msg: "G_SHUFFLE_VECTOR cannot change element type", MI);
1923
1924 // Don't check that all operands are vector because scalars are used in
1925 // place of 1 element vectors.
1926 int SrcNumElts = Src0Ty.isVector() ? Src0Ty.getNumElements() : 1;
1927 int DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
1928
1929 ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();
1930
1931 if (static_cast<int>(MaskIdxes.size()) != DstNumElts)
1932 report(msg: "Wrong result type for shufflemask", MI);
1933
1934 for (int Idx : MaskIdxes) {
1935 if (Idx < 0)
1936 continue;
1937
1938 if (Idx >= 2 * SrcNumElts)
1939 report(msg: "Out of bounds shuffle index", MI);
1940 }
1941
1942 break;
1943 }
1944
1945 case TargetOpcode::G_SPLAT_VECTOR: {
1946 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1947 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1948
1949 if (!DstTy.isScalableVector()) {
1950 report(msg: "Destination type must be a scalable vector", MI);
1951 break;
1952 }
1953
1954 if (!SrcTy.isScalar() && !SrcTy.isPointer()) {
1955 report(msg: "Source type must be a scalar or pointer", MI);
1956 break;
1957 }
1958
1959 if (TypeSize::isKnownGT(LHS: DstTy.getElementType().getSizeInBits(),
1960 RHS: SrcTy.getSizeInBits())) {
1961 report(msg: "Element type of the destination must be the same size or smaller "
1962 "than the source type",
1963 MI);
1964 break;
1965 }
1966
1967 break;
1968 }
1969 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1970 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1971 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1972 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1973
1974 if (!DstTy.isScalar() && !DstTy.isPointer()) {
1975 report(msg: "Destination type must be a scalar or pointer", MI);
1976 break;
1977 }
1978
1979 if (!SrcTy.isVector()) {
1980 report(msg: "First source must be a vector", MI);
1981 break;
1982 }
1983
1984 auto TLI = MF->getSubtarget().getTargetLowering();
1985 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
1986 report(msg: "Index type must match VectorIdxTy", MI);
1987 break;
1988 }
1989
1990 break;
1991 }
1992 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1993 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1994 LLT VecTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1995 LLT ScaTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1996 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 3).getReg());
1997
1998 if (!DstTy.isVector()) {
1999 report(msg: "Destination type must be a vector", MI);
2000 break;
2001 }
2002
2003 if (VecTy != DstTy) {
2004 report(msg: "Destination type and vector type must match", MI);
2005 break;
2006 }
2007
2008 if (!ScaTy.isScalar() && !ScaTy.isPointer()) {
2009 report(msg: "Inserted element must be a scalar or pointer", MI);
2010 break;
2011 }
2012
2013 auto TLI = MF->getSubtarget().getTargetLowering();
2014 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2015 report(msg: "Index type must match VectorIdxTy", MI);
2016 break;
2017 }
2018
2019 break;
2020 }
2021 case TargetOpcode::G_DYN_STACKALLOC: {
2022 const MachineOperand &DstOp = MI->getOperand(i: 0);
2023 const MachineOperand &AllocOp = MI->getOperand(i: 1);
2024 const MachineOperand &AlignOp = MI->getOperand(i: 2);
2025
2026 if (!DstOp.isReg() || !MRI->getType(Reg: DstOp.getReg()).isPointer()) {
2027 report(msg: "dst operand 0 must be a pointer type", MI);
2028 break;
2029 }
2030
2031 if (!AllocOp.isReg() || !MRI->getType(Reg: AllocOp.getReg()).isScalar()) {
2032 report(msg: "src operand 1 must be a scalar reg type", MI);
2033 break;
2034 }
2035
2036 if (!AlignOp.isImm()) {
2037 report(msg: "src operand 2 must be an immediate type", MI);
2038 break;
2039 }
2040 break;
2041 }
2042 case TargetOpcode::G_MEMCPY_INLINE:
2043 case TargetOpcode::G_MEMCPY:
2044 case TargetOpcode::G_MEMMOVE: {
2045 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2046 if (MMOs.size() != 2) {
2047 report(msg: "memcpy/memmove must have 2 memory operands", MI);
2048 break;
2049 }
2050
2051 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) ||
2052 (MMOs[1]->isStore() || !MMOs[1]->isLoad())) {
2053 report(msg: "wrong memory operand types", MI);
2054 break;
2055 }
2056
2057 if (MMOs[0]->getSize() != MMOs[1]->getSize())
2058 report(msg: "inconsistent memory operand sizes", MI);
2059
2060 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2061 LLT SrcPtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2062
2063 if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) {
2064 report(msg: "memory instruction operand must be a pointer", MI);
2065 break;
2066 }
2067
2068 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2069 report(msg: "inconsistent store address space", MI);
2070 if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())
2071 report(msg: "inconsistent load address space", MI);
2072
2073 if (Opc != TargetOpcode::G_MEMCPY_INLINE)
2074 if (!MI->getOperand(i: 3).isImm() || (MI->getOperand(i: 3).getImm() & ~1LL))
2075 report(msg: "'tail' flag (operand 3) must be an immediate 0 or 1", MI);
2076
2077 break;
2078 }
2079 case TargetOpcode::G_BZERO:
2080 case TargetOpcode::G_MEMSET: {
2081 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2082 std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";
2083 if (MMOs.size() != 1) {
2084 report(Msg: Twine(Name, " must have 1 memory operand"), MI);
2085 break;
2086 }
2087
2088 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
2089 report(Msg: Twine(Name, " memory operand must be a store"), MI);
2090 break;
2091 }
2092
2093 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2094 if (!DstPtrTy.isPointer()) {
2095 report(Msg: Twine(Name, " operand must be a pointer"), MI);
2096 break;
2097 }
2098
2099 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2100 report(Msg: "inconsistent " + Twine(Name, " address space"), MI);
2101
2102 if (!MI->getOperand(i: MI->getNumOperands() - 1).isImm() ||
2103 (MI->getOperand(i: MI->getNumOperands() - 1).getImm() & ~1LL))
2104 report(msg: "'tail' flag (last operand) must be an immediate 0 or 1", MI);
2105
2106 break;
2107 }
2108 case TargetOpcode::G_UBSANTRAP: {
2109 const MachineOperand &KindOp = MI->getOperand(i: 0);
2110 if (!MI->getOperand(i: 0).isImm()) {
2111 report(msg: "Crash kind must be an immediate", MO: &KindOp, MONum: 0);
2112 break;
2113 }
2114 int64_t Kind = MI->getOperand(i: 0).getImm();
2115 if (!isInt<8>(x: Kind))
2116 report(msg: "Crash kind must be 8 bit wide", MO: &KindOp, MONum: 0);
2117 break;
2118 }
2119 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
2120 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
2121 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2122 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2123 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2124 if (!DstTy.isScalar())
2125 report(msg: "Vector reduction requires a scalar destination type", MI);
2126 if (!Src1Ty.isScalar())
2127 report(msg: "Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI);
2128 if (!Src2Ty.isVector())
2129 report(msg: "Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);
2130 break;
2131 }
2132 case TargetOpcode::G_VECREDUCE_FADD:
2133 case TargetOpcode::G_VECREDUCE_FMUL:
2134 case TargetOpcode::G_VECREDUCE_FMAX:
2135 case TargetOpcode::G_VECREDUCE_FMIN:
2136 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
2137 case TargetOpcode::G_VECREDUCE_FMINIMUM:
2138 case TargetOpcode::G_VECREDUCE_ADD:
2139 case TargetOpcode::G_VECREDUCE_MUL:
2140 case TargetOpcode::G_VECREDUCE_AND:
2141 case TargetOpcode::G_VECREDUCE_OR:
2142 case TargetOpcode::G_VECREDUCE_XOR:
2143 case TargetOpcode::G_VECREDUCE_SMAX:
2144 case TargetOpcode::G_VECREDUCE_SMIN:
2145 case TargetOpcode::G_VECREDUCE_UMAX:
2146 case TargetOpcode::G_VECREDUCE_UMIN: {
2147 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2148 if (!DstTy.isScalar())
2149 report(msg: "Vector reduction requires a scalar destination type", MI);
2150 break;
2151 }
2152
2153 case TargetOpcode::G_SBFX:
2154 case TargetOpcode::G_UBFX: {
2155 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2156 if (DstTy.isVector()) {
2157 report(msg: "Bitfield extraction is not supported on vectors", MI);
2158 break;
2159 }
2160 break;
2161 }
2162 case TargetOpcode::G_SHL:
2163 case TargetOpcode::G_LSHR:
2164 case TargetOpcode::G_ASHR:
2165 case TargetOpcode::G_ROTR:
2166 case TargetOpcode::G_ROTL: {
2167 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2168 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2169 if (Src1Ty.isVector() != Src2Ty.isVector()) {
2170 report(msg: "Shifts and rotates require operands to be either all scalars or "
2171 "all vectors",
2172 MI);
2173 break;
2174 }
2175 break;
2176 }
2177 case TargetOpcode::G_LLROUND:
2178 case TargetOpcode::G_LROUND: {
2179 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2180 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2181 if (!DstTy.isValid() || !SrcTy.isValid())
2182 break;
2183 if (SrcTy.isPointer() || DstTy.isPointer()) {
2184 StringRef Op = SrcTy.isPointer() ? "Source" : "Destination";
2185 report(Msg: Twine(Op, " operand must not be a pointer type"), MI);
2186 } else if (SrcTy.isScalar()) {
2187 verifyAllRegOpsScalar(MI: *MI, MRI: *MRI);
2188 break;
2189 } else if (SrcTy.isVector()) {
2190 verifyVectorElementMatch(Ty0: SrcTy, Ty1: DstTy, MI);
2191 break;
2192 }
2193 break;
2194 }
2195 case TargetOpcode::G_IS_FPCLASS: {
2196 LLT DestTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2197 LLT DestEltTy = DestTy.getScalarType();
2198 if (!DestEltTy.isScalar()) {
2199 report(msg: "Destination must be a scalar or vector of scalars", MI);
2200 break;
2201 }
2202 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2203 LLT SrcEltTy = SrcTy.getScalarType();
2204 if (!SrcEltTy.isScalar()) {
2205 report(msg: "Source must be a scalar or vector of scalars", MI);
2206 break;
2207 }
2208 if (!verifyVectorElementMatch(Ty0: DestTy, Ty1: SrcTy, MI))
2209 break;
2210 const MachineOperand &TestMO = MI->getOperand(i: 2);
2211 if (!TestMO.isImm()) {
2212 report(msg: "floating-point class set (operand 2) must be an immediate", MI);
2213 break;
2214 }
2215 int64_t Test = TestMO.getImm();
2216 if (Test < 0 || Test > fcAllFlags) {
2217 report(msg: "Incorrect floating-point class set (operand 2)", MI);
2218 break;
2219 }
2220 break;
2221 }
2222 case TargetOpcode::G_PREFETCH: {
2223 const MachineOperand &AddrOp = MI->getOperand(i: 0);
2224 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer()) {
2225 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 0);
2226 break;
2227 }
2228 const MachineOperand &RWOp = MI->getOperand(i: 1);
2229 if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {
2230 report(msg: "rw operand must be an immediate 0-1", MO: &RWOp, MONum: 1);
2231 break;
2232 }
2233 const MachineOperand &LocalityOp = MI->getOperand(i: 2);
2234 if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {
2235 report(msg: "locality operand must be an immediate 0-3", MO: &LocalityOp, MONum: 2);
2236 break;
2237 }
2238 const MachineOperand &CacheTypeOp = MI->getOperand(i: 3);
2239 if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {
2240 report(msg: "cache type operand must be an immediate 0-1", MO: &CacheTypeOp, MONum: 3);
2241 break;
2242 }
2243 break;
2244 }
2245 case TargetOpcode::G_ASSERT_ALIGN: {
2246 if (MI->getOperand(i: 2).getImm() < 1)
2247 report(msg: "alignment immediate must be >= 1", MI);
2248 break;
2249 }
2250 case TargetOpcode::G_CONSTANT_POOL: {
2251 if (!MI->getOperand(i: 1).isCPI())
2252 report(msg: "Src operand 1 must be a constant pool index", MI);
2253 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
2254 report(msg: "Dst operand 0 must be a pointer", MI);
2255 break;
2256 }
2257 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {
2258 const MachineOperand &AddrOp = MI->getOperand(i: 1);
2259 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer())
2260 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 1);
2261 break;
2262 }
2263 default:
2264 break;
2265 }
2266}
2267
2268void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
2269 const MCInstrDesc &MCID = MI->getDesc();
2270 if (MI->getNumOperands() < MCID.getNumOperands()) {
2271 report(msg: "Too few operands", MI);
2272 OS << MCID.getNumOperands() << " operands expected, but "
2273 << MI->getNumOperands() << " given.\n";
2274 }
2275
2276 if (MI->getFlag(Flag: MachineInstr::NoConvergent) && !MCID.isConvergent())
2277 report(msg: "NoConvergent flag expected only on convergent instructions.", MI);
2278
2279 if (MI->isPHI()) {
2280 if (MF->getProperties().hasNoPHIs())
2281 report(msg: "Found PHI instruction with NoPHIs property set", MI);
2282
2283 if (FirstNonPHI)
2284 report(msg: "Found PHI instruction after non-PHI", MI);
2285 } else if (FirstNonPHI == nullptr)
2286 FirstNonPHI = MI;
2287
2288 // Check the tied operands.
2289 if (MI->isInlineAsm())
2290 verifyInlineAsm(MI);
2291
2292 // Check that unspillable terminators define a reg and have at most one use.
2293 if (TII->isUnspillableTerminator(MI)) {
2294 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
2295 report(msg: "Unspillable Terminator does not define a reg", MI);
2296 Register Def = MI->getOperand(i: 0).getReg();
2297 if (Def.isVirtual() && !MF->getProperties().hasNoPHIs() &&
2298 std::distance(first: MRI->use_nodbg_begin(RegNo: Def), last: MRI->use_nodbg_end()) > 1)
2299 report(msg: "Unspillable Terminator expected to have at most one use!", MI);
2300 }
2301
2302 // A fully-formed DBG_VALUE must have a location. Ignore partially formed
2303 // DBG_VALUEs: these are convenient to use in tests, but should never get
2304 // generated.
2305 if (MI->isDebugValue() && MI->getNumOperands() == 4)
2306 if (!MI->getDebugLoc())
2307 report(msg: "Missing DebugLoc for debug instruction", MI);
2308
2309 // Meta instructions should never be the subject of debug value tracking,
2310 // they don't create a value in the output program at all.
2311 if (MI->isMetaInstruction() && MI->peekDebugInstrNum())
2312 report(msg: "Metadata instruction should not have a value tracking number", MI);
2313
2314 // Check the MachineMemOperands for basic consistency.
2315 for (MachineMemOperand *Op : MI->memoperands()) {
2316 if (Op->isLoad() && !MI->mayLoad())
2317 report(msg: "Missing mayLoad flag", MI);
2318 if (Op->isStore() && !MI->mayStore())
2319 report(msg: "Missing mayStore flag", MI);
2320 }
2321
2322 // Debug values must not have a slot index.
2323 // Other instructions must have one, unless they are inside a bundle.
2324 if (LiveInts) {
2325 bool mapped = !LiveInts->isNotInMIMap(Instr: *MI);
2326 if (MI->isDebugOrPseudoInstr()) {
2327 if (mapped)
2328 report(msg: "Debug instruction has a slot index", MI);
2329 } else if (MI->isInsideBundle()) {
2330 if (mapped)
2331 report(msg: "Instruction inside bundle has a slot index", MI);
2332 } else {
2333 if (!mapped)
2334 report(msg: "Missing slot index", MI);
2335 }
2336 }
2337
2338 unsigned Opc = MCID.getOpcode();
2339 if (isPreISelGenericOpcode(Opcode: Opc) || isPreISelGenericOptimizationHint(Opcode: Opc)) {
2340 verifyPreISelGenericInstruction(MI);
2341 return;
2342 }
2343
2344 StringRef ErrorInfo;
2345 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
2346 report(msg: ErrorInfo.data(), MI);
2347
2348 // Verify properties of various specific instruction types
2349 switch (MI->getOpcode()) {
2350 case TargetOpcode::COPY: {
2351 const MachineOperand &DstOp = MI->getOperand(i: 0);
2352 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2353 const Register SrcReg = SrcOp.getReg();
2354 const Register DstReg = DstOp.getReg();
2355
2356 LLT DstTy = MRI->getType(Reg: DstReg);
2357 LLT SrcTy = MRI->getType(Reg: SrcReg);
2358 if (SrcTy.isValid() && DstTy.isValid()) {
2359 // If both types are valid, check that the types are the same.
2360 if (SrcTy != DstTy) {
2361 report(msg: "Copy Instruction is illegal with mismatching types", MI);
2362 OS << "Def = " << DstTy << ", Src = " << SrcTy << '\n';
2363 }
2364
2365 break;
2366 }
2367
2368 if (!SrcTy.isValid() && !DstTy.isValid())
2369 break;
2370
2371 // If we have only one valid type, this is likely a copy between a virtual
2372 // and physical register.
2373 TypeSize SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2374 TypeSize DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2375 if (SrcReg.isPhysical() && DstTy.isValid()) {
2376 const TargetRegisterClass *SrcRC =
2377 TRI->getMinimalPhysRegClassLLT(Reg: SrcReg, Ty: DstTy);
2378 if (SrcRC)
2379 SrcSize = TRI->getRegSizeInBits(RC: *SrcRC);
2380 }
2381
2382 if (DstReg.isPhysical() && SrcTy.isValid()) {
2383 const TargetRegisterClass *DstRC =
2384 TRI->getMinimalPhysRegClassLLT(Reg: DstReg, Ty: SrcTy);
2385 if (DstRC)
2386 DstSize = TRI->getRegSizeInBits(RC: *DstRC);
2387 }
2388
2389 // The next two checks allow COPY between physical and virtual registers,
2390 // when the virtual register has a scalable size and the physical register
2391 // has a fixed size. These checks allow COPY between *potentialy* mismatched
2392 // sizes. However, once RegisterBankSelection occurs, MachineVerifier should
2393 // be able to resolve a fixed size for the scalable vector, and at that
2394 // point this function will know for sure whether the sizes are mismatched
2395 // and correctly report a size mismatch.
2396 if (SrcReg.isPhysical() && DstReg.isVirtual() && DstSize.isScalable() &&
2397 !SrcSize.isScalable())
2398 break;
2399 if (SrcReg.isVirtual() && DstReg.isPhysical() && SrcSize.isScalable() &&
2400 !DstSize.isScalable())
2401 break;
2402
2403 if (SrcSize.isNonZero() && DstSize.isNonZero() && SrcSize != DstSize) {
2404 if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {
2405 report(msg: "Copy Instruction is illegal with mismatching sizes", MI);
2406 OS << "Def Size = " << DstSize << ", Src Size = " << SrcSize << '\n';
2407 }
2408 }
2409 break;
2410 }
2411 case TargetOpcode::STATEPOINT: {
2412 StatepointOpers SO(MI);
2413 if (!MI->getOperand(i: SO.getIDPos()).isImm() ||
2414 !MI->getOperand(i: SO.getNBytesPos()).isImm() ||
2415 !MI->getOperand(i: SO.getNCallArgsPos()).isImm()) {
2416 report(msg: "meta operands to STATEPOINT not constant!", MI);
2417 break;
2418 }
2419
2420 auto VerifyStackMapConstant = [&](unsigned Offset) {
2421 if (Offset >= MI->getNumOperands()) {
2422 report(msg: "stack map constant to STATEPOINT is out of range!", MI);
2423 return;
2424 }
2425 if (!MI->getOperand(i: Offset - 1).isImm() ||
2426 MI->getOperand(i: Offset - 1).getImm() != StackMaps::ConstantOp ||
2427 !MI->getOperand(i: Offset).isImm())
2428 report(msg: "stack map constant to STATEPOINT not well formed!", MI);
2429 };
2430 VerifyStackMapConstant(SO.getCCIdx());
2431 VerifyStackMapConstant(SO.getFlagsIdx());
2432 VerifyStackMapConstant(SO.getNumDeoptArgsIdx());
2433 VerifyStackMapConstant(SO.getNumGCPtrIdx());
2434 VerifyStackMapConstant(SO.getNumAllocaIdx());
2435 VerifyStackMapConstant(SO.getNumGcMapEntriesIdx());
2436
2437 // Verify that all explicit statepoint defs are tied to gc operands as
2438 // they are expected to be a relocation of gc operands.
2439 unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx();
2440 unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2;
2441 for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) {
2442 unsigned UseOpIdx;
2443 if (!MI->isRegTiedToUseOperand(DefOpIdx: Idx, UseOpIdx: &UseOpIdx)) {
2444 report(msg: "STATEPOINT defs expected to be tied", MI);
2445 break;
2446 }
2447 if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) {
2448 report(msg: "STATEPOINT def tied to non-gc operand", MI);
2449 break;
2450 }
2451 }
2452
2453 // TODO: verify we have properly encoded deopt arguments
2454 } break;
2455 case TargetOpcode::INSERT_SUBREG: {
2456 unsigned InsertedSize;
2457 if (unsigned SubIdx = MI->getOperand(i: 2).getSubReg())
2458 InsertedSize = TRI->getSubRegIdxSize(Idx: SubIdx);
2459 else
2460 InsertedSize = TRI->getRegSizeInBits(Reg: MI->getOperand(i: 2).getReg(), MRI: *MRI);
2461 unsigned SubRegSize = TRI->getSubRegIdxSize(Idx: MI->getOperand(i: 3).getImm());
2462 if (SubRegSize < InsertedSize) {
2463 report(msg: "INSERT_SUBREG expected inserted value to have equal or lesser "
2464 "size than the subreg it was inserted into", MI);
2465 break;
2466 }
2467 } break;
2468 case TargetOpcode::REG_SEQUENCE: {
2469 unsigned NumOps = MI->getNumOperands();
2470 if (!(NumOps & 1)) {
2471 report(msg: "Invalid number of operands for REG_SEQUENCE", MI);
2472 break;
2473 }
2474
2475 for (unsigned I = 1; I != NumOps; I += 2) {
2476 const MachineOperand &RegOp = MI->getOperand(i: I);
2477 const MachineOperand &SubRegOp = MI->getOperand(i: I + 1);
2478
2479 if (!RegOp.isReg())
2480 report(msg: "Invalid register operand for REG_SEQUENCE", MO: &RegOp, MONum: I);
2481
2482 if (!SubRegOp.isImm() || SubRegOp.getImm() == 0 ||
2483 SubRegOp.getImm() >= TRI->getNumSubRegIndices()) {
2484 report(msg: "Invalid subregister index operand for REG_SEQUENCE",
2485 MO: &SubRegOp, MONum: I + 1);
2486 }
2487 }
2488
2489 Register DstReg = MI->getOperand(i: 0).getReg();
2490 if (DstReg.isPhysical())
2491 report(msg: "REG_SEQUENCE does not support physical register results", MI);
2492
2493 if (MI->getOperand(i: 0).getSubReg())
2494 report(msg: "Invalid subreg result for REG_SEQUENCE", MI);
2495
2496 break;
2497 }
2498 }
2499}
2500
2501void
2502MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
2503 const MachineInstr *MI = MO->getParent();
2504 const MCInstrDesc &MCID = MI->getDesc();
2505 unsigned NumDefs = MCID.getNumDefs();
2506 if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)
2507 NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;
2508
2509 // The first MCID.NumDefs operands must be explicit register defines
2510 if (MONum < NumDefs) {
2511 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2512 if (!MO->isReg())
2513 report(msg: "Explicit definition must be a register", MO, MONum);
2514 else if (!MO->isDef() && !MCOI.isOptionalDef())
2515 report(msg: "Explicit definition marked as use", MO, MONum);
2516 else if (MO->isImplicit())
2517 report(msg: "Explicit definition marked as implicit", MO, MONum);
2518 } else if (MONum < MCID.getNumOperands()) {
2519 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2520 // Don't check if it's the last operand in a variadic instruction. See,
2521 // e.g., LDM_RET in the arm back end. Check non-variadic operands only.
2522 bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1;
2523 if (!IsOptional) {
2524 if (MO->isReg()) {
2525 if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs())
2526 report(msg: "Explicit operand marked as def", MO, MONum);
2527 if (MO->isImplicit())
2528 report(msg: "Explicit operand marked as implicit", MO, MONum);
2529 }
2530
2531 // Check that an instruction has register operands only as expected.
2532 if (MCOI.OperandType == MCOI::OPERAND_REGISTER &&
2533 !MO->isReg() && !MO->isFI())
2534 report(msg: "Expected a register operand.", MO, MONum);
2535 if (MO->isReg()) {
2536 if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE ||
2537 (MCOI.OperandType == MCOI::OPERAND_PCREL &&
2538 !TII->isPCRelRegisterOperandLegal(MO: *MO)))
2539 report(msg: "Expected a non-register operand.", MO, MONum);
2540 }
2541 }
2542
2543 int TiedTo = MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::TIED_TO);
2544 if (TiedTo != -1) {
2545 if (!MO->isReg())
2546 report(msg: "Tied use must be a register", MO, MONum);
2547 else if (!MO->isTied())
2548 report(msg: "Operand should be tied", MO, MONum);
2549 else if (unsigned(TiedTo) != MI->findTiedOperandIdx(OpIdx: MONum))
2550 report(msg: "Tied def doesn't match MCInstrDesc", MO, MONum);
2551 else if (MO->getReg().isPhysical()) {
2552 const MachineOperand &MOTied = MI->getOperand(i: TiedTo);
2553 if (!MOTied.isReg())
2554 report(msg: "Tied counterpart must be a register", MO: &MOTied, MONum: TiedTo);
2555 else if (MOTied.getReg().isPhysical() &&
2556 MO->getReg() != MOTied.getReg())
2557 report(msg: "Tied physical registers must match.", MO: &MOTied, MONum: TiedTo);
2558 }
2559 } else if (MO->isReg() && MO->isTied())
2560 report(msg: "Explicit operand should not be tied", MO, MONum);
2561 } else if (!MI->isVariadic()) {
2562 // ARM adds %reg0 operands to indicate predicates. We'll allow that.
2563 if (!MO->isValidExcessOperand())
2564 report(msg: "Extra explicit operand on non-variadic instruction", MO, MONum);
2565 }
2566
2567 switch (MO->getType()) {
2568 case MachineOperand::MO_Register: {
2569 // Verify debug flag on debug instructions. Check this first because reg0
2570 // indicates an undefined debug value.
2571 if (MI->isDebugInstr() && MO->isUse()) {
2572 if (!MO->isDebug())
2573 report(msg: "Register operand must be marked debug", MO, MONum);
2574 } else if (MO->isDebug()) {
2575 report(msg: "Register operand must not be marked debug", MO, MONum);
2576 }
2577
2578 const Register Reg = MO->getReg();
2579 if (!Reg)
2580 return;
2581 if (MRI->tracksLiveness() && !MI->isDebugInstr())
2582 checkLiveness(MO, MONum);
2583
2584 if (MO->isDef() && MO->isUndef() && !MO->getSubReg() &&
2585 MO->getReg().isVirtual()) // TODO: Apply to physregs too
2586 report(msg: "Undef virtual register def operands require a subregister", MO, MONum);
2587
2588 // Verify the consistency of tied operands.
2589 if (MO->isTied()) {
2590 unsigned OtherIdx = MI->findTiedOperandIdx(OpIdx: MONum);
2591 const MachineOperand &OtherMO = MI->getOperand(i: OtherIdx);
2592 if (!OtherMO.isReg())
2593 report(msg: "Must be tied to a register", MO, MONum);
2594 if (!OtherMO.isTied())
2595 report(msg: "Missing tie flags on tied operand", MO, MONum);
2596 if (MI->findTiedOperandIdx(OpIdx: OtherIdx) != MONum)
2597 report(msg: "Inconsistent tie links", MO, MONum);
2598 if (MONum < MCID.getNumDefs()) {
2599 if (OtherIdx < MCID.getNumOperands()) {
2600 if (-1 == MCID.getOperandConstraint(OpNum: OtherIdx, Constraint: MCOI::TIED_TO))
2601 report(msg: "Explicit def tied to explicit use without tie constraint",
2602 MO, MONum);
2603 } else {
2604 if (!OtherMO.isImplicit())
2605 report(msg: "Explicit def should be tied to implicit use", MO, MONum);
2606 }
2607 }
2608 }
2609
2610 // Verify two-address constraints after the twoaddressinstruction pass.
2611 // Both twoaddressinstruction pass and phi-node-elimination pass call
2612 // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification
2613 // after twoaddressinstruction pass not after phi-node-elimination pass. So
2614 // we shouldn't use the IsSSA as the condition, we should based on
2615 // TiedOpsRewritten property to verify two-address constraints, this
2616 // property will be set in twoaddressinstruction pass.
2617 unsigned DefIdx;
2618 if (MF->getProperties().hasTiedOpsRewritten() && MO->isUse() &&
2619 MI->isRegTiedToDefOperand(UseOpIdx: MONum, DefOpIdx: &DefIdx) &&
2620 Reg != MI->getOperand(i: DefIdx).getReg())
2621 report(msg: "Two-address instruction operands must be identical", MO, MONum);
2622
2623 // Check register classes.
2624 unsigned SubIdx = MO->getSubReg();
2625
2626 if (Reg.isPhysical()) {
2627 if (SubIdx) {
2628 report(msg: "Illegal subregister index for physical register", MO, MONum);
2629 return;
2630 }
2631 if (MONum < MCID.getNumOperands()) {
2632 if (const TargetRegisterClass *DRC =
2633 TII->getRegClass(MCID, OpNum: MONum, TRI, MF: *MF)) {
2634 if (!DRC->contains(Reg)) {
2635 report(msg: "Illegal physical register for instruction", MO, MONum);
2636 OS << printReg(Reg, TRI) << " is not a "
2637 << TRI->getRegClassName(Class: DRC) << " register.\n";
2638 }
2639 }
2640 }
2641 if (MO->isRenamable()) {
2642 if (MRI->isReserved(PhysReg: Reg)) {
2643 report(msg: "isRenamable set on reserved register", MO, MONum);
2644 return;
2645 }
2646 }
2647 } else {
2648 // Virtual register.
2649 const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);
2650 if (!RC) {
2651 // This is a generic virtual register.
2652
2653 // Do not allow undef uses for generic virtual registers. This ensures
2654 // getVRegDef can never fail and return null on a generic register.
2655 //
2656 // FIXME: This restriction should probably be broadened to all SSA
2657 // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still
2658 // run on the SSA function just before phi elimination.
2659 if (MO->isUndef())
2660 report(msg: "Generic virtual register use cannot be undef", MO, MONum);
2661
2662 // Debug value instruction is permitted to use undefined vregs.
2663 // This is a performance measure to skip the overhead of immediately
2664 // pruning unused debug operands. The final undef substitution occurs
2665 // when debug values are allocated in LDVImpl::handleDebugValue, so
2666 // these verifications always apply after this pass.
2667 if (isFunctionTracksDebugUserValues || !MO->isUse() ||
2668 !MI->isDebugValue() || !MRI->def_empty(RegNo: Reg)) {
2669 // If we're post-Select, we can't have gvregs anymore.
2670 if (isFunctionSelected) {
2671 report(msg: "Generic virtual register invalid in a Selected function",
2672 MO, MONum);
2673 return;
2674 }
2675
2676 // The gvreg must have a type and it must not have a SubIdx.
2677 LLT Ty = MRI->getType(Reg);
2678 if (!Ty.isValid()) {
2679 report(msg: "Generic virtual register must have a valid type", MO,
2680 MONum);
2681 return;
2682 }
2683
2684 const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
2685 const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo();
2686
2687 // If we're post-RegBankSelect, the gvreg must have a bank.
2688 if (!RegBank && isFunctionRegBankSelected) {
2689 report(msg: "Generic virtual register must have a bank in a "
2690 "RegBankSelected function",
2691 MO, MONum);
2692 return;
2693 }
2694
2695 // Make sure the register fits into its register bank if any.
2696 if (RegBank && Ty.isValid() && !Ty.isScalableVector() &&
2697 RBI->getMaximumSize(RegBankID: RegBank->getID()) < Ty.getSizeInBits()) {
2698 report(msg: "Register bank is too small for virtual register", MO,
2699 MONum);
2700 OS << "Register bank " << RegBank->getName() << " too small("
2701 << RBI->getMaximumSize(RegBankID: RegBank->getID()) << ") to fit "
2702 << Ty.getSizeInBits() << "-bits\n";
2703 return;
2704 }
2705 }
2706
2707 if (SubIdx) {
2708 report(msg: "Generic virtual register does not allow subregister index", MO,
2709 MONum);
2710 return;
2711 }
2712
2713 // If this is a target specific instruction and this operand
2714 // has register class constraint, the virtual register must
2715 // comply to it.
2716 if (!isPreISelGenericOpcode(Opcode: MCID.getOpcode()) &&
2717 MONum < MCID.getNumOperands() &&
2718 TII->getRegClass(MCID, OpNum: MONum, TRI, MF: *MF)) {
2719 report(msg: "Virtual register does not match instruction constraint", MO,
2720 MONum);
2721 OS << "Expect register class "
2722 << TRI->getRegClassName(Class: TII->getRegClass(MCID, OpNum: MONum, TRI, MF: *MF))
2723 << " but got nothing\n";
2724 return;
2725 }
2726
2727 break;
2728 }
2729 if (SubIdx) {
2730 const TargetRegisterClass *SRC =
2731 TRI->getSubClassWithSubReg(RC, Idx: SubIdx);
2732 if (!SRC) {
2733 report(msg: "Invalid subregister index for virtual register", MO, MONum);
2734 OS << "Register class " << TRI->getRegClassName(Class: RC)
2735 << " does not support subreg index "
2736 << TRI->getSubRegIndexName(SubIdx) << '\n';
2737 return;
2738 }
2739 if (RC != SRC) {
2740 report(msg: "Invalid register class for subregister index", MO, MONum);
2741 OS << "Register class " << TRI->getRegClassName(Class: RC)
2742 << " does not fully support subreg index "
2743 << TRI->getSubRegIndexName(SubIdx) << '\n';
2744 return;
2745 }
2746 }
2747 if (MONum < MCID.getNumOperands()) {
2748 if (const TargetRegisterClass *DRC =
2749 TII->getRegClass(MCID, OpNum: MONum, TRI, MF: *MF)) {
2750 if (SubIdx) {
2751 const TargetRegisterClass *SuperRC =
2752 TRI->getLargestLegalSuperClass(RC, *MF);
2753 if (!SuperRC) {
2754 report(msg: "No largest legal super class exists.", MO, MONum);
2755 return;
2756 }
2757 DRC = TRI->getMatchingSuperRegClass(A: SuperRC, B: DRC, Idx: SubIdx);
2758 if (!DRC) {
2759 report(msg: "No matching super-reg register class.", MO, MONum);
2760 return;
2761 }
2762 }
2763 if (!RC->hasSuperClassEq(RC: DRC)) {
2764 report(msg: "Illegal virtual register for instruction", MO, MONum);
2765 OS << "Expected a " << TRI->getRegClassName(Class: DRC)
2766 << " register, but got a " << TRI->getRegClassName(Class: RC)
2767 << " register\n";
2768 }
2769 }
2770 }
2771 }
2772 break;
2773 }
2774
2775 case MachineOperand::MO_RegisterMask:
2776 regMasks.push_back(Elt: MO->getRegMask());
2777 break;
2778
2779 case MachineOperand::MO_MachineBasicBlock:
2780 if (MI->isPHI() && !MO->getMBB()->isSuccessor(MBB: MI->getParent()))
2781 report(msg: "PHI operand is not in the CFG", MO, MONum);
2782 break;
2783
2784 case MachineOperand::MO_FrameIndex:
2785 if (LiveStks && LiveStks->hasInterval(Slot: MO->getIndex()) &&
2786 LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
2787 int FI = MO->getIndex();
2788 LiveInterval &LI = LiveStks->getInterval(Slot: FI);
2789 SlotIndex Idx = LiveInts->getInstructionIndex(Instr: *MI);
2790
2791 bool stores = MI->mayStore();
2792 bool loads = MI->mayLoad();
2793 // For a memory-to-memory move, we need to check if the frame
2794 // index is used for storing or loading, by inspecting the
2795 // memory operands.
2796 if (stores && loads) {
2797 for (auto *MMO : MI->memoperands()) {
2798 const PseudoSourceValue *PSV = MMO->getPseudoValue();
2799 if (PSV == nullptr) continue;
2800 const FixedStackPseudoSourceValue *Value =
2801 dyn_cast<FixedStackPseudoSourceValue>(Val: PSV);
2802 if (Value == nullptr) continue;
2803 if (Value->getFrameIndex() != FI) continue;
2804
2805 if (MMO->isStore())
2806 loads = false;
2807 else
2808 stores = false;
2809 break;
2810 }
2811 if (loads == stores)
2812 report(msg: "Missing fixed stack memoperand.", MI);
2813 }
2814 if (loads && !LI.liveAt(index: Idx.getRegSlot(EC: true))) {
2815 report(msg: "Instruction loads from dead spill slot", MO, MONum);
2816 OS << "Live stack: " << LI << '\n';
2817 }
2818 if (stores && !LI.liveAt(index: Idx.getRegSlot())) {
2819 report(msg: "Instruction stores to dead spill slot", MO, MONum);
2820 OS << "Live stack: " << LI << '\n';
2821 }
2822 }
2823 break;
2824
2825 case MachineOperand::MO_CFIIndex:
2826 if (MO->getCFIIndex() >= MF->getFrameInstructions().size())
2827 report(msg: "CFI instruction has invalid index", MO, MONum);
2828 break;
2829
2830 default:
2831 break;
2832 }
2833}
2834
2835void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
2836 unsigned MONum, SlotIndex UseIdx,
2837 const LiveRange &LR,
2838 VirtRegOrUnit VRegOrUnit,
2839 LaneBitmask LaneMask) {
2840 const MachineInstr *MI = MO->getParent();
2841
2842 if (!LR.verify()) {
2843 report(msg: "invalid live range", MO, MONum);
2844 report_context_liverange(LR);
2845 report_context_vreg_regunit(VRegOrUnit);
2846 report_context(Pos: UseIdx);
2847 return;
2848 }
2849
2850 LiveQueryResult LRQ = LR.Query(Idx: UseIdx);
2851 bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut());
2852 // Check if we have a segment at the use, note however that we only need one
2853 // live subregister range, the others may be dead.
2854 if (!HasValue && LaneMask.none()) {
2855 report(msg: "No live segment at use", MO, MONum);
2856 report_context_liverange(LR);
2857 report_context_vreg_regunit(VRegOrUnit);
2858 report_context(Pos: UseIdx);
2859 }
2860 if (MO->isKill() && !LRQ.isKill()) {
2861 report(msg: "Live range continues after kill flag", MO, MONum);
2862 report_context_liverange(LR);
2863 report_context_vreg_regunit(VRegOrUnit);
2864 if (LaneMask.any())
2865 report_context_lanemask(LaneMask);
2866 report_context(Pos: UseIdx);
2867 }
2868}
2869
2870void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
2871 unsigned MONum, SlotIndex DefIdx,
2872 const LiveRange &LR,
2873 VirtRegOrUnit VRegOrUnit,
2874 bool SubRangeCheck,
2875 LaneBitmask LaneMask) {
2876 if (!LR.verify()) {
2877 report(msg: "invalid live range", MO, MONum);
2878 report_context_liverange(LR);
2879 report_context_vreg_regunit(VRegOrUnit);
2880 if (LaneMask.any())
2881 report_context_lanemask(LaneMask);
2882 report_context(Pos: DefIdx);
2883 }
2884
2885 if (const VNInfo *VNI = LR.getVNInfoAt(Idx: DefIdx)) {
2886 // The LR can correspond to the whole reg and its def slot is not obliged
2887 // to be the same as the MO' def slot. E.g. when we check here "normal"
2888 // subreg MO but there is other EC subreg MO in the same instruction so the
2889 // whole reg has EC def slot and differs from the currently checked MO' def
2890 // slot. For example:
2891 // %0 [16e,32r:0) 0@16e L..3 [16e,32r:0) 0@16e L..C [16r,32r:0) 0@16r
2892 // Check that there is an early-clobber def of the same superregister
2893 // somewhere is performed in visitMachineFunctionAfter()
2894 if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) ||
2895 !SlotIndex::isSameInstr(A: VNI->def, B: DefIdx) ||
2896 (VNI->def != DefIdx &&
2897 (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) {
2898 report(msg: "Inconsistent valno->def", MO, MONum);
2899 report_context_liverange(LR);
2900 report_context_vreg_regunit(VRegOrUnit);
2901 if (LaneMask.any())
2902 report_context_lanemask(LaneMask);
2903 report_context(VNI: *VNI);
2904 report_context(Pos: DefIdx);
2905 }
2906 } else {
2907 report(msg: "No live segment at def", MO, MONum);
2908 report_context_liverange(LR);
2909 report_context_vreg_regunit(VRegOrUnit);
2910 if (LaneMask.any())
2911 report_context_lanemask(LaneMask);
2912 report_context(Pos: DefIdx);
2913 }
2914 // Check that, if the dead def flag is present, LiveInts agree.
2915 if (MO->isDead()) {
2916 LiveQueryResult LRQ = LR.Query(Idx: DefIdx);
2917 if (!LRQ.isDeadDef()) {
2918 assert(VRegOrUnit.isVirtualReg() && "Expecting a virtual register.");
2919 // A dead subreg def only tells us that the specific subreg is dead. There
2920 // could be other non-dead defs of other subregs, or we could have other
2921 // parts of the register being live through the instruction. So unless we
2922 // are checking liveness for a subrange it is ok for the live range to
2923 // continue, given that we have a dead def of a subregister.
2924 if (SubRangeCheck || MO->getSubReg() == 0) {
2925 report(msg: "Live range continues after dead def flag", MO, MONum);
2926 report_context_liverange(LR);
2927 report_context_vreg_regunit(VRegOrUnit);
2928 if (LaneMask.any())
2929 report_context_lanemask(LaneMask);
2930 }
2931 }
2932 }
2933}
2934
2935void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
2936 const MachineInstr *MI = MO->getParent();
2937 const Register Reg = MO->getReg();
2938 const unsigned SubRegIdx = MO->getSubReg();
2939
2940 const LiveInterval *LI = nullptr;
2941 if (LiveInts && Reg.isVirtual()) {
2942 if (LiveInts->hasInterval(Reg)) {
2943 LI = &LiveInts->getInterval(Reg);
2944 if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&
2945 !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(VReg: Reg))
2946 report(msg: "Live interval for subreg operand has no subranges", MO, MONum);
2947 } else {
2948 report(msg: "Virtual register has no live interval", MO, MONum);
2949 }
2950 }
2951
2952 // Both use and def operands can read a register.
2953 if (MO->readsReg()) {
2954 if (MO->isKill())
2955 addRegWithSubRegs(RV&: regsKilled, Reg);
2956
2957 // Check that LiveVars knows this kill (unless we are inside a bundle, in
2958 // which case we have already checked that LiveVars knows any kills on the
2959 // bundle header instead).
2960 if (LiveVars && Reg.isVirtual() && MO->isKill() &&
2961 !MI->isBundledWithPred()) {
2962 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
2963 if (!is_contained(Range&: VI.Kills, Element: MI))
2964 report(msg: "Kill missing from LiveVariables", MO, MONum);
2965 }
2966
2967 // Check LiveInts liveness and kill.
2968 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
2969 SlotIndex UseIdx;
2970 if (MI->isPHI()) {
2971 // PHI use occurs on the edge, so check for live out here instead.
2972 UseIdx = LiveInts->getMBBEndIdx(
2973 mbb: MI->getOperand(i: MONum + 1).getMBB()).getPrevSlot();
2974 } else {
2975 UseIdx = LiveInts->getInstructionIndex(Instr: *MI);
2976 }
2977 // Check the cached regunit intervals.
2978 if (Reg.isPhysical() && !isReserved(Reg)) {
2979 for (MCRegUnit Unit : TRI->regunits(Reg: Reg.asMCReg())) {
2980 if (MRI->isReservedRegUnit(Unit))
2981 continue;
2982 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
2983 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LR, VRegOrUnit: VirtRegOrUnit(Unit));
2984 }
2985 }
2986
2987 if (Reg.isVirtual()) {
2988 // This is a virtual register interval.
2989 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
2990
2991 if (LI->hasSubRanges() && !MO->isDef()) {
2992 LaneBitmask MOMask = SubRegIdx != 0
2993 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
2994 : MRI->getMaxLaneMaskForVReg(Reg);
2995 LaneBitmask LiveInMask;
2996 for (const LiveInterval::SubRange &SR : LI->subranges()) {
2997 if ((MOMask & SR.LaneMask).none())
2998 continue;
2999 checkLivenessAtUse(MO, MONum, UseIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg),
3000 LaneMask: SR.LaneMask);
3001 LiveQueryResult LRQ = SR.Query(Idx: UseIdx);
3002 if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))
3003 LiveInMask |= SR.LaneMask;
3004 }
3005 // At least parts of the register has to be live at the use.
3006 if ((LiveInMask & MOMask).none()) {
3007 report(msg: "No live subrange at use", MO, MONum);
3008 report_context(LI: *LI);
3009 report_context(Pos: UseIdx);
3010 }
3011 // For PHIs all lanes should be live
3012 if (MI->isPHI() && LiveInMask != MOMask) {
3013 report(msg: "Not all lanes of PHI source live at use", MO, MONum);
3014 report_context(LI: *LI);
3015 report_context(Pos: UseIdx);
3016 }
3017 }
3018 }
3019 }
3020
3021 // Use of a dead register.
3022 if (!regsLive.count(V: Reg)) {
3023 if (Reg.isPhysical()) {
3024 // Reserved registers may be used even when 'dead'.
3025 bool Bad = !isReserved(Reg);
3026 // We are fine if just any subregister has a defined value.
3027 if (Bad) {
3028
3029 for (const MCPhysReg &SubReg : TRI->subregs(Reg)) {
3030 if (regsLive.count(V: SubReg)) {
3031 Bad = false;
3032 break;
3033 }
3034 }
3035 }
3036 // If there is an additional implicit-use of a super register we stop
3037 // here. By definition we are fine if the super register is not
3038 // (completely) dead, if the complete super register is dead we will
3039 // get a report for its operand.
3040 if (Bad) {
3041 for (const MachineOperand &MOP : MI->uses()) {
3042 if (!MOP.isReg() || !MOP.isImplicit())
3043 continue;
3044
3045 if (!MOP.getReg().isPhysical())
3046 continue;
3047
3048 if (MOP.getReg() != Reg &&
3049 all_of(Range: TRI->regunits(Reg), P: [&](const MCRegUnit RegUnit) {
3050 return llvm::is_contained(Range: TRI->regunits(Reg: MOP.getReg()),
3051 Element: RegUnit);
3052 }))
3053 Bad = false;
3054 }
3055 }
3056 if (Bad)
3057 report(msg: "Using an undefined physical register", MO, MONum);
3058 } else if (MRI->def_empty(RegNo: Reg)) {
3059 report(msg: "Reading virtual register without a def", MO, MONum);
3060 } else {
3061 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3062 // We don't know which virtual registers are live in, so only complain
3063 // if vreg was killed in this MBB. Otherwise keep track of vregs that
3064 // must be live in. PHI instructions are handled separately.
3065 if (MInfo.regsKilled.count(V: Reg))
3066 report(msg: "Using a killed virtual register", MO, MONum);
3067 else if (!MI->isPHI())
3068 MInfo.vregsLiveIn.insert(KV: std::make_pair(x: Reg, y&: MI));
3069 }
3070 }
3071 }
3072
3073 if (MO->isDef()) {
3074 // Register defined.
3075 // TODO: verify that earlyclobber ops are not used.
3076 if (MO->isDead())
3077 addRegWithSubRegs(RV&: regsDead, Reg);
3078 else
3079 addRegWithSubRegs(RV&: regsDefined, Reg);
3080
3081 // Verify SSA form.
3082 if (MRI->isSSA() && Reg.isVirtual() &&
3083 std::next(x: MRI->def_begin(RegNo: Reg)) != MRI->def_end())
3084 report(msg: "Multiple virtual register defs in SSA form", MO, MONum);
3085
3086 // Check LiveInts for a live segment, but only for virtual registers.
3087 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3088 SlotIndex DefIdx = LiveInts->getInstructionIndex(Instr: *MI);
3089 DefIdx = DefIdx.getRegSlot(EC: MO->isEarlyClobber());
3090
3091 if (Reg.isVirtual()) {
3092 checkLivenessAtDef(MO, MONum, DefIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3093
3094 if (LI->hasSubRanges()) {
3095 LaneBitmask MOMask = SubRegIdx != 0
3096 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3097 : MRI->getMaxLaneMaskForVReg(Reg);
3098 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3099 if ((SR.LaneMask & MOMask).none())
3100 continue;
3101 checkLivenessAtDef(MO, MONum, DefIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg), SubRangeCheck: true,
3102 LaneMask: SR.LaneMask);
3103 }
3104 }
3105 }
3106 }
3107 }
3108}
3109
3110// This function gets called after visiting all instructions in a bundle. The
3111// argument points to the bundle header.
3112// Normal stand-alone instructions are also considered 'bundles', and this
3113// function is called for all of them.
3114void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {
3115 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3116 set_union(S1&: MInfo.regsKilled, S2: regsKilled);
3117 set_subtract(S1&: regsLive, S2: regsKilled); regsKilled.clear();
3118 // Kill any masked registers.
3119 while (!regMasks.empty()) {
3120 const uint32_t *Mask = regMasks.pop_back_val();
3121 for (Register Reg : regsLive)
3122 if (Reg.isPhysical() &&
3123 MachineOperand::clobbersPhysReg(RegMask: Mask, PhysReg: Reg.asMCReg()))
3124 regsDead.push_back(Elt: Reg);
3125 }
3126 set_subtract(S1&: regsLive, S2: regsDead); regsDead.clear();
3127 set_union(S1&: regsLive, S2: regsDefined); regsDefined.clear();
3128}
3129
3130void
3131MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {
3132 MBBInfoMap[MBB].regsLiveOut = regsLive;
3133 regsLive.clear();
3134
3135 if (Indexes) {
3136 SlotIndex stop = Indexes->getMBBEndIdx(mbb: MBB);
3137 if (!(stop > lastIndex)) {
3138 report(msg: "Block ends before last instruction index", MBB);
3139 OS << "Block ends at " << stop << " last instruction was at " << lastIndex
3140 << '\n';
3141 }
3142 lastIndex = stop;
3143 }
3144}
3145
3146namespace {
3147// This implements a set of registers that serves as a filter: can filter other
3148// sets by passing through elements not in the filter and blocking those that
3149// are. Any filter implicitly includes the full set of physical registers upon
3150// creation, thus filtering them all out. The filter itself as a set only grows,
3151// and needs to be as efficient as possible.
3152struct VRegFilter {
3153 // Add elements to the filter itself. \pre Input set \p FromRegSet must have
3154 // no duplicates. Both virtual and physical registers are fine.
3155 template <typename RegSetT> void add(const RegSetT &FromRegSet) {
3156 SmallVector<Register, 0> VRegsBuffer;
3157 filterAndAdd(FromRegSet, VRegsBuffer);
3158 }
3159 // Filter \p FromRegSet through the filter and append passed elements into \p
3160 // ToVRegs. All elements appended are then added to the filter itself.
3161 // \returns true if anything changed.
3162 template <typename RegSetT>
3163 bool filterAndAdd(const RegSetT &FromRegSet,
3164 SmallVectorImpl<Register> &ToVRegs) {
3165 unsigned SparseUniverse = Sparse.size();
3166 unsigned NewSparseUniverse = SparseUniverse;
3167 unsigned NewDenseSize = Dense.size();
3168 size_t Begin = ToVRegs.size();
3169 for (Register Reg : FromRegSet) {
3170 if (!Reg.isVirtual())
3171 continue;
3172 unsigned Index = Reg.virtRegIndex();
3173 if (Index < SparseUniverseMax) {
3174 if (Index < SparseUniverse && Sparse.test(Idx: Index))
3175 continue;
3176 NewSparseUniverse = std::max(a: NewSparseUniverse, b: Index + 1);
3177 } else {
3178 if (Dense.count(V: Reg))
3179 continue;
3180 ++NewDenseSize;
3181 }
3182 ToVRegs.push_back(Elt: Reg);
3183 }
3184 size_t End = ToVRegs.size();
3185 if (Begin == End)
3186 return false;
3187 // Reserving space in sets once performs better than doing so continuously
3188 // and pays easily for double look-ups (even in Dense with SparseUniverseMax
3189 // tuned all the way down) and double iteration (the second one is over a
3190 // SmallVector, which is a lot cheaper compared to DenseSet or BitVector).
3191 Sparse.resize(N: NewSparseUniverse);
3192 Dense.reserve(Size: NewDenseSize);
3193 for (unsigned I = Begin; I < End; ++I) {
3194 Register Reg = ToVRegs[I];
3195 unsigned Index = Reg.virtRegIndex();
3196 if (Index < SparseUniverseMax)
3197 Sparse.set(Index);
3198 else
3199 Dense.insert(V: Reg);
3200 }
3201 return true;
3202 }
3203
3204private:
3205 static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8;
3206 // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyound
3207 // are tracked by Dense. The only purpose of the threashold and the Dense set
3208 // is to have a reasonably growing memory usage in pathological cases (large
3209 // number of very sparse VRegFilter instances live at the same time). In
3210 // practice even in the worst-by-execution time cases having all elements
3211 // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more
3212 // space efficient than if tracked by Dense. The threashold is set to keep the
3213 // worst-case memory usage within 2x of figures determined empirically for
3214 // "all Dense" scenario in such worst-by-execution-time cases.
3215 BitVector Sparse;
3216 DenseSet<Register> Dense;
3217};
3218
3219// Implements both a transfer function and a (binary, in-place) join operator
3220// for a dataflow over register sets with set union join and filtering transfer
3221// (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.
3222// Maintains out_b as its state, allowing for O(n) iteration over it at any
3223// time, where n is the size of the set (as opposed to O(U) where U is the
3224// universe). filter_b implicitly contains all physical registers at all times.
3225class FilteringVRegSet {
3226 VRegFilter Filter;
3227 SmallVector<Register, 0> VRegs;
3228
3229public:
3230 // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.
3231 // Both virtual and physical registers are fine.
3232 template <typename RegSetT> void addToFilter(const RegSetT &RS) {
3233 Filter.add(RS);
3234 }
3235 // Passes \p RS through the filter_b (transfer function) and adds what's left
3236 // to itself (out_b).
3237 template <typename RegSetT> bool add(const RegSetT &RS) {
3238 // Double-duty the Filter: to maintain VRegs a set (and the join operation
3239 // a set union) just add everything being added here to the Filter as well.
3240 return Filter.filterAndAdd(RS, VRegs);
3241 }
3242 using const_iterator = decltype(VRegs)::const_iterator;
3243 const_iterator begin() const { return VRegs.begin(); }
3244 const_iterator end() const { return VRegs.end(); }
3245 size_t size() const { return VRegs.size(); }
3246};
3247} // namespace
3248
3249// Calculate the largest possible vregsPassed sets. These are the registers that
3250// can pass through an MBB live, but may not be live every time. It is assumed
3251// that all vregsPassed sets are empty before the call.
3252void MachineVerifier::calcRegsPassed() {
3253 if (MF->empty())
3254 // ReversePostOrderTraversal doesn't handle empty functions.
3255 return;
3256
3257 for (const MachineBasicBlock *MB :
3258 ReversePostOrderTraversal<const MachineFunction *>(MF)) {
3259 FilteringVRegSet VRegs;
3260 BBInfo &Info = MBBInfoMap[MB];
3261 assert(Info.reachable);
3262
3263 VRegs.addToFilter(RS: Info.regsKilled);
3264 VRegs.addToFilter(RS: Info.regsLiveOut);
3265 for (const MachineBasicBlock *Pred : MB->predecessors()) {
3266 const BBInfo &PredInfo = MBBInfoMap[Pred];
3267 if (!PredInfo.reachable)
3268 continue;
3269
3270 VRegs.add(RS: PredInfo.regsLiveOut);
3271 VRegs.add(RS: PredInfo.vregsPassed);
3272 }
3273 Info.vregsPassed.reserve(Size: VRegs.size());
3274 Info.vregsPassed.insert_range(R&: VRegs);
3275 }
3276}
3277
3278// Calculate the set of virtual registers that must be passed through each basic
3279// block in order to satisfy the requirements of successor blocks. This is very
3280// similar to calcRegsPassed, only backwards.
3281void MachineVerifier::calcRegsRequired() {
3282 // First push live-in regs to predecessors' vregsRequired.
3283 SmallPtrSet<const MachineBasicBlock*, 8> todo;
3284 for (const auto &MBB : *MF) {
3285 BBInfo &MInfo = MBBInfoMap[&MBB];
3286 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3287 BBInfo &PInfo = MBBInfoMap[Pred];
3288 if (PInfo.addRequired(RM: MInfo.vregsLiveIn))
3289 todo.insert(Ptr: Pred);
3290 }
3291
3292 // Handle the PHI node.
3293 for (const MachineInstr &MI : MBB.phis()) {
3294 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3295 // Skip those Operands which are undef regs or not regs.
3296 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())
3297 continue;
3298
3299 // Get register and predecessor for one PHI edge.
3300 Register Reg = MI.getOperand(i).getReg();
3301 const MachineBasicBlock *Pred = MI.getOperand(i: i + 1).getMBB();
3302
3303 BBInfo &PInfo = MBBInfoMap[Pred];
3304 if (PInfo.addRequired(Reg))
3305 todo.insert(Ptr: Pred);
3306 }
3307 }
3308 }
3309
3310 // Iteratively push vregsRequired to predecessors. This will converge to the
3311 // same final state regardless of DenseSet iteration order.
3312 while (!todo.empty()) {
3313 const MachineBasicBlock *MBB = *todo.begin();
3314 todo.erase(Ptr: MBB);
3315 BBInfo &MInfo = MBBInfoMap[MBB];
3316 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
3317 if (Pred == MBB)
3318 continue;
3319 BBInfo &SInfo = MBBInfoMap[Pred];
3320 if (SInfo.addRequired(RS: MInfo.vregsRequired))
3321 todo.insert(Ptr: Pred);
3322 }
3323 }
3324}
3325
3326// Check PHI instructions at the beginning of MBB. It is assumed that
3327// calcRegsPassed has been run so BBInfo::isLiveOut is valid.
3328void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {
3329 BBInfo &MInfo = MBBInfoMap[&MBB];
3330
3331 SmallPtrSet<const MachineBasicBlock*, 8> seen;
3332 for (const MachineInstr &Phi : MBB) {
3333 if (!Phi.isPHI())
3334 break;
3335 seen.clear();
3336
3337 const MachineOperand &MODef = Phi.getOperand(i: 0);
3338 if (!MODef.isReg() || !MODef.isDef()) {
3339 report(msg: "Expected first PHI operand to be a register def", MO: &MODef, MONum: 0);
3340 continue;
3341 }
3342 if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||
3343 MODef.isEarlyClobber() || MODef.isDebug())
3344 report(msg: "Unexpected flag on PHI operand", MO: &MODef, MONum: 0);
3345 Register DefReg = MODef.getReg();
3346 if (!DefReg.isVirtual())
3347 report(msg: "Expected first PHI operand to be a virtual register", MO: &MODef, MONum: 0);
3348
3349 for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {
3350 const MachineOperand &MO0 = Phi.getOperand(i: I);
3351 if (!MO0.isReg()) {
3352 report(msg: "Expected PHI operand to be a register", MO: &MO0, MONum: I);
3353 continue;
3354 }
3355 if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||
3356 MO0.isDebug() || MO0.isTied())
3357 report(msg: "Unexpected flag on PHI operand", MO: &MO0, MONum: I);
3358
3359 const MachineOperand &MO1 = Phi.getOperand(i: I + 1);
3360 if (!MO1.isMBB()) {
3361 report(msg: "Expected PHI operand to be a basic block", MO: &MO1, MONum: I + 1);
3362 continue;
3363 }
3364
3365 const MachineBasicBlock &Pre = *MO1.getMBB();
3366 if (!Pre.isSuccessor(MBB: &MBB)) {
3367 report(msg: "PHI input is not a predecessor block", MO: &MO1, MONum: I + 1);
3368 continue;
3369 }
3370
3371 if (MInfo.reachable) {
3372 seen.insert(Ptr: &Pre);
3373 BBInfo &PrInfo = MBBInfoMap[&Pre];
3374 if (!MO0.isUndef() && PrInfo.reachable &&
3375 !PrInfo.isLiveOut(Reg: MO0.getReg()))
3376 report(msg: "PHI operand is not live-out from predecessor", MO: &MO0, MONum: I);
3377 }
3378 }
3379
3380 // Did we see all predecessors?
3381 if (MInfo.reachable) {
3382 for (MachineBasicBlock *Pred : MBB.predecessors()) {
3383 if (!seen.count(Ptr: Pred)) {
3384 report(msg: "Missing PHI operand", MI: &Phi);
3385 OS << printMBBReference(MBB: *Pred)
3386 << " is a predecessor according to the CFG.\n";
3387 }
3388 }
3389 }
3390 }
3391}
3392
3393static void
3394verifyConvergenceControl(const MachineFunction &MF, MachineDominatorTree &DT,
3395 std::function<void(const Twine &Message)> FailureCB,
3396 raw_ostream &OS) {
3397 MachineConvergenceVerifier CV;
3398 CV.initialize(OS: &OS, FailureCB, F: MF);
3399
3400 for (const auto &MBB : MF) {
3401 CV.visit(BB: MBB);
3402 for (const auto &MI : MBB.instrs())
3403 CV.visit(I: MI);
3404 }
3405
3406 if (CV.sawTokens()) {
3407 DT.recalculate(Func&: const_cast<MachineFunction &>(MF));
3408 CV.verify(DT);
3409 }
3410}
3411
3412void MachineVerifier::visitMachineFunctionAfter() {
3413 auto FailureCB = [this](const Twine &Message) {
3414 report(msg: Message.str().c_str(), MF);
3415 };
3416 verifyConvergenceControl(MF: *MF, DT, FailureCB, OS);
3417
3418 calcRegsPassed();
3419
3420 for (const MachineBasicBlock &MBB : *MF)
3421 checkPHIOps(MBB);
3422
3423 // Now check liveness info if available
3424 calcRegsRequired();
3425
3426 // Check for killed virtual registers that should be live out.
3427 for (const auto &MBB : *MF) {
3428 BBInfo &MInfo = MBBInfoMap[&MBB];
3429 for (Register VReg : MInfo.vregsRequired)
3430 if (MInfo.regsKilled.count(V: VReg)) {
3431 report(msg: "Virtual register killed in block, but needed live out.", MBB: &MBB);
3432 OS << "Virtual register " << printReg(Reg: VReg)
3433 << " is used after the block.\n";
3434 }
3435 }
3436
3437 if (!MF->empty()) {
3438 BBInfo &MInfo = MBBInfoMap[&MF->front()];
3439 for (Register VReg : MInfo.vregsRequired) {
3440 report(msg: "Virtual register defs don't dominate all uses.", MF);
3441 report_context_vreg(VReg);
3442 }
3443 }
3444
3445 if (LiveVars)
3446 verifyLiveVariables();
3447 if (LiveInts)
3448 verifyLiveIntervals();
3449
3450 // Check live-in list of each MBB. If a register is live into MBB, check
3451 // that the register is in regsLiveOut of each predecessor block. Since
3452 // this must come from a definition in the predecesssor or its live-in
3453 // list, this will catch a live-through case where the predecessor does not
3454 // have the register in its live-in list. This currently only checks
3455 // registers that have no aliases, are not allocatable and are not
3456 // reserved, which could mean a condition code register for instance.
3457 if (MRI->tracksLiveness())
3458 for (const auto &MBB : *MF)
3459 for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) {
3460 MCRegister LiveInReg = P.PhysReg;
3461 bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid();
3462 if (hasAliases || isAllocatable(Reg: LiveInReg) || isReserved(Reg: LiveInReg))
3463 continue;
3464 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3465 BBInfo &PInfo = MBBInfoMap[Pred];
3466 if (!PInfo.regsLiveOut.count(V: LiveInReg)) {
3467 report(msg: "Live in register not found to be live out from predecessor.",
3468 MBB: &MBB);
3469 OS << TRI->getName(RegNo: LiveInReg) << " not found to be live out from "
3470 << printMBBReference(MBB: *Pred) << '\n';
3471 }
3472 }
3473 }
3474
3475 for (auto CSInfo : MF->getCallSitesInfo())
3476 if (!CSInfo.first->isCall())
3477 report(msg: "Call site info referencing instruction that is not call", MF);
3478
3479 // If there's debug-info, check that we don't have any duplicate value
3480 // tracking numbers.
3481 if (MF->getFunction().getSubprogram()) {
3482 DenseSet<unsigned> SeenNumbers;
3483 for (const auto &MBB : *MF) {
3484 for (const auto &MI : MBB) {
3485 if (auto Num = MI.peekDebugInstrNum()) {
3486 auto Result = SeenNumbers.insert(V: (unsigned)Num);
3487 if (!Result.second)
3488 report(msg: "Instruction has a duplicated value tracking number", MI: &MI);
3489 }
3490 }
3491 }
3492 }
3493}
3494
3495void MachineVerifier::verifyLiveVariables() {
3496 assert(LiveVars && "Don't call verifyLiveVariables without LiveVars");
3497 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3498 Register Reg = Register::index2VirtReg(Index: I);
3499 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
3500 for (const auto &MBB : *MF) {
3501 BBInfo &MInfo = MBBInfoMap[&MBB];
3502
3503 // Our vregsRequired should be identical to LiveVariables' AliveBlocks
3504 if (MInfo.vregsRequired.count(V: Reg)) {
3505 if (!VI.AliveBlocks.test(Idx: MBB.getNumber())) {
3506 report(msg: "LiveVariables: Block missing from AliveBlocks", MBB: &MBB);
3507 OS << "Virtual register " << printReg(Reg)
3508 << " must be live through the block.\n";
3509 }
3510 } else {
3511 if (VI.AliveBlocks.test(Idx: MBB.getNumber())) {
3512 report(msg: "LiveVariables: Block should not be in AliveBlocks", MBB: &MBB);
3513 OS << "Virtual register " << printReg(Reg)
3514 << " is not needed live through the block.\n";
3515 }
3516 }
3517 }
3518 }
3519}
3520
3521void MachineVerifier::verifyLiveIntervals() {
3522 assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
3523 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3524 Register Reg = Register::index2VirtReg(Index: I);
3525
3526 // Spilling and splitting may leave unused registers around. Skip them.
3527 if (MRI->reg_nodbg_empty(RegNo: Reg))
3528 continue;
3529
3530 if (!LiveInts->hasInterval(Reg)) {
3531 report(msg: "Missing live interval for virtual register", MF);
3532 OS << printReg(Reg, TRI) << " still has defs or uses\n";
3533 continue;
3534 }
3535
3536 const LiveInterval &LI = LiveInts->getInterval(Reg);
3537 assert(Reg == LI.reg() && "Invalid reg to interval mapping");
3538 verifyLiveInterval(LI);
3539 }
3540
3541 // Verify all the cached regunit intervals.
3542 for (unsigned i = 0, e = TRI->getNumRegUnits(); i != e; ++i)
3543 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit: i))
3544 verifyLiveRange(*LR, VirtRegOrUnit(i));
3545}
3546
3547void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
3548 const VNInfo *VNI,
3549 VirtRegOrUnit VRegOrUnit,
3550 LaneBitmask LaneMask) {
3551 if (VNI->isUnused())
3552 return;
3553
3554 const VNInfo *DefVNI = LR.getVNInfoAt(Idx: VNI->def);
3555
3556 if (!DefVNI) {
3557 report(msg: "Value not live at VNInfo def and not marked unused", MF);
3558 report_context(LR, VRegOrUnit, LaneMask);
3559 report_context(VNI: *VNI);
3560 return;
3561 }
3562
3563 if (DefVNI != VNI) {
3564 report(msg: "Live segment at def has different VNInfo", MF);
3565 report_context(LR, VRegOrUnit, LaneMask);
3566 report_context(VNI: *VNI);
3567 return;
3568 }
3569
3570 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: VNI->def);
3571 if (!MBB) {
3572 report(msg: "Invalid VNInfo definition index", MF);
3573 report_context(LR, VRegOrUnit, LaneMask);
3574 report_context(VNI: *VNI);
3575 return;
3576 }
3577
3578 if (VNI->isPHIDef()) {
3579 if (VNI->def != LiveInts->getMBBStartIdx(mbb: MBB)) {
3580 report(msg: "PHIDef VNInfo is not defined at MBB start", MBB);
3581 report_context(LR, VRegOrUnit, LaneMask);
3582 report_context(VNI: *VNI);
3583 }
3584 return;
3585 }
3586
3587 // Non-PHI def.
3588 const MachineInstr *MI = LiveInts->getInstructionFromIndex(index: VNI->def);
3589 if (!MI) {
3590 report(msg: "No instruction at VNInfo def index", MBB);
3591 report_context(LR, VRegOrUnit, LaneMask);
3592 report_context(VNI: *VNI);
3593 return;
3594 }
3595
3596 bool hasDef = false;
3597 bool isEarlyClobber = false;
3598 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3599 if (!MOI->isReg() || !MOI->isDef())
3600 continue;
3601 if (VRegOrUnit.isVirtualReg()) {
3602 if (MOI->getReg() != VRegOrUnit.asVirtualReg())
3603 continue;
3604 } else {
3605 if (!MOI->getReg().isPhysical() ||
3606 !TRI->hasRegUnit(Reg: MOI->getReg(), RegUnit: VRegOrUnit.asMCRegUnit()))
3607 continue;
3608 }
3609 if (LaneMask.any() &&
3610 (TRI->getSubRegIndexLaneMask(SubIdx: MOI->getSubReg()) & LaneMask).none())
3611 continue;
3612 hasDef = true;
3613 if (MOI->isEarlyClobber())
3614 isEarlyClobber = true;
3615 }
3616
3617 if (!hasDef) {
3618 report(msg: "Defining instruction does not modify register", MI);
3619 report_context(LR, VRegOrUnit, LaneMask);
3620 report_context(VNI: *VNI);
3621 }
3622
3623 // Early clobber defs begin at USE slots, but other defs must begin at
3624 // DEF slots.
3625 if (isEarlyClobber) {
3626 if (!VNI->def.isEarlyClobber()) {
3627 report(msg: "Early clobber def must be at an early-clobber slot", MBB);
3628 report_context(LR, VRegOrUnit, LaneMask);
3629 report_context(VNI: *VNI);
3630 }
3631 } else if (!VNI->def.isRegister()) {
3632 report(msg: "Non-PHI, non-early clobber def must be at a register slot", MBB);
3633 report_context(LR, VRegOrUnit, LaneMask);
3634 report_context(VNI: *VNI);
3635 }
3636}
3637
3638void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
3639 const LiveRange::const_iterator I,
3640 VirtRegOrUnit VRegOrUnit,
3641 LaneBitmask LaneMask) {
3642 const LiveRange::Segment &S = *I;
3643 const VNInfo *VNI = S.valno;
3644 assert(VNI && "Live segment has no valno");
3645
3646 if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(ValNo: VNI->id)) {
3647 report(msg: "Foreign valno in live segment", MF);
3648 report_context(LR, VRegOrUnit, LaneMask);
3649 report_context(S);
3650 report_context(VNI: *VNI);
3651 }
3652
3653 if (VNI->isUnused()) {
3654 report(msg: "Live segment valno is marked unused", MF);
3655 report_context(LR, VRegOrUnit, LaneMask);
3656 report_context(S);
3657 }
3658
3659 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: S.start);
3660 if (!MBB) {
3661 report(msg: "Bad start of live segment, no basic block", MF);
3662 report_context(LR, VRegOrUnit, LaneMask);
3663 report_context(S);
3664 return;
3665 }
3666 SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(mbb: MBB);
3667 if (S.start != MBBStartIdx && S.start != VNI->def) {
3668 report(msg: "Live segment must begin at MBB entry or valno def", MBB);
3669 report_context(LR, VRegOrUnit, LaneMask);
3670 report_context(S);
3671 }
3672
3673 const MachineBasicBlock *EndMBB =
3674 LiveInts->getMBBFromIndex(index: S.end.getPrevSlot());
3675 if (!EndMBB) {
3676 report(msg: "Bad end of live segment, no basic block", MF);
3677 report_context(LR, VRegOrUnit, LaneMask);
3678 report_context(S);
3679 return;
3680 }
3681
3682 // Checks for non-live-out segments.
3683 if (S.end != LiveInts->getMBBEndIdx(mbb: EndMBB)) {
3684 // RegUnit intervals are allowed dead phis.
3685 if (!VRegOrUnit.isVirtualReg() && VNI->isPHIDef() && S.start == VNI->def &&
3686 S.end == VNI->def.getDeadSlot())
3687 return;
3688
3689 // The live segment is ending inside EndMBB
3690 const MachineInstr *MI =
3691 LiveInts->getInstructionFromIndex(index: S.end.getPrevSlot());
3692 if (!MI) {
3693 report(msg: "Live segment doesn't end at a valid instruction", MBB: EndMBB);
3694 report_context(LR, VRegOrUnit, LaneMask);
3695 report_context(S);
3696 return;
3697 }
3698
3699 // The block slot must refer to a basic block boundary.
3700 if (S.end.isBlock()) {
3701 report(msg: "Live segment ends at B slot of an instruction", MBB: EndMBB);
3702 report_context(LR, VRegOrUnit, LaneMask);
3703 report_context(S);
3704 }
3705
3706 if (S.end.isDead()) {
3707 // Segment ends on the dead slot.
3708 // That means there must be a dead def.
3709 if (!SlotIndex::isSameInstr(A: S.start, B: S.end)) {
3710 report(msg: "Live segment ending at dead slot spans instructions", MBB: EndMBB);
3711 report_context(LR, VRegOrUnit, LaneMask);
3712 report_context(S);
3713 }
3714 }
3715
3716 // After tied operands are rewritten, a live segment can only end at an
3717 // early-clobber slot if it is being redefined by an early-clobber def.
3718 // TODO: Before tied operands are rewritten, a live segment can only end at
3719 // an early-clobber slot if the last use is tied to an early-clobber def.
3720 if (MF->getProperties().hasTiedOpsRewritten() && S.end.isEarlyClobber()) {
3721 if (I + 1 == LR.end() || (I + 1)->start != S.end) {
3722 report(msg: "Live segment ending at early clobber slot must be "
3723 "redefined by an EC def in the same instruction",
3724 MBB: EndMBB);
3725 report_context(LR, VRegOrUnit, LaneMask);
3726 report_context(S);
3727 }
3728 }
3729
3730 // The following checks only apply to virtual registers. Physreg liveness
3731 // is too weird to check.
3732 if (VRegOrUnit.isVirtualReg()) {
3733 // A live segment can end with either a redefinition, a kill flag on a
3734 // use, or a dead flag on a def.
3735 bool hasRead = false;
3736 bool hasSubRegDef = false;
3737 bool hasDeadDef = false;
3738 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3739 if (!MOI->isReg() || MOI->getReg() != VRegOrUnit.asVirtualReg())
3740 continue;
3741 unsigned Sub = MOI->getSubReg();
3742 LaneBitmask SLM =
3743 Sub != 0 ? TRI->getSubRegIndexLaneMask(SubIdx: Sub) : LaneBitmask::getAll();
3744 if (MOI->isDef()) {
3745 if (Sub != 0) {
3746 hasSubRegDef = true;
3747 // An operand %0:sub0 reads %0:sub1..n. Invert the lane
3748 // mask for subregister defs. Read-undef defs will be handled by
3749 // readsReg below.
3750 SLM = ~SLM;
3751 }
3752 if (MOI->isDead())
3753 hasDeadDef = true;
3754 }
3755 if (LaneMask.any() && (LaneMask & SLM).none())
3756 continue;
3757 if (MOI->readsReg())
3758 hasRead = true;
3759 }
3760 if (S.end.isDead()) {
3761 // Make sure that the corresponding machine operand for a "dead" live
3762 // range has the dead flag. We cannot perform this check for subregister
3763 // liveranges as partially dead values are allowed.
3764 if (LaneMask.none() && !hasDeadDef) {
3765 report(
3766 msg: "Instruction ending live segment on dead slot has no dead flag",
3767 MI);
3768 report_context(LR, VRegOrUnit, LaneMask);
3769 report_context(S);
3770 }
3771 } else {
3772 if (!hasRead) {
3773 // When tracking subregister liveness, the main range must start new
3774 // values on partial register writes, even if there is no read.
3775 if (!MRI->shouldTrackSubRegLiveness(VReg: VRegOrUnit.asVirtualReg()) ||
3776 LaneMask.any() || !hasSubRegDef) {
3777 report(msg: "Instruction ending live segment doesn't read the register",
3778 MI);
3779 report_context(LR, VRegOrUnit, LaneMask);
3780 report_context(S);
3781 }
3782 }
3783 }
3784 }
3785 }
3786
3787 // Now check all the basic blocks in this live segment.
3788 MachineFunction::const_iterator MFI = MBB->getIterator();
3789 // Is this live segment the beginning of a non-PHIDef VN?
3790 if (S.start == VNI->def && !VNI->isPHIDef()) {
3791 // Not live-in to any blocks.
3792 if (MBB == EndMBB)
3793 return;
3794 // Skip this block.
3795 ++MFI;
3796 }
3797
3798 SmallVector<SlotIndex, 4> Undefs;
3799 if (LaneMask.any()) {
3800 LiveInterval &OwnerLI = LiveInts->getInterval(Reg: VRegOrUnit.asVirtualReg());
3801 OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, MRI: *MRI, Indexes: *Indexes);
3802 }
3803
3804 while (true) {
3805 assert(LiveInts->isLiveInToMBB(LR, &*MFI));
3806 // We don't know how to track physregs into a landing pad.
3807 if (!VRegOrUnit.isVirtualReg() && MFI->isEHPad()) {
3808 if (&*MFI == EndMBB)
3809 break;
3810 ++MFI;
3811 continue;
3812 }
3813
3814 // Is VNI a PHI-def in the current block?
3815 bool IsPHI = VNI->isPHIDef() &&
3816 VNI->def == LiveInts->getMBBStartIdx(mbb: &*MFI);
3817
3818 // Check that VNI is live-out of all predecessors.
3819 for (const MachineBasicBlock *Pred : MFI->predecessors()) {
3820 SlotIndex PEnd = LiveInts->getMBBEndIdx(mbb: Pred);
3821 // Predecessor of landing pad live-out on last call.
3822 if (MFI->isEHPad()) {
3823 for (const MachineInstr &MI : llvm::reverse(C: *Pred)) {
3824 if (MI.isCall()) {
3825 PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex();
3826 break;
3827 }
3828 }
3829 }
3830 const VNInfo *PVNI = LR.getVNInfoBefore(Idx: PEnd);
3831
3832 // All predecessors must have a live-out value. However for a phi
3833 // instruction with subregister intervals
3834 // only one of the subregisters (not necessarily the current one) needs to
3835 // be defined.
3836 if (!PVNI && (LaneMask.none() || !IsPHI)) {
3837 if (LiveRangeCalc::isJointlyDominated(MBB: Pred, Defs: Undefs, Indexes: *Indexes))
3838 continue;
3839 report(msg: "Register not marked live out of predecessor", MBB: Pred);
3840 report_context(LR, VRegOrUnit, LaneMask);
3841 report_context(VNI: *VNI);
3842 OS << " live into " << printMBBReference(MBB: *MFI) << '@'
3843 << LiveInts->getMBBStartIdx(mbb: &*MFI) << ", not live before " << PEnd
3844 << '\n';
3845 continue;
3846 }
3847
3848 // Only PHI-defs can take different predecessor values.
3849 if (!IsPHI && PVNI != VNI) {
3850 report(msg: "Different value live out of predecessor", MBB: Pred);
3851 report_context(LR, VRegOrUnit, LaneMask);
3852 OS << "Valno #" << PVNI->id << " live out of "
3853 << printMBBReference(MBB: *Pred) << '@' << PEnd << "\nValno #" << VNI->id
3854 << " live into " << printMBBReference(MBB: *MFI) << '@'
3855 << LiveInts->getMBBStartIdx(mbb: &*MFI) << '\n';
3856 }
3857 }
3858 if (&*MFI == EndMBB)
3859 break;
3860 ++MFI;
3861 }
3862}
3863
3864void MachineVerifier::verifyLiveRange(const LiveRange &LR,
3865 VirtRegOrUnit VRegOrUnit,
3866 LaneBitmask LaneMask) {
3867 for (const VNInfo *VNI : LR.valnos)
3868 verifyLiveRangeValue(LR, VNI, VRegOrUnit, LaneMask);
3869
3870 for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
3871 verifyLiveRangeSegment(LR, I, VRegOrUnit, LaneMask);
3872}
3873
3874void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
3875 Register Reg = LI.reg();
3876 assert(Reg.isVirtual());
3877 verifyLiveRange(LR: LI, VRegOrUnit: VirtRegOrUnit(Reg));
3878
3879 if (LI.hasSubRanges()) {
3880 LaneBitmask Mask;
3881 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
3882 for (const LiveInterval::SubRange &SR : LI.subranges()) {
3883 if ((Mask & SR.LaneMask).any()) {
3884 report(msg: "Lane masks of sub ranges overlap in live interval", MF);
3885 report_context(LI);
3886 }
3887 if ((SR.LaneMask & ~MaxMask).any()) {
3888 report(msg: "Subrange lanemask is invalid", MF);
3889 report_context(LI);
3890 }
3891 if (SR.empty()) {
3892 report(msg: "Subrange must not be empty", MF);
3893 report_context(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
3894 }
3895 Mask |= SR.LaneMask;
3896 verifyLiveRange(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
3897 if (!LI.covers(Other: SR)) {
3898 report(msg: "A Subrange is not covered by the main range", MF);
3899 report_context(LI);
3900 }
3901 }
3902 }
3903
3904 // Check the LI only has one connected component.
3905 ConnectedVNInfoEqClasses ConEQ(*LiveInts);
3906 unsigned NumComp = ConEQ.Classify(LR: LI);
3907 if (NumComp > 1) {
3908 report(msg: "Multiple connected components in live interval", MF);
3909 report_context(LI);
3910 for (unsigned comp = 0; comp != NumComp; ++comp) {
3911 OS << comp << ": valnos";
3912 for (const VNInfo *I : LI.valnos)
3913 if (comp == ConEQ.getEqClass(VNI: I))
3914 OS << ' ' << I->id;
3915 OS << '\n';
3916 }
3917 }
3918}
3919
3920namespace {
3921
3922 // FrameSetup and FrameDestroy can have zero adjustment, so using a single
3923 // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the
3924 // value is zero.
3925 // We use a bool plus an integer to capture the stack state.
3926struct StackStateOfBB {
3927 StackStateOfBB() = default;
3928 StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup)
3929 : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),
3930 ExitIsSetup(ExitSetup) {}
3931
3932 // Can be negative, which means we are setting up a frame.
3933 int EntryValue = 0;
3934 int ExitValue = 0;
3935 bool EntryIsSetup = false;
3936 bool ExitIsSetup = false;
3937};
3938
3939} // end anonymous namespace
3940
3941/// Make sure on every path through the CFG, a FrameSetup <n> is always followed
3942/// by a FrameDestroy <n>, stack adjustments are identical on all
3943/// CFG edges to a merge point, and frame is destroyed at end of a return block.
3944void MachineVerifier::verifyStackFrame() {
3945 unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
3946 unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
3947 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
3948 return;
3949
3950 SmallVector<StackStateOfBB, 8> SPState;
3951 SPState.resize(N: MF->getNumBlockIDs());
3952 df_iterator_default_set<const MachineBasicBlock*> Reachable;
3953
3954 // Visit the MBBs in DFS order.
3955 for (df_ext_iterator<const MachineFunction *,
3956 df_iterator_default_set<const MachineBasicBlock *>>
3957 DFI = df_ext_begin(G: MF, S&: Reachable), DFE = df_ext_end(G: MF, S&: Reachable);
3958 DFI != DFE; ++DFI) {
3959 const MachineBasicBlock *MBB = *DFI;
3960
3961 StackStateOfBB BBState;
3962 // Check the exit state of the DFS stack predecessor.
3963 if (DFI.getPathLength() >= 2) {
3964 const MachineBasicBlock *StackPred = DFI.getPath(n: DFI.getPathLength() - 2);
3965 assert(Reachable.count(StackPred) &&
3966 "DFS stack predecessor is already visited.\n");
3967 BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;
3968 BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;
3969 BBState.ExitValue = BBState.EntryValue;
3970 BBState.ExitIsSetup = BBState.EntryIsSetup;
3971 }
3972
3973 if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) {
3974 report(msg: "Call frame size on entry does not match value computed from "
3975 "predecessor",
3976 MBB);
3977 OS << "Call frame size on entry " << MBB->getCallFrameSize()
3978 << " does not match value computed from predecessor "
3979 << -BBState.EntryValue << '\n';
3980 }
3981
3982 // Update stack state by checking contents of MBB.
3983 for (const auto &I : *MBB) {
3984 if (I.getOpcode() == FrameSetupOpcode) {
3985 if (BBState.ExitIsSetup)
3986 report(msg: "FrameSetup is after another FrameSetup", MI: &I);
3987 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
3988 report(msg: "AdjustsStack not set in presence of a frame pseudo "
3989 "instruction.", MI: &I);
3990 BBState.ExitValue -= TII->getFrameTotalSize(I);
3991 BBState.ExitIsSetup = true;
3992 }
3993
3994 if (I.getOpcode() == FrameDestroyOpcode) {
3995 int Size = TII->getFrameTotalSize(I);
3996 if (!BBState.ExitIsSetup)
3997 report(msg: "FrameDestroy is not after a FrameSetup", MI: &I);
3998 int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :
3999 BBState.ExitValue;
4000 if (BBState.ExitIsSetup && AbsSPAdj != Size) {
4001 report(msg: "FrameDestroy <n> is after FrameSetup <m>", MI: &I);
4002 OS << "FrameDestroy <" << Size << "> is after FrameSetup <"
4003 << AbsSPAdj << ">.\n";
4004 }
4005 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4006 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4007 "instruction.", MI: &I);
4008 BBState.ExitValue += Size;
4009 BBState.ExitIsSetup = false;
4010 }
4011 }
4012 SPState[MBB->getNumber()] = BBState;
4013
4014 // Make sure the exit state of any predecessor is consistent with the entry
4015 // state.
4016 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
4017 if (Reachable.count(Ptr: Pred) &&
4018 (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue ||
4019 SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {
4020 report(msg: "The exit stack state of a predecessor is inconsistent.", MBB);
4021 OS << "Predecessor " << printMBBReference(MBB: *Pred) << " has exit state ("
4022 << SPState[Pred->getNumber()].ExitValue << ", "
4023 << SPState[Pred->getNumber()].ExitIsSetup << "), while "
4024 << printMBBReference(MBB: *MBB) << " has entry state ("
4025 << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";
4026 }
4027 }
4028
4029 // Make sure the entry state of any successor is consistent with the exit
4030 // state.
4031 for (const MachineBasicBlock *Succ : MBB->successors()) {
4032 if (Reachable.count(Ptr: Succ) &&
4033 (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue ||
4034 SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {
4035 report(msg: "The entry stack state of a successor is inconsistent.", MBB);
4036 OS << "Successor " << printMBBReference(MBB: *Succ) << " has entry state ("
4037 << SPState[Succ->getNumber()].EntryValue << ", "
4038 << SPState[Succ->getNumber()].EntryIsSetup << "), while "
4039 << printMBBReference(MBB: *MBB) << " has exit state ("
4040 << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";
4041 }
4042 }
4043
4044 // Make sure a basic block with return ends with zero stack adjustment.
4045 if (!MBB->empty() && MBB->back().isReturn()) {
4046 if (BBState.ExitIsSetup)
4047 report(msg: "A return block ends with a FrameSetup.", MBB);
4048 if (BBState.ExitValue)
4049 report(msg: "A return block ends with a nonzero stack adjustment.", MBB);
4050 }
4051 }
4052}
4053
4054void MachineVerifier::verifyStackProtector() {
4055 const MachineFrameInfo &MFI = MF->getFrameInfo();
4056 if (!MFI.hasStackProtectorIndex())
4057 return;
4058 // Only applicable when the offsets of frame objects have been determined,
4059 // which is indicated by a non-zero stack size.
4060 if (!MFI.getStackSize())
4061 return;
4062 const TargetFrameLowering &TFI = *MF->getSubtarget().getFrameLowering();
4063 bool StackGrowsDown =
4064 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
4065 unsigned FI = MFI.getStackProtectorIndex();
4066 int64_t SPStart = MFI.getObjectOffset(ObjectIdx: FI);
4067 int64_t SPEnd = SPStart + MFI.getObjectSize(ObjectIdx: FI);
4068 for (unsigned I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
4069 if (I == FI)
4070 continue;
4071 if (MFI.isDeadObjectIndex(ObjectIdx: I))
4072 continue;
4073 // FIXME: Skip non-default stack objects, as some targets may place them
4074 // above the stack protector. This is a workaround for the fact that
4075 // backends such as AArch64 may place SVE stack objects *above* the stack
4076 // protector.
4077 if (MFI.getStackID(ObjectIdx: I) != TargetStackID::Default)
4078 continue;
4079 // Skip variable-sized objects because they do not have a fixed offset.
4080 if (MFI.isVariableSizedObjectIndex(ObjectIdx: I))
4081 continue;
4082 // FIXME: Skip spill slots which may be allocated above the stack protector.
4083 // Ideally this would only skip callee-saved registers, but we don't have
4084 // that information here. For example, spill-slots used for scavenging are
4085 // not described in CalleeSavedInfo.
4086 if (MFI.isSpillSlotObjectIndex(ObjectIdx: I))
4087 continue;
4088 int64_t ObjStart = MFI.getObjectOffset(ObjectIdx: I);
4089 int64_t ObjEnd = ObjStart + MFI.getObjectSize(ObjectIdx: I);
4090 if (SPStart < ObjEnd && ObjStart < SPEnd) {
4091 report(msg: "Stack protector overlaps with another stack object", MF);
4092 break;
4093 }
4094 if ((StackGrowsDown && SPStart <= ObjStart) ||
4095 (!StackGrowsDown && SPStart >= ObjStart)) {
4096 report(msg: "Stack protector is not the top-most object on the stack", MF);
4097 break;
4098 }
4099 }
4100}
4101