1//===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Pass to verify generated machine code. The following is checked:
10//
11// Operand counts: All explicit operands must be present.
12//
13// Register classes: All physical and virtual register operands must be
14// compatible with the register class required by the instruction descriptor.
15//
16// Register live intervals: Registers must be defined only once, and must be
17// defined before use.
18//
19// The machine code verifier is enabled with the command-line option
20// -verify-machineinstrs.
21//===----------------------------------------------------------------------===//
22
23#include "llvm/CodeGen/MachineVerifier.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/DenseMap.h"
26#include "llvm/ADT/DenseSet.h"
27#include "llvm/ADT/DepthFirstIterator.h"
28#include "llvm/ADT/PostOrderIterator.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SetOperations.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/StringRef.h"
34#include "llvm/ADT/Twine.h"
35#include "llvm/CodeGen/CodeGenCommonISel.h"
36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
37#include "llvm/CodeGen/LiveInterval.h"
38#include "llvm/CodeGen/LiveIntervals.h"
39#include "llvm/CodeGen/LiveRangeCalc.h"
40#include "llvm/CodeGen/LiveStacks.h"
41#include "llvm/CodeGen/LiveVariables.h"
42#include "llvm/CodeGen/MachineBasicBlock.h"
43#include "llvm/CodeGen/MachineConvergenceVerifier.h"
44#include "llvm/CodeGen/MachineDominators.h"
45#include "llvm/CodeGen/MachineFrameInfo.h"
46#include "llvm/CodeGen/MachineFunction.h"
47#include "llvm/CodeGen/MachineFunctionPass.h"
48#include "llvm/CodeGen/MachineInstr.h"
49#include "llvm/CodeGen/MachineInstrBundle.h"
50#include "llvm/CodeGen/MachineMemOperand.h"
51#include "llvm/CodeGen/MachineOperand.h"
52#include "llvm/CodeGen/MachineRegisterInfo.h"
53#include "llvm/CodeGen/PseudoSourceValue.h"
54#include "llvm/CodeGen/RegisterBank.h"
55#include "llvm/CodeGen/RegisterBankInfo.h"
56#include "llvm/CodeGen/SlotIndexes.h"
57#include "llvm/CodeGen/StackMaps.h"
58#include "llvm/CodeGen/TargetInstrInfo.h"
59#include "llvm/CodeGen/TargetLowering.h"
60#include "llvm/CodeGen/TargetOpcodes.h"
61#include "llvm/CodeGen/TargetRegisterInfo.h"
62#include "llvm/CodeGen/TargetSubtargetInfo.h"
63#include "llvm/CodeGenTypes/LowLevelType.h"
64#include "llvm/IR/BasicBlock.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/EHPersonalities.h"
67#include "llvm/IR/Function.h"
68#include "llvm/IR/InlineAsm.h"
69#include "llvm/IR/Instructions.h"
70#include "llvm/InitializePasses.h"
71#include "llvm/MC/LaneBitmask.h"
72#include "llvm/MC/MCAsmInfo.h"
73#include "llvm/MC/MCDwarf.h"
74#include "llvm/MC/MCInstrDesc.h"
75#include "llvm/MC/MCRegisterInfo.h"
76#include "llvm/MC/MCTargetOptions.h"
77#include "llvm/Pass.h"
78#include "llvm/Support/Casting.h"
79#include "llvm/Support/ErrorHandling.h"
80#include "llvm/Support/ManagedStatic.h"
81#include "llvm/Support/MathExtras.h"
82#include "llvm/Support/ModRef.h"
83#include "llvm/Support/Mutex.h"
84#include "llvm/Support/raw_ostream.h"
85#include "llvm/Target/TargetMachine.h"
86#include <algorithm>
87#include <cassert>
88#include <cstddef>
89#include <cstdint>
90#include <iterator>
91#include <string>
92#include <utility>
93
94using namespace llvm;
95
96namespace {
97
98/// Used the by the ReportedErrors class to guarantee only one error is reported
99/// at one time.
100static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock;
101
102static bool hasPhysRegClassForType(const TargetRegisterInfo &TRI,
103 MCRegister Reg, LLT Ty) {
104 assert(Reg.isPhysical() && "reg must be a physical register");
105 assert(Ty.isValid() && "expected a valid type");
106
107 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
108 if (TRI.isTypeLegalForClass(RC: *RC, T: Ty))
109 return true;
110
111 return llvm::any_of(Range: TRI.regclasses(), P: [&](const TargetRegisterClass &RC) {
112 return RC.contains(Reg) && TRI.isTypeLegalForClass(RC, T: Ty);
113 });
114}
115
116struct MachineVerifier {
117 MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b,
118 raw_ostream *OS, bool AbortOnError = true)
119 : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b),
120 ReportedErrs(AbortOnError) {}
121
122 MachineVerifier(Pass *pass, const char *b, raw_ostream *OS,
123 bool AbortOnError = true)
124 : PASS(pass), OS(OS ? *OS : nulls()), Banner(b),
125 ReportedErrs(AbortOnError) {}
126
127 MachineVerifier(const char *b, LiveVariables *LiveVars,
128 LiveIntervals *LiveInts, LiveStacks *LiveStks,
129 SlotIndexes *Indexes, raw_ostream *OS,
130 bool AbortOnError = true)
131 : OS(OS ? *OS : nulls()), Banner(b), LiveVars(LiveVars),
132 LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes),
133 ReportedErrs(AbortOnError) {}
134
135 /// \returns true if no problems were found.
136 bool verify(const MachineFunction &MF);
137
138 MachineFunctionAnalysisManager *MFAM = nullptr;
139 Pass *const PASS = nullptr;
140 raw_ostream &OS;
141 const char *Banner;
142 const MachineFunction *MF = nullptr;
143 const TargetMachine *TM = nullptr;
144 const TargetInstrInfo *TII = nullptr;
145 const TargetRegisterInfo *TRI = nullptr;
146 const MachineRegisterInfo *MRI = nullptr;
147 const RegisterBankInfo *RBI = nullptr;
148
149 // Avoid querying the MachineFunctionProperties for each operand.
150 bool isFunctionRegBankSelected = false;
151 bool isFunctionSelected = false;
152 bool isFunctionTracksDebugUserValues = false;
153
154 using RegVector = SmallVector<Register, 16>;
155 using RegMaskVector = SmallVector<const uint32_t *, 4>;
156 using RegSet = DenseSet<Register>;
157 using RegMap = DenseMap<Register, const MachineInstr *>;
158 using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;
159
160 const MachineInstr *FirstNonPHI = nullptr;
161 const MachineInstr *FirstTerminator = nullptr;
162 BlockSet FunctionBlocks;
163
164 BitVector regsReserved;
165 RegSet regsLive;
166 RegVector regsDefined, regsDead, regsKilled;
167 RegMaskVector regMasks;
168
169 SlotIndex lastIndex;
170
171 // Add Reg and any sub-registers to RV
172 void addRegWithSubRegs(RegVector &RV, Register Reg) {
173 RV.push_back(Elt: Reg);
174 if (Reg.isPhysical())
175 append_range(C&: RV, R: TRI->subregs(Reg: Reg.asMCReg()));
176 }
177
178 struct BBInfo {
179 // Is this MBB reachable from the MF entry point?
180 bool reachable = false;
181
182 // Vregs that must be live in because they are used without being
183 // defined. Map value is the user. vregsLiveIn doesn't include regs
184 // that only are used by PHI nodes.
185 RegMap vregsLiveIn;
186
187 // Regs killed in MBB. They may be defined again, and will then be in both
188 // regsKilled and regsLiveOut.
189 RegSet regsKilled;
190
191 // Regs defined in MBB and live out. Note that vregs passing through may
192 // be live out without being mentioned here.
193 RegSet regsLiveOut;
194
195 // Vregs that pass through MBB untouched. This set is disjoint from
196 // regsKilled and regsLiveOut.
197 RegSet vregsPassed;
198
199 // Vregs that must pass through MBB because they are needed by a successor
200 // block. This set is disjoint from regsLiveOut.
201 RegSet vregsRequired;
202
203 // Set versions of block's predecessor and successor lists.
204 BlockSet Preds, Succs;
205
206 BBInfo() = default;
207
208 // Add register to vregsRequired if it belongs there. Return true if
209 // anything changed.
210 bool addRequired(Register Reg) {
211 if (!Reg.isVirtual())
212 return false;
213 if (regsLiveOut.count(V: Reg))
214 return false;
215 return vregsRequired.insert(V: Reg).second;
216 }
217
218 // Same for a full set.
219 bool addRequired(const RegSet &RS) {
220 bool Changed = false;
221 for (Register Reg : RS)
222 Changed |= addRequired(Reg);
223 return Changed;
224 }
225
226 // Same for a full map.
227 bool addRequired(const RegMap &RM) {
228 bool Changed = false;
229 for (const auto &I : RM)
230 Changed |= addRequired(Reg: I.first);
231 return Changed;
232 }
233
234 // Live-out registers are either in regsLiveOut or vregsPassed.
235 bool isLiveOut(Register Reg) const {
236 return regsLiveOut.count(V: Reg) || vregsPassed.count(V: Reg);
237 }
238 };
239
240 // Extra register info per MBB.
241 DenseMap<const MachineBasicBlock *, BBInfo> MBBInfoMap;
242
243 bool isReserved(Register Reg) {
244 return Reg.id() < regsReserved.size() && regsReserved.test(Idx: Reg.id());
245 }
246
247 bool isAllocatable(Register Reg) const {
248 return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(RegNo: Reg) &&
249 !regsReserved.test(Idx: Reg.id());
250 }
251
252 // Analysis information if available
253 LiveVariables *LiveVars = nullptr;
254 LiveIntervals *LiveInts = nullptr;
255 LiveStacks *LiveStks = nullptr;
256 SlotIndexes *Indexes = nullptr;
257
258 /// A class to track the number of reported error and to guarantee that only
259 /// one error is reported at one time.
260 class ReportedErrors {
261 unsigned NumReported = 0;
262 bool AbortOnError;
263
264 public:
265 /// \param AbortOnError -- If set, abort after printing the first error.
266 ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {}
267
268 ~ReportedErrors() {
269 if (!hasError())
270 return;
271 if (AbortOnError)
272 report_fatal_error(reason: "Found " + Twine(NumReported) +
273 " machine code errors.");
274 // Since we haven't aborted, release the lock to allow other threads to
275 // report errors.
276 ReportedErrorsLock->unlock();
277 }
278
279 /// Increment the number of reported errors.
280 /// \returns true if this is the first reported error.
281 bool increment() {
282 // If this is the first error this thread has encountered, grab the lock
283 // to prevent other threads from reporting errors at the same time.
284 // Otherwise we assume we already have the lock.
285 if (!hasError())
286 ReportedErrorsLock->lock();
287 ++NumReported;
288 return NumReported == 1;
289 }
290
291 /// \returns true if an error was reported.
292 bool hasError() { return NumReported; }
293 };
294 ReportedErrors ReportedErrs;
295
296 // This is calculated only when trying to verify convergence control tokens.
297 // Similar to the LLVM IR verifier, we calculate this locally instead of
298 // relying on the pass manager.
299 MachineDominatorTree DT;
300
301 void visitMachineFunctionBefore();
302 void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
303 void visitMachineBundleBefore(const MachineInstr *MI);
304
305 /// Verify that all of \p MI's virtual register operands are scalars.
306 /// \returns True if all virtual register operands are scalar. False
307 /// otherwise.
308 bool verifyAllRegOpsScalar(const MachineInstr &MI,
309 const MachineRegisterInfo &MRI);
310 bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);
311
312 bool verifyGIntrinsicSideEffects(const MachineInstr *MI);
313 bool verifyGIntrinsicConvergence(const MachineInstr *MI);
314 void verifyPreISelGenericInstruction(const MachineInstr *MI);
315
316 void visitMachineInstrBefore(const MachineInstr *MI);
317 void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
318 void visitMachineBundleAfter(const MachineInstr *MI);
319 void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);
320 void visitMachineFunctionAfter();
321
322 void report(const char *msg, const MachineFunction *MF);
323 void report(const char *msg, const MachineBasicBlock *MBB);
324 void report(const char *msg, const MachineInstr *MI);
325 void report(const char *msg, const MachineOperand *MO, unsigned MONum,
326 LLT MOVRegType = LLT{});
327 void report(const Twine &Msg, const MachineInstr *MI);
328
329 void report_context(const LiveInterval &LI) const;
330 void report_context(const LiveRange &LR, VirtRegOrUnit VRegOrUnit,
331 LaneBitmask LaneMask) const;
332 void report_context(const LiveRange::Segment &S) const;
333 void report_context(const VNInfo &VNI) const;
334 void report_context(SlotIndex Pos) const;
335 void report_context(MCPhysReg PhysReg) const;
336 void report_context_liverange(const LiveRange &LR) const;
337 void report_context_lanemask(LaneBitmask LaneMask) const;
338 void report_context_vreg(Register VReg) const;
339 void report_context_vreg_regunit(VirtRegOrUnit VRegOrUnit) const;
340
341 void verifyInlineAsm(const MachineInstr *MI);
342
343 void checkLiveness(const MachineOperand *MO, unsigned MONum);
344 void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,
345 SlotIndex UseIdx, const LiveRange &LR,
346 VirtRegOrUnit VRegOrUnit,
347 LaneBitmask LaneMask = LaneBitmask::getNone());
348 void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,
349 SlotIndex DefIdx, const LiveRange &LR,
350 VirtRegOrUnit VRegOrUnit, bool SubRangeCheck = false,
351 LaneBitmask LaneMask = LaneBitmask::getNone());
352
353 void markReachable(const MachineBasicBlock *MBB);
354 void calcRegsPassed();
355 void checkPHIOps(const MachineBasicBlock &MBB);
356
357 void calcRegsRequired();
358 void verifyLiveVariables();
359 void verifyLiveIntervals();
360 void verifyLiveInterval(const LiveInterval &);
361 void verifyLiveRangeValue(const LiveRange &, const VNInfo *, VirtRegOrUnit,
362 LaneBitmask);
363 void verifyLiveRangeSegment(const LiveRange &,
364 const LiveRange::const_iterator I, VirtRegOrUnit,
365 LaneBitmask);
366 void verifyLiveRange(const LiveRange &, VirtRegOrUnit,
367 LaneBitmask LaneMask = LaneBitmask::getNone());
368
369 void verifyStackFrame();
370 /// Check that the stack protector is the top-most object in the stack.
371 void verifyStackProtector();
372
373 void verifySlotIndexes() const;
374 void verifyProperties(const MachineFunction &MF);
375};
376
377struct MachineVerifierLegacyPass : public MachineFunctionPass {
378 static char ID; // Pass ID, replacement for typeid
379
380 const std::string Banner;
381
382 MachineVerifierLegacyPass(std::string banner = std::string())
383 : MachineFunctionPass(ID), Banner(std::move(banner)) {}
384
385 void getAnalysisUsage(AnalysisUsage &AU) const override {
386 AU.addUsedIfAvailable<LiveStacksWrapperLegacy>();
387 AU.addUsedIfAvailable<LiveVariablesWrapperPass>();
388 AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
389 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
390 AU.setPreservesAll();
391 MachineFunctionPass::getAnalysisUsage(AU);
392 }
393
394 bool runOnMachineFunction(MachineFunction &MF) override {
395 // Skip functions that have known verification problems.
396 // FIXME: Remove this mechanism when all problematic passes have been
397 // fixed.
398 if (MF.getProperties().hasFailsVerification())
399 return false;
400
401 MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);
402 return false;
403 }
404};
405
406} // end anonymous namespace
407
408PreservedAnalyses
409MachineVerifierPass::run(MachineFunction &MF,
410 MachineFunctionAnalysisManager &MFAM) {
411 // Skip functions that have known verification problems.
412 // FIXME: Remove this mechanism when all problematic passes have been
413 // fixed.
414 if (MF.getProperties().hasFailsVerification())
415 return PreservedAnalyses::all();
416 MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);
417 return PreservedAnalyses::all();
418}
419
420char MachineVerifierLegacyPass::ID = 0;
421
422INITIALIZE_PASS(MachineVerifierLegacyPass, "machineverifier",
423 "Verify generated machine code", false, false)
424
425FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {
426 return new MachineVerifierLegacyPass(Banner);
427}
428
429void llvm::verifyMachineFunction(const std::string &Banner,
430 const MachineFunction &MF) {
431 // TODO: Use MFAM after porting below analyses.
432 // LiveVariables *LiveVars;
433 // LiveIntervals *LiveInts;
434 // LiveStacks *LiveStks;
435 // SlotIndexes *Indexes;
436 MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);
437}
438
439bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,
440 bool AbortOnError) const {
441 return MachineVerifier(p, Banner, OS, AbortOnError).verify(MF: *this);
442}
443
444bool MachineFunction::verify(MachineFunctionAnalysisManager &MFAM,
445 const char *Banner, raw_ostream *OS,
446 bool AbortOnError) const {
447 return MachineVerifier(MFAM, Banner, OS, AbortOnError).verify(MF: *this);
448}
449
450bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,
451 const char *Banner, raw_ostream *OS,
452 bool AbortOnError) const {
453 return MachineVerifier(Banner, /*LiveVars=*/nullptr, LiveInts,
454 /*LiveStks=*/nullptr, Indexes, OS, AbortOnError)
455 .verify(MF: *this);
456}
457
458void MachineVerifier::verifySlotIndexes() const {
459 if (Indexes == nullptr)
460 return;
461
462 // Ensure the IdxMBB list is sorted by slot indexes.
463 SlotIndex Last;
464 for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),
465 E = Indexes->MBBIndexEnd(); I != E; ++I) {
466 assert(!Last.isValid() || I->first > Last);
467 Last = I->first;
468 }
469}
470
471void MachineVerifier::verifyProperties(const MachineFunction &MF) {
472 // If a pass has introduced virtual registers without clearing the
473 // NoVRegs property (or set it without allocating the vregs)
474 // then report an error.
475 if (MF.getProperties().hasNoVRegs() && MRI->getNumVirtRegs())
476 report(msg: "Function has NoVRegs property but there are VReg operands", MF: &MF);
477}
478
479bool MachineVerifier::verify(const MachineFunction &MF) {
480 this->MF = &MF;
481 TM = &MF.getTarget();
482 TII = MF.getSubtarget().getInstrInfo();
483 TRI = MF.getSubtarget().getRegisterInfo();
484 RBI = MF.getSubtarget().getRegBankInfo();
485 MRI = &MF.getRegInfo();
486
487 const MachineFunctionProperties &Props = MF.getProperties();
488 const bool isFunctionFailedISel = Props.hasFailedISel();
489
490 // If we're mid-GlobalISel and we already triggered the fallback path then
491 // it's expected that the MIR is somewhat broken but that's ok since we'll
492 // reset it and clear the FailedISel attribute in ResetMachineFunctions.
493 if (isFunctionFailedISel)
494 return true;
495
496 isFunctionRegBankSelected = Props.hasRegBankSelected();
497 isFunctionSelected = Props.hasSelected();
498 isFunctionTracksDebugUserValues = Props.hasTracksDebugUserValues();
499
500 if (PASS) {
501 auto *LISWrapper = PASS->getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
502 LiveInts = LISWrapper ? &LISWrapper->getLIS() : nullptr;
503 // We don't want to verify LiveVariables if LiveIntervals is available.
504 auto *LVWrapper = PASS->getAnalysisIfAvailable<LiveVariablesWrapperPass>();
505 if (!LiveInts)
506 LiveVars = LVWrapper ? &LVWrapper->getLV() : nullptr;
507 auto *LSWrapper = PASS->getAnalysisIfAvailable<LiveStacksWrapperLegacy>();
508 LiveStks = LSWrapper ? &LSWrapper->getLS() : nullptr;
509 auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>();
510 Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
511 }
512 if (MFAM) {
513 MachineFunction &Func = const_cast<MachineFunction &>(MF);
514 LiveInts = MFAM->getCachedResult<LiveIntervalsAnalysis>(IR&: Func);
515 if (!LiveInts)
516 LiveVars = MFAM->getCachedResult<LiveVariablesAnalysis>(IR&: Func);
517 // TODO: LiveStks = MFAM->getCachedResult<LiveStacksAnalysis>(Func);
518 Indexes = MFAM->getCachedResult<SlotIndexesAnalysis>(IR&: Func);
519 }
520
521 verifySlotIndexes();
522
523 verifyProperties(MF);
524
525 visitMachineFunctionBefore();
526 for (const MachineBasicBlock &MBB : MF) {
527 visitMachineBasicBlockBefore(MBB: &MBB);
528 // Keep track of the current bundle header.
529 const MachineInstr *CurBundle = nullptr;
530 // Do we expect the next instruction to be part of the same bundle?
531 bool InBundle = false;
532
533 for (const MachineInstr &MI : MBB.instrs()) {
534 if (MI.getParent() != &MBB) {
535 report(msg: "Bad instruction parent pointer", MBB: &MBB);
536 OS << "Instruction: " << MI;
537 continue;
538 }
539
540 // Check for consistent bundle flags.
541 if (InBundle && !MI.isBundledWithPred())
542 report(msg: "Missing BundledPred flag, "
543 "BundledSucc was set on predecessor",
544 MI: &MI);
545 if (!InBundle && MI.isBundledWithPred())
546 report(msg: "BundledPred flag is set, "
547 "but BundledSucc not set on predecessor",
548 MI: &MI);
549
550 // Is this a bundle header?
551 if (!MI.isInsideBundle()) {
552 if (CurBundle)
553 visitMachineBundleAfter(MI: CurBundle);
554 CurBundle = &MI;
555 visitMachineBundleBefore(MI: CurBundle);
556 } else if (!CurBundle)
557 report(msg: "No bundle header", MI: &MI);
558 visitMachineInstrBefore(MI: &MI);
559 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
560 const MachineOperand &Op = MI.getOperand(i: I);
561 if (Op.getParent() != &MI) {
562 // Make sure to use correct addOperand / removeOperand / ChangeTo
563 // functions when replacing operands of a MachineInstr.
564 report(msg: "Instruction has operand with wrong parent set", MI: &MI);
565 }
566
567 visitMachineOperand(MO: &Op, MONum: I);
568 }
569
570 // Was this the last bundled instruction?
571 InBundle = MI.isBundledWithSucc();
572 }
573 if (CurBundle)
574 visitMachineBundleAfter(MI: CurBundle);
575 if (InBundle)
576 report(msg: "BundledSucc flag set on last instruction in block", MI: &MBB.back());
577 visitMachineBasicBlockAfter(MBB: &MBB);
578 }
579 visitMachineFunctionAfter();
580
581 // Clean up.
582 regsLive.clear();
583 regsDefined.clear();
584 regsDead.clear();
585 regsKilled.clear();
586 regMasks.clear();
587 MBBInfoMap.clear();
588
589 return !ReportedErrs.hasError();
590}
591
592void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
593 assert(MF);
594 OS << '\n';
595 if (ReportedErrs.increment()) {
596 if (Banner)
597 OS << "# " << Banner << '\n';
598
599 if (LiveInts != nullptr)
600 LiveInts->print(O&: OS);
601 else
602 MF->print(OS, Indexes);
603 }
604
605 OS << "*** Bad machine code: " << msg << " ***\n"
606 << "- function: " << MF->getName() << '\n';
607}
608
609void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
610 assert(MBB);
611 report(msg, MF: MBB->getParent());
612 OS << "- basic block: " << printMBBReference(MBB: *MBB) << ' ' << MBB->getName()
613 << " (" << (const void *)MBB << ')';
614 if (Indexes)
615 OS << " [" << Indexes->getMBBStartIdx(mbb: MBB) << ';'
616 << Indexes->getMBBEndIdx(mbb: MBB) << ')';
617 OS << '\n';
618}
619
620void MachineVerifier::report(const char *msg, const MachineInstr *MI) {
621 assert(MI);
622 report(msg, MBB: MI->getParent());
623 OS << "- instruction: ";
624 if (Indexes && Indexes->hasIndex(instr: *MI))
625 OS << Indexes->getInstructionIndex(MI: *MI) << '\t';
626 MI->print(OS, /*IsStandalone=*/true);
627}
628
629void MachineVerifier::report(const char *msg, const MachineOperand *MO,
630 unsigned MONum, LLT MOVRegType) {
631 assert(MO);
632 report(msg, MI: MO->getParent());
633 OS << "- operand " << MONum << ": ";
634 MO->print(os&: OS, TypeToPrint: MOVRegType, TRI);
635 OS << '\n';
636}
637
638void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) {
639 report(msg: Msg.str().c_str(), MI);
640}
641
642void MachineVerifier::report_context(SlotIndex Pos) const {
643 OS << "- at: " << Pos << '\n';
644}
645
646void MachineVerifier::report_context(const LiveInterval &LI) const {
647 OS << "- interval: " << LI << '\n';
648}
649
650void MachineVerifier::report_context(const LiveRange &LR,
651 VirtRegOrUnit VRegOrUnit,
652 LaneBitmask LaneMask) const {
653 report_context_liverange(LR);
654 report_context_vreg_regunit(VRegOrUnit);
655 if (LaneMask.any())
656 report_context_lanemask(LaneMask);
657}
658
659void MachineVerifier::report_context(const LiveRange::Segment &S) const {
660 OS << "- segment: " << S << '\n';
661}
662
663void MachineVerifier::report_context(const VNInfo &VNI) const {
664 OS << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n";
665}
666
667void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
668 OS << "- liverange: " << LR << '\n';
669}
670
671void MachineVerifier::report_context(MCPhysReg PReg) const {
672 OS << "- p. register: " << printReg(Reg: PReg, TRI) << '\n';
673}
674
675void MachineVerifier::report_context_vreg(Register VReg) const {
676 OS << "- v. register: " << printReg(Reg: VReg, TRI) << '\n';
677}
678
679void MachineVerifier::report_context_vreg_regunit(
680 VirtRegOrUnit VRegOrUnit) const {
681 if (VRegOrUnit.isVirtualReg()) {
682 report_context_vreg(VReg: VRegOrUnit.asVirtualReg());
683 } else {
684 OS << "- regunit: " << printRegUnit(Unit: VRegOrUnit.asMCRegUnit(), TRI)
685 << '\n';
686 }
687}
688
689void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {
690 OS << "- lanemask: " << PrintLaneMask(LaneMask) << '\n';
691}
692
693void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
694 BBInfo &MInfo = MBBInfoMap[MBB];
695 if (!MInfo.reachable) {
696 MInfo.reachable = true;
697 for (const MachineBasicBlock *Succ : MBB->successors())
698 markReachable(MBB: Succ);
699 }
700}
701
702void MachineVerifier::visitMachineFunctionBefore() {
703 lastIndex = SlotIndex();
704 regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()
705 : TRI->getReservedRegs(MF: *MF);
706
707 if (!MF->empty())
708 markReachable(MBB: &MF->front());
709
710 // Build a set of the basic blocks in the function.
711 FunctionBlocks.clear();
712 for (const auto &MBB : *MF) {
713 FunctionBlocks.insert(Ptr: &MBB);
714 BBInfo &MInfo = MBBInfoMap[&MBB];
715
716 MInfo.Preds.insert_range(R: MBB.predecessors());
717 if (MInfo.Preds.size() != MBB.pred_size())
718 report(msg: "MBB has duplicate entries in its predecessor list.", MBB: &MBB);
719
720 MInfo.Succs.insert_range(R: MBB.successors());
721 if (MInfo.Succs.size() != MBB.succ_size())
722 report(msg: "MBB has duplicate entries in its successor list.", MBB: &MBB);
723 }
724
725 // Check that the register use lists are sane.
726 MRI->verifyUseLists();
727
728 if (!MF->empty()) {
729 verifyStackFrame();
730 verifyStackProtector();
731 }
732}
733
734static bool hasPHIs(const MachineFunction &MF) {
735 return !MF.getProperties().hasNoPHIs() &&
736 any_of(Range: MF, P: [](const MachineBasicBlock &MBB) {
737 return !MBB.phis().empty();
738 });
739}
740
741void
742MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
743 FirstTerminator = nullptr;
744 FirstNonPHI = nullptr;
745
746 if (MRI->tracksLiveness() && hasPHIs(MF: *MF)) {
747 // If this block has allocatable physical registers live-in, check that
748 // it is an entry block or landing pad.
749 for (const auto &LI : MBB->liveins()) {
750 if (isAllocatable(Reg: LI.PhysReg) && !MBB->isEHPad() &&
751 MBB->getIterator() != MBB->getParent()->begin() &&
752 !MBB->isInlineAsmBrIndirectTarget()) {
753 report(msg: "MBB has allocatable live-in, but isn't entry, landing-pad, or "
754 "inlineasm-br-indirect-target.",
755 MBB);
756 report_context(PReg: LI.PhysReg);
757 }
758 }
759 }
760
761 if (MBB->isIRBlockAddressTaken()) {
762 if (!MBB->getAddressTakenIRBlock()->hasAddressTaken())
763 report(msg: "ir-block-address-taken is associated with basic block not used by "
764 "a blockaddress.",
765 MBB);
766 }
767
768 // Count the number of landing pad successors.
769 SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs;
770 for (const auto *succ : MBB->successors()) {
771 if (succ->isEHPad())
772 LandingPadSuccs.insert(Ptr: succ);
773 if (!FunctionBlocks.count(Ptr: succ))
774 report(msg: "MBB has successor that isn't part of the function.", MBB);
775 if (!MBBInfoMap[succ].Preds.count(Ptr: MBB)) {
776 report(msg: "Inconsistent CFG", MBB);
777 OS << "MBB is not in the predecessor list of the successor "
778 << printMBBReference(MBB: *succ) << ".\n";
779 }
780 }
781
782 // Check the predecessor list.
783 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
784 if (!FunctionBlocks.count(Ptr: Pred))
785 report(msg: "MBB has predecessor that isn't part of the function.", MBB);
786 if (!MBBInfoMap[Pred].Succs.count(Ptr: MBB)) {
787 report(msg: "Inconsistent CFG", MBB);
788 OS << "MBB is not in the successor list of the predecessor "
789 << printMBBReference(MBB: *Pred) << ".\n";
790 }
791 }
792
793 const MCAsmInfo &AsmInfo = TM->getMCAsmInfo();
794 const BasicBlock *BB = MBB->getBasicBlock();
795 const Function &F = MF->getFunction();
796 if (LandingPadSuccs.size() > 1 &&
797 !(AsmInfo.getExceptionHandlingType() == ExceptionHandling::SjLj && BB &&
798 isa<SwitchInst>(Val: BB->getTerminator())) &&
799 !isScopedEHPersonality(Pers: classifyEHPersonality(Pers: F.getPersonalityFn())))
800 report(msg: "MBB has more than one landing pad successor", MBB);
801
802 // Call analyzeBranch. If it succeeds, there several more conditions to check.
803 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
804 SmallVector<MachineOperand, 4> Cond;
805 if (!TII->analyzeBranch(MBB&: *const_cast<MachineBasicBlock *>(MBB), TBB, FBB,
806 Cond)) {
807 // Ok, analyzeBranch thinks it knows what's going on with this block. Let's
808 // check whether its answers match up with reality.
809 if (!TBB && !FBB) {
810 // Block falls through to its successor.
811 if (!MBB->empty() && MBB->back().isBarrier() &&
812 !TII->isPredicated(MI: MBB->back())) {
813 report(msg: "MBB exits via unconditional fall-through but ends with a "
814 "barrier instruction!", MBB);
815 }
816 if (!Cond.empty()) {
817 report(msg: "MBB exits via unconditional fall-through but has a condition!",
818 MBB);
819 }
820 } else if (TBB && !FBB && Cond.empty()) {
821 // Block unconditionally branches somewhere.
822 if (MBB->empty()) {
823 report(msg: "MBB exits via unconditional branch but doesn't contain "
824 "any instructions!", MBB);
825 } else if (!MBB->back().isBarrier()) {
826 report(msg: "MBB exits via unconditional branch but doesn't end with a "
827 "barrier instruction!", MBB);
828 } else if (!MBB->back().isTerminator()) {
829 report(msg: "MBB exits via unconditional branch but the branch isn't a "
830 "terminator instruction!", MBB);
831 }
832 } else if (TBB && !FBB && !Cond.empty()) {
833 // Block conditionally branches somewhere, otherwise falls through.
834 if (MBB->empty()) {
835 report(msg: "MBB exits via conditional branch/fall-through but doesn't "
836 "contain any instructions!", MBB);
837 } else if (MBB->back().isBarrier()) {
838 report(msg: "MBB exits via conditional branch/fall-through but ends with a "
839 "barrier instruction!", MBB);
840 } else if (!MBB->back().isTerminator()) {
841 report(msg: "MBB exits via conditional branch/fall-through but the branch "
842 "isn't a terminator instruction!", MBB);
843 }
844 } else if (TBB && FBB) {
845 // Block conditionally branches somewhere, otherwise branches
846 // somewhere else.
847 if (MBB->empty()) {
848 report(msg: "MBB exits via conditional branch/branch but doesn't "
849 "contain any instructions!", MBB);
850 } else if (!MBB->back().isBarrier()) {
851 report(msg: "MBB exits via conditional branch/branch but doesn't end with a "
852 "barrier instruction!", MBB);
853 } else if (!MBB->back().isTerminator()) {
854 report(msg: "MBB exits via conditional branch/branch but the branch "
855 "isn't a terminator instruction!", MBB);
856 }
857 if (Cond.empty()) {
858 report(msg: "MBB exits via conditional branch/branch but there's no "
859 "condition!", MBB);
860 }
861 } else {
862 report(msg: "analyzeBranch returned invalid data!", MBB);
863 }
864
865 // Now check that the successors match up with the answers reported by
866 // analyzeBranch.
867 if (TBB && !MBB->isSuccessor(MBB: TBB))
868 report(msg: "MBB exits via jump or conditional branch, but its target isn't a "
869 "CFG successor!",
870 MBB);
871 if (FBB && !MBB->isSuccessor(MBB: FBB))
872 report(msg: "MBB exits via conditional branch, but its target isn't a CFG "
873 "successor!",
874 MBB);
875
876 // There might be a fallthrough to the next block if there's either no
877 // unconditional true branch, or if there's a condition, and one of the
878 // branches is missing.
879 bool Fallthrough = !TBB || (!Cond.empty() && !FBB);
880
881 // A conditional fallthrough must be an actual CFG successor, not
882 // unreachable. (Conversely, an unconditional fallthrough might not really
883 // be a successor, because the block might end in unreachable.)
884 if (!Cond.empty() && !FBB) {
885 MachineFunction::const_iterator MBBI = std::next(x: MBB->getIterator());
886 if (MBBI == MF->end()) {
887 report(msg: "MBB conditionally falls through out of function!", MBB);
888 } else if (!MBB->isSuccessor(MBB: &*MBBI))
889 report(msg: "MBB exits via conditional branch/fall-through but the CFG "
890 "successors don't match the actual successors!",
891 MBB);
892 }
893
894 // Verify that there aren't any extra un-accounted-for successors.
895 for (const MachineBasicBlock *SuccMBB : MBB->successors()) {
896 // If this successor is one of the branch targets, it's okay.
897 if (SuccMBB == TBB || SuccMBB == FBB)
898 continue;
899 // If we might have a fallthrough, and the successor is the fallthrough
900 // block, that's also ok.
901 if (Fallthrough && SuccMBB == MBB->getNextNode())
902 continue;
903 // Also accept successors which are for exception-handling or might be
904 // inlineasm_br targets.
905 if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())
906 continue;
907 report(msg: "MBB has unexpected successors which are not branch targets, "
908 "fallthrough, EHPads, or inlineasm_br targets.",
909 MBB);
910 }
911 }
912
913 regsLive.clear();
914 if (MRI->tracksLiveness()) {
915 for (const auto &LI : MBB->liveins()) {
916 if (!LI.PhysReg.isPhysical()) {
917 report(msg: "MBB live-in list contains non-physical register", MBB);
918 continue;
919 }
920 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: LI.PhysReg));
921 }
922 }
923
924 const MachineFrameInfo &MFI = MF->getFrameInfo();
925 BitVector PR = MFI.getPristineRegs(MF: *MF);
926 for (unsigned I : PR.set_bits())
927 regsLive.insert_range(R: TRI->subregs_inclusive(Reg: I));
928
929 regsKilled.clear();
930 regsDefined.clear();
931
932 if (Indexes)
933 lastIndex = Indexes->getMBBStartIdx(mbb: MBB);
934}
935
936// This function gets called for all bundle headers, including normal
937// stand-alone unbundled instructions.
938void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
939 if (Indexes && Indexes->hasIndex(instr: *MI)) {
940 SlotIndex idx = Indexes->getInstructionIndex(MI: *MI);
941 if (!(idx > lastIndex)) {
942 report(msg: "Instruction index out of order", MI);
943 OS << "Last instruction was at " << lastIndex << '\n';
944 }
945 lastIndex = idx;
946 }
947
948 // Ensure non-terminators don't follow terminators.
949 if (MI->isTerminator()) {
950 if (!FirstTerminator)
951 FirstTerminator = MI;
952 } else if (FirstTerminator) {
953 // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to
954 // precede non-terminators.
955 if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) {
956 report(msg: "Non-terminator instruction after the first terminator", MI);
957 OS << "First terminator was:\t" << *FirstTerminator;
958 }
959 }
960}
961
962// The operands on an INLINEASM instruction must follow a template.
963// Verify that the flag operands make sense.
964void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
965 // The first two operands on INLINEASM are the asm string and global flags.
966 if (MI->getNumOperands() < 2) {
967 report(msg: "Too few operands on inline asm", MI);
968 return;
969 }
970 if (!MI->getOperand(i: 0).isSymbol())
971 report(msg: "Asm string must be an external symbol", MI);
972 if (!MI->getOperand(i: 1).isImm())
973 report(msg: "Asm flags must be an immediate", MI);
974 // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
975 // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,
976 // and Extra_IsConvergent = 32, Extra_MayUnwind = 64.
977 if (!isUInt<7>(x: MI->getOperand(i: 1).getImm()))
978 report(msg: "Unknown asm flags", MO: &MI->getOperand(i: 1), MONum: 1);
979
980 static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");
981
982 unsigned OpNo = InlineAsm::MIOp_FirstOperand;
983 unsigned NumOps;
984 for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
985 const MachineOperand &MO = MI->getOperand(i: OpNo);
986 // There may be implicit ops after the fixed operands.
987 if (!MO.isImm())
988 break;
989 const InlineAsm::Flag F(MO.getImm());
990 NumOps = 1 + F.getNumOperandRegisters();
991 }
992
993 if (OpNo > MI->getNumOperands())
994 report(msg: "Missing operands in last group", MI);
995
996 // An optional MDNode follows the groups.
997 if (OpNo < MI->getNumOperands() && MI->getOperand(i: OpNo).isMetadata())
998 ++OpNo;
999
1000 // All trailing operands must be implicit registers.
1001 for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
1002 const MachineOperand &MO = MI->getOperand(i: OpNo);
1003 if (!MO.isReg() || !MO.isImplicit())
1004 report(msg: "Expected implicit register after groups", MO: &MO, MONum: OpNo);
1005 }
1006
1007 if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) {
1008 const MachineBasicBlock *MBB = MI->getParent();
1009
1010 for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands();
1011 i != e; ++i) {
1012 const MachineOperand &MO = MI->getOperand(i);
1013
1014 if (!MO.isMBB())
1015 continue;
1016
1017 // Check the successor & predecessor lists look ok, assume they are
1018 // not. Find the indirect target without going through the successors.
1019 const MachineBasicBlock *IndirectTargetMBB = MO.getMBB();
1020 if (!IndirectTargetMBB) {
1021 report(msg: "INLINEASM_BR indirect target does not exist", MO: &MO, MONum: i);
1022 break;
1023 }
1024
1025 if (!MBB->isSuccessor(MBB: IndirectTargetMBB))
1026 report(msg: "INLINEASM_BR indirect target missing from successor list", MO: &MO,
1027 MONum: i);
1028
1029 if (!IndirectTargetMBB->isPredecessor(MBB))
1030 report(msg: "INLINEASM_BR indirect target predecessor list missing parent",
1031 MO: &MO, MONum: i);
1032 }
1033 }
1034}
1035
1036bool MachineVerifier::verifyAllRegOpsScalar(const MachineInstr &MI,
1037 const MachineRegisterInfo &MRI) {
1038 if (none_of(Range: MI.explicit_operands(), P: [&MRI](const MachineOperand &Op) {
1039 if (!Op.isReg())
1040 return false;
1041 const auto Reg = Op.getReg();
1042 if (Reg.isPhysical())
1043 return false;
1044 return !MRI.getType(Reg).isScalar();
1045 }))
1046 return true;
1047 report(msg: "All register operands must have scalar types", MI: &MI);
1048 return false;
1049}
1050
1051/// Check that types are consistent when two operands need to have the same
1052/// number of vector elements.
1053/// \return true if the types are valid.
1054bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,
1055 const MachineInstr *MI) {
1056 if (Ty0.isVector() != Ty1.isVector()) {
1057 report(msg: "operand types must be all-vector or all-scalar", MI);
1058 // Generally we try to report as many issues as possible at once, but in
1059 // this case it's not clear what should we be comparing the size of the
1060 // scalar with: the size of the whole vector or its lane. Instead of
1061 // making an arbitrary choice and emitting not so helpful message, let's
1062 // avoid the extra noise and stop here.
1063 return false;
1064 }
1065
1066 if (Ty0.isVector() && Ty0.getElementCount() != Ty1.getElementCount()) {
1067 report(msg: "operand types must preserve number of vector elements", MI);
1068 return false;
1069 }
1070
1071 return true;
1072}
1073
1074bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
1075 auto Opcode = MI->getOpcode();
1076 bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
1077 Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
1078 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1079 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1080 AttributeSet Attrs = Intrinsic::getFnAttributes(
1081 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1082 bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
1083 if (NoSideEffects && DeclHasSideEffects) {
1084 report(Msg: Twine(TII->getName(Opcode),
1085 " used with intrinsic that accesses memory"),
1086 MI);
1087 return false;
1088 }
1089 if (!NoSideEffects && !DeclHasSideEffects) {
1090 report(Msg: Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI);
1091 return false;
1092 }
1093 }
1094
1095 return true;
1096}
1097
1098bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
1099 auto Opcode = MI->getOpcode();
1100 bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
1101 Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
1102 unsigned IntrID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
1103 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
1104 AttributeSet Attrs = Intrinsic::getFnAttributes(
1105 C&: MF->getFunction().getContext(), id: static_cast<Intrinsic::ID>(IntrID));
1106 bool DeclIsConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
1107 if (NotConvergent && DeclIsConvergent) {
1108 report(Msg: Twine(TII->getName(Opcode), " used with a convergent intrinsic"),
1109 MI);
1110 return false;
1111 }
1112 if (!NotConvergent && !DeclIsConvergent) {
1113 report(
1114 Msg: Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"),
1115 MI);
1116 return false;
1117 }
1118 }
1119
1120 return true;
1121}
1122
1123void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
1124 if (isFunctionSelected)
1125 report(msg: "Unexpected generic instruction in a Selected function", MI);
1126
1127 const MCInstrDesc &MCID = MI->getDesc();
1128 unsigned NumOps = MI->getNumOperands();
1129
1130 // Branches must reference a basic block if they are not indirect
1131 if (MI->isBranch() && !MI->isIndirectBranch()) {
1132 bool HasMBB = false;
1133 for (const MachineOperand &Op : MI->operands()) {
1134 if (Op.isMBB()) {
1135 HasMBB = true;
1136 break;
1137 }
1138 }
1139
1140 if (!HasMBB) {
1141 report(msg: "Branch instruction is missing a basic block operand or "
1142 "isIndirectBranch property",
1143 MI);
1144 }
1145 }
1146
1147 // Check types.
1148 SmallVector<LLT, 4> Types;
1149 for (unsigned I = 0, E = std::min(a: MCID.getNumOperands(), b: NumOps);
1150 I != E; ++I) {
1151 if (!MCID.operands()[I].isGenericType())
1152 continue;
1153 // Generic instructions specify type equality constraints between some of
1154 // their operands. Make sure these are consistent.
1155 size_t TypeIdx = MCID.operands()[I].getGenericTypeIndex();
1156 Types.resize(N: std::max(a: TypeIdx + 1, b: Types.size()));
1157
1158 const MachineOperand *MO = &MI->getOperand(i: I);
1159 if (!MO->isReg()) {
1160 report(msg: "generic instruction must use register operands", MI);
1161 continue;
1162 }
1163
1164 LLT OpTy = MRI->getType(Reg: MO->getReg());
1165 // Don't report a type mismatch if there is no actual mismatch, only a
1166 // type missing, to reduce noise:
1167 if (OpTy.isValid()) {
1168 // Only the first valid type for a type index will be printed: don't
1169 // overwrite it later so it's always clear which type was expected:
1170 if (!Types[TypeIdx].isValid())
1171 Types[TypeIdx] = OpTy;
1172 else if (Types[TypeIdx] != OpTy)
1173 report(msg: "Type mismatch in generic instruction", MO, MONum: I, MOVRegType: OpTy);
1174 } else {
1175 // Generic instructions must have types attached to their operands.
1176 report(msg: "Generic instruction is missing a virtual register type", MO, MONum: I);
1177 }
1178 }
1179
1180 // Generic opcodes must not have physical register operands.
1181 for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
1182 const MachineOperand *MO = &MI->getOperand(i: I);
1183 if (MO->isReg() && MO->getReg().isPhysical())
1184 report(msg: "Generic instruction cannot have physical register", MO, MONum: I);
1185 }
1186
1187 // Avoid out of bounds in checks below. This was already reported earlier.
1188 if (MI->getNumOperands() < MCID.getNumOperands())
1189 return;
1190
1191 StringRef ErrorInfo;
1192 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
1193 report(msg: ErrorInfo.data(), MI);
1194
1195 // Verify properties of various specific instruction types
1196 unsigned Opc = MI->getOpcode();
1197 switch (Opc) {
1198 case TargetOpcode::G_ASSERT_SEXT:
1199 case TargetOpcode::G_ASSERT_ZEXT: {
1200 std::string OpcName =
1201 Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT";
1202 if (!MI->getOperand(i: 2).isImm()) {
1203 report(Msg: Twine(OpcName, " expects an immediate operand #2"), MI);
1204 break;
1205 }
1206
1207 Register Dst = MI->getOperand(i: 0).getReg();
1208 Register Src = MI->getOperand(i: 1).getReg();
1209 LLT SrcTy = MRI->getType(Reg: Src);
1210 int64_t Imm = MI->getOperand(i: 2).getImm();
1211 if (Imm <= 0) {
1212 report(Msg: Twine(OpcName, " size must be >= 1"), MI);
1213 break;
1214 }
1215
1216 if (Imm >= SrcTy.getScalarSizeInBits()) {
1217 report(Msg: Twine(OpcName, " size must be less than source bit width"), MI);
1218 break;
1219 }
1220
1221 const RegisterBank *SrcRB = RBI->getRegBank(Reg: Src, MRI: *MRI, TRI: *TRI);
1222 const RegisterBank *DstRB = RBI->getRegBank(Reg: Dst, MRI: *MRI, TRI: *TRI);
1223
1224 // Allow only the source bank to be set.
1225 if ((SrcRB && DstRB && SrcRB != DstRB) || (DstRB && !SrcRB)) {
1226 report(Msg: Twine(OpcName, " cannot change register bank"), MI);
1227 break;
1228 }
1229
1230 // Don't allow a class change. Do allow member class->regbank.
1231 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: Dst);
1232 if (DstRC && DstRC != MRI->getRegClassOrNull(Reg: Src)) {
1233 report(
1234 Msg: Twine(OpcName, " source and destination register classes must match"),
1235 MI);
1236 break;
1237 }
1238
1239 break;
1240 }
1241
1242 case TargetOpcode::G_CONSTANT:
1243 case TargetOpcode::G_FCONSTANT: {
1244 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1245 if (DstTy.isVector())
1246 report(msg: "Instruction cannot use a vector result type", MI);
1247
1248 if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {
1249 if (!MI->getOperand(i: 1).isCImm()) {
1250 report(msg: "G_CONSTANT operand must be cimm", MI);
1251 break;
1252 }
1253
1254 const ConstantInt *CI = MI->getOperand(i: 1).getCImm();
1255 if (CI->getBitWidth() != DstTy.getSizeInBits())
1256 report(msg: "inconsistent constant size", MI);
1257 } else {
1258 if (!MI->getOperand(i: 1).isFPImm()) {
1259 report(msg: "G_FCONSTANT operand must be fpimm", MI);
1260 break;
1261 }
1262 const ConstantFP *CF = MI->getOperand(i: 1).getFPImm();
1263
1264 if (APFloat::getSizeInBits(Sem: CF->getValueAPF().getSemantics()) !=
1265 DstTy.getSizeInBits()) {
1266 report(msg: "inconsistent constant size", MI);
1267 }
1268 }
1269
1270 break;
1271 }
1272 case TargetOpcode::G_LOAD:
1273 case TargetOpcode::G_STORE:
1274 case TargetOpcode::G_ZEXTLOAD:
1275 case TargetOpcode::G_SEXTLOAD:
1276 case TargetOpcode::G_FPEXTLOAD:
1277 case TargetOpcode::G_FPTRUNCSTORE: {
1278 LLT ValTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1279 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1280 if (!PtrTy.isPointer())
1281 report(msg: "Generic memory instruction must access a pointer", MI);
1282
1283 // Generic loads and stores must have a single MachineMemOperand
1284 // describing that access.
1285 if (!MI->hasOneMemOperand()) {
1286 report(msg: "Generic instruction accessing memory must have one mem operand",
1287 MI);
1288 } else {
1289 const MachineMemOperand &MMO = **MI->memoperands_begin();
1290 if (isa<GExtLoad>(Val: *MI)) {
1291 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1292 RHS: ValTy.getSizeInBits()))
1293 report(msg: "Generic extload must have a narrower memory type", MI);
1294 } else if (isa<GFPTruncStore>(Val: *MI)) {
1295 if (TypeSize::isKnownGE(LHS: MMO.getSizeInBits().getValue(),
1296 RHS: ValTy.getSizeInBits()))
1297 report(msg: "Generic truncstore must have a narrower memory type", MI);
1298 } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
1299 if (TypeSize::isKnownGT(LHS: MMO.getSize().getValue(),
1300 RHS: ValTy.getSizeInBytes()))
1301 report(msg: "load memory size cannot exceed result size", MI);
1302
1303 if (MMO.getRanges()) {
1304 ConstantInt *i =
1305 mdconst::extract<ConstantInt>(MD: MMO.getRanges()->getOperand(I: 0));
1306 const LLT RangeTy = LLT::scalar(SizeInBits: i->getIntegerType()->getBitWidth());
1307 const LLT MemTy = MMO.getMemoryType();
1308 if (MemTy.getScalarType() != RangeTy ||
1309 ValTy.isScalar() != MemTy.isScalar() ||
1310 (ValTy.isVector() &&
1311 ValTy.getNumElements() != MemTy.getNumElements())) {
1312 report(msg: "range is incompatible with the result type", MI);
1313 }
1314 }
1315 } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
1316 if (TypeSize::isKnownLT(LHS: ValTy.getSizeInBytes(),
1317 RHS: MMO.getSize().getValue()))
1318 report(msg: "store memory size cannot exceed value size", MI);
1319 }
1320
1321 const AtomicOrdering Order = MMO.getSuccessOrdering();
1322 if (isa<GAnyStore>(Val: *MI)) {
1323 if (Order == AtomicOrdering::Acquire ||
1324 Order == AtomicOrdering::AcquireRelease)
1325 report(msg: "atomic store cannot use acquire ordering", MI);
1326
1327 } else {
1328 if (Order == AtomicOrdering::Release ||
1329 Order == AtomicOrdering::AcquireRelease)
1330 report(msg: "atomic load cannot use release ordering", MI);
1331 }
1332 }
1333
1334 break;
1335 }
1336 case TargetOpcode::G_PHI: {
1337 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1338 if (!DstTy.isValid() || !all_of(Range: drop_begin(RangeOrContainer: MI->operands()),
1339 P: [this, &DstTy](const MachineOperand &MO) {
1340 if (!MO.isReg())
1341 return true;
1342 LLT Ty = MRI->getType(Reg: MO.getReg());
1343 if (!Ty.isValid() || (Ty != DstTy))
1344 return false;
1345 return true;
1346 }))
1347 report(msg: "Generic Instruction G_PHI has operands with incompatible/missing "
1348 "types",
1349 MI);
1350 break;
1351 }
1352 case TargetOpcode::G_BITCAST: {
1353 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1354 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1355 if (!DstTy.isValid() || !SrcTy.isValid())
1356 break;
1357
1358 if (SrcTy.isPointer() != DstTy.isPointer())
1359 report(msg: "bitcast cannot convert between pointers and other types", MI);
1360
1361 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1362 report(msg: "bitcast sizes must match", MI);
1363
1364 bool SameType = SrcTy.getKind() == DstTy.getKind();
1365 if (SameType && SrcTy.isPointerOrPointerVector())
1366 SameType &= SrcTy.getAddressSpace() == DstTy.getAddressSpace();
1367
1368 SameType &= SrcTy.getScalarSizeInBits() == DstTy.getScalarSizeInBits();
1369
1370 if (SameType && SrcTy.isVector())
1371 SameType &= SrcTy.getElementCount() == DstTy.getElementCount();
1372 if (SameType && SrcTy.isFloatOrFloatVector())
1373 SameType &= SrcTy.getFpSemantics() == DstTy.getFpSemantics();
1374
1375 if (SameType)
1376 report(msg: "bitcast must change the type", MI);
1377
1378 break;
1379 }
1380 case TargetOpcode::G_INTTOPTR:
1381 case TargetOpcode::G_PTRTOINT:
1382 case TargetOpcode::G_ADDRSPACE_CAST: {
1383 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1384 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1385 if (!DstTy.isValid() || !SrcTy.isValid())
1386 break;
1387
1388 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1389
1390 DstTy = DstTy.getScalarType();
1391 SrcTy = SrcTy.getScalarType();
1392
1393 if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {
1394 if (!DstTy.isPointer())
1395 report(msg: "inttoptr result type must be a pointer", MI);
1396 if (SrcTy.isPointer())
1397 report(msg: "inttoptr source type must not be a pointer", MI);
1398 } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {
1399 if (!SrcTy.isPointer())
1400 report(msg: "ptrtoint source type must be a pointer", MI);
1401 if (DstTy.isPointer())
1402 report(msg: "ptrtoint result type must not be a pointer", MI);
1403 } else {
1404 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
1405 if (!SrcTy.isPointer() || !DstTy.isPointer())
1406 report(msg: "addrspacecast types must be pointers", MI);
1407 else {
1408 if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())
1409 report(msg: "addrspacecast must convert different address spaces", MI);
1410 }
1411 }
1412
1413 break;
1414 }
1415 case TargetOpcode::G_PTR_ADD: {
1416 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1417 LLT PtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1418 LLT OffsetTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1419 if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())
1420 break;
1421
1422 if (!PtrTy.isPointerOrPointerVector())
1423 report(msg: "gep first operand must be a pointer", MI);
1424
1425 if (OffsetTy.isPointerOrPointerVector())
1426 report(msg: "gep offset operand must not be a pointer", MI);
1427
1428 if (PtrTy.isPointerOrPointerVector()) {
1429 const DataLayout &DL = MF->getDataLayout();
1430 unsigned AS = PtrTy.getAddressSpace();
1431 unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;
1432 if (OffsetTy.getScalarSizeInBits() != IndexSizeInBits) {
1433 report(msg: "gep offset operand must match index size for address space",
1434 MI);
1435 }
1436 }
1437
1438 // TODO: Is the offset allowed to be a scalar with a vector?
1439 break;
1440 }
1441 case TargetOpcode::G_PTRMASK: {
1442 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1443 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1444 LLT MaskTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1445 if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid())
1446 break;
1447
1448 if (!DstTy.isPointerOrPointerVector())
1449 report(msg: "ptrmask result type must be a pointer", MI);
1450
1451 if (!MaskTy.getScalarType().isScalar())
1452 report(msg: "ptrmask mask type must be an integer", MI);
1453
1454 verifyVectorElementMatch(Ty0: DstTy, Ty1: MaskTy, MI);
1455 break;
1456 }
1457 case TargetOpcode::G_SEXT:
1458 case TargetOpcode::G_ZEXT:
1459 case TargetOpcode::G_ANYEXT:
1460 case TargetOpcode::G_TRUNC:
1461 case TargetOpcode::G_TRUNC_SSAT_S:
1462 case TargetOpcode::G_TRUNC_SSAT_U:
1463 case TargetOpcode::G_TRUNC_USAT_U:
1464 case TargetOpcode::G_FPEXT:
1465 case TargetOpcode::G_FPTRUNC: {
1466 // Number of operands and presense of types is already checked (and
1467 // reported in case of any issues), so no need to report them again. As
1468 // we're trying to report as many issues as possible at once, however, the
1469 // instructions aren't guaranteed to have the right number of operands or
1470 // types attached to them at this point
1471 assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
1472 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1473 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1474 if (!DstTy.isValid() || !SrcTy.isValid())
1475 break;
1476
1477 if (DstTy.isPointerOrPointerVector() || SrcTy.isPointerOrPointerVector())
1478 report(msg: "Generic extend/truncate can not operate on pointers", MI);
1479
1480 verifyVectorElementMatch(Ty0: DstTy, Ty1: SrcTy, MI);
1481
1482 unsigned DstSize = DstTy.getScalarSizeInBits();
1483 unsigned SrcSize = SrcTy.getScalarSizeInBits();
1484 switch (MI->getOpcode()) {
1485 default:
1486 if (DstSize <= SrcSize)
1487 report(msg: "Generic extend has destination type no larger than source", MI);
1488 break;
1489 case TargetOpcode::G_TRUNC:
1490 case TargetOpcode::G_TRUNC_SSAT_S:
1491 case TargetOpcode::G_TRUNC_SSAT_U:
1492 case TargetOpcode::G_TRUNC_USAT_U:
1493 case TargetOpcode::G_FPTRUNC:
1494 if (DstSize >= SrcSize)
1495 report(msg: "Generic truncate has destination type no smaller than source",
1496 MI);
1497 break;
1498 }
1499 break;
1500 }
1501 case TargetOpcode::G_SELECT: {
1502 LLT SelTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1503 LLT CondTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1504 if (!SelTy.isValid() || !CondTy.isValid())
1505 break;
1506
1507 // Scalar condition select on a vector is valid.
1508 if (CondTy.isVector())
1509 verifyVectorElementMatch(Ty0: SelTy, Ty1: CondTy, MI);
1510 break;
1511 }
1512 case TargetOpcode::G_MERGE_VALUES: {
1513 // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,
1514 // e.g. s2N = MERGE sN, sN
1515 // Merging multiple scalars into a vector is not allowed, should use
1516 // G_BUILD_VECTOR for that.
1517 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1518 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1519 if (DstTy.isVector() || SrcTy.isVector())
1520 report(msg: "G_MERGE_VALUES cannot operate on vectors", MI);
1521
1522 const unsigned NumOps = MI->getNumOperands();
1523 if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1))
1524 report(msg: "G_MERGE_VALUES result size is inconsistent", MI);
1525
1526 for (unsigned I = 2; I != NumOps; ++I) {
1527 if (MRI->getType(Reg: MI->getOperand(i: I).getReg()) != SrcTy)
1528 report(msg: "G_MERGE_VALUES source types do not match", MI);
1529 }
1530
1531 break;
1532 }
1533 case TargetOpcode::G_UNMERGE_VALUES: {
1534 unsigned NumDsts = MI->getNumOperands() - 1;
1535 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1536 for (unsigned i = 1; i < NumDsts; ++i) {
1537 if (MRI->getType(Reg: MI->getOperand(i).getReg()) != DstTy) {
1538 report(msg: "G_UNMERGE_VALUES destination types do not match", MI);
1539 break;
1540 }
1541 }
1542
1543 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: NumDsts).getReg());
1544 if (DstTy.isVector()) {
1545 // This case is the converse of G_CONCAT_VECTORS.
1546 if (!SrcTy.isVector() ||
1547 (SrcTy.getScalarType() != DstTy.getScalarType() &&
1548 !SrcTy.isPointerVector()) ||
1549 SrcTy.isScalableVector() != DstTy.isScalableVector() ||
1550 SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1551 report(msg: "G_UNMERGE_VALUES source operand does not match vector "
1552 "destination operands",
1553 MI);
1554 } else if (SrcTy.isVector()) {
1555 // This case is the converse of G_BUILD_VECTOR, but relaxed to allow
1556 // mismatched types as long as the total size matches:
1557 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<4 x s32>)
1558 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())
1559 report(msg: "G_UNMERGE_VALUES vector source operand does not match scalar "
1560 "destination operands",
1561 MI);
1562 } else {
1563 // This case is the converse of G_MERGE_VALUES.
1564 if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits()) {
1565 report(msg: "G_UNMERGE_VALUES scalar source operand does not match scalar "
1566 "destination operands",
1567 MI);
1568 }
1569 }
1570 break;
1571 }
1572 case TargetOpcode::G_BUILD_VECTOR: {
1573 // Source types must be scalars, dest type a vector. Total size of scalars
1574 // must match the dest vector size.
1575 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1576 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1577 if (!DstTy.isVector() || SrcEltTy.isVector()) {
1578 report(msg: "G_BUILD_VECTOR must produce a vector from scalar operands", MI);
1579 break;
1580 }
1581
1582 if (DstTy.getElementType() != SrcEltTy)
1583 report(msg: "G_BUILD_VECTOR result element type must match source type", MI);
1584
1585 if (DstTy.getNumElements() != MI->getNumOperands() - 1)
1586 report(msg: "G_BUILD_VECTOR must have an operand for each element", MI);
1587
1588 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1589 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1590 report(msg: "G_BUILD_VECTOR source operand types are not homogeneous", MI);
1591
1592 break;
1593 }
1594 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1595 // Source types must be scalars, dest type a vector. Scalar types must be
1596 // larger than the dest vector elt type, as this is a truncating operation.
1597 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1598 LLT SrcEltTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1599 if (!DstTy.isVector() || SrcEltTy.isVector())
1600 report(msg: "G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands",
1601 MI);
1602 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1603 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1604 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not homogeneous",
1605 MI);
1606 if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits())
1607 report(msg: "G_BUILD_VECTOR_TRUNC source operand types are not larger than "
1608 "dest elt type",
1609 MI);
1610 break;
1611 }
1612 case TargetOpcode::G_CONCAT_VECTORS: {
1613 // Source types should be vectors, and total size should match the dest
1614 // vector size.
1615 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1616 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1617 if (!DstTy.isVector() || !SrcTy.isVector())
1618 report(msg: "G_CONCAT_VECTOR requires vector source and destination operands",
1619 MI);
1620
1621 if (MI->getNumOperands() < 3)
1622 report(msg: "G_CONCAT_VECTOR requires at least 2 source operands", MI);
1623
1624 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands(), N: 2))
1625 if (MRI->getType(Reg: MI->getOperand(i: 1).getReg()) != MRI->getType(Reg: MO.getReg()))
1626 report(msg: "G_CONCAT_VECTOR source operand types are not homogeneous", MI);
1627 if (DstTy.getElementCount() !=
1628 SrcTy.getElementCount() * (MI->getNumOperands() - 1))
1629 report(msg: "G_CONCAT_VECTOR num dest and source elements should match", MI);
1630 break;
1631 }
1632 case TargetOpcode::G_ICMP:
1633 case TargetOpcode::G_FCMP: {
1634 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1635 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1636
1637 if ((DstTy.isVector() != SrcTy.isVector()) ||
1638 (DstTy.isVector() &&
1639 DstTy.getElementCount() != SrcTy.getElementCount()))
1640 report(msg: "Generic vector icmp/fcmp must preserve number of lanes", MI);
1641
1642 break;
1643 }
1644 case TargetOpcode::G_SCMP:
1645 case TargetOpcode::G_UCMP: {
1646 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1647 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1648
1649 if (SrcTy.isPointerOrPointerVector()) {
1650 report(msg: "Generic scmp/ucmp does not support pointers as operands", MI);
1651 break;
1652 }
1653
1654 if (DstTy.isPointerOrPointerVector()) {
1655 report(msg: "Generic scmp/ucmp does not support pointers as a result", MI);
1656 break;
1657 }
1658
1659 if (DstTy.getScalarSizeInBits() < 2) {
1660 report(msg: "Result type must be at least 2 bits wide", MI);
1661 break;
1662 }
1663
1664 if ((DstTy.isVector() != SrcTy.isVector()) ||
1665 (DstTy.isVector() &&
1666 DstTy.getElementCount() != SrcTy.getElementCount())) {
1667 report(msg: "Generic vector scmp/ucmp must preserve number of lanes", MI);
1668 break;
1669 }
1670
1671 break;
1672 }
1673 case TargetOpcode::G_EXTRACT: {
1674 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1675 if (!SrcOp.isReg()) {
1676 report(msg: "extract source must be a register", MI);
1677 break;
1678 }
1679
1680 const MachineOperand &OffsetOp = MI->getOperand(i: 2);
1681 if (!OffsetOp.isImm()) {
1682 report(msg: "extract offset must be a constant", MI);
1683 break;
1684 }
1685
1686 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1687 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1688 if (SrcSize == DstSize)
1689 report(msg: "extract source must be larger than result", MI);
1690
1691 if (DstSize + OffsetOp.getImm() > SrcSize)
1692 report(msg: "extract reads past end of register", MI);
1693 break;
1694 }
1695 case TargetOpcode::G_INSERT: {
1696 const MachineOperand &SrcOp = MI->getOperand(i: 2);
1697 if (!SrcOp.isReg()) {
1698 report(msg: "insert source must be a register", MI);
1699 break;
1700 }
1701
1702 const MachineOperand &OffsetOp = MI->getOperand(i: 3);
1703 if (!OffsetOp.isImm()) {
1704 report(msg: "insert offset must be a constant", MI);
1705 break;
1706 }
1707
1708 unsigned DstSize = MRI->getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
1709 unsigned SrcSize = MRI->getType(Reg: SrcOp.getReg()).getSizeInBits();
1710
1711 if (DstSize <= SrcSize)
1712 report(msg: "inserted size must be smaller than total register", MI);
1713
1714 if (SrcSize + OffsetOp.getImm() > DstSize)
1715 report(msg: "insert writes past end of register", MI);
1716
1717 break;
1718 }
1719 case TargetOpcode::G_JUMP_TABLE: {
1720 if (!MI->getOperand(i: 1).isJTI())
1721 report(msg: "G_JUMP_TABLE source operand must be a jump table index", MI);
1722 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1723 if (!DstTy.isPointer())
1724 report(msg: "G_JUMP_TABLE dest operand must have a pointer type", MI);
1725 break;
1726 }
1727 case TargetOpcode::G_BRJT: {
1728 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
1729 report(msg: "G_BRJT src operand 0 must be a pointer type", MI);
1730
1731 if (!MI->getOperand(i: 1).isJTI())
1732 report(msg: "G_BRJT src operand 1 must be a jump table index", MI);
1733
1734 const auto &IdxOp = MI->getOperand(i: 2);
1735 if (!IdxOp.isReg() || MRI->getType(Reg: IdxOp.getReg()).isPointer())
1736 report(msg: "G_BRJT src operand 2 must be a scalar reg type", MI);
1737 break;
1738 }
1739 case TargetOpcode::G_INTRINSIC:
1740 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1741 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1742 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
1743 // TODO: Should verify number of def and use operands, but the current
1744 // interface requires passing in IR types for mangling.
1745 const MachineOperand &IntrIDOp = MI->getOperand(i: MI->getNumExplicitDefs());
1746 if (!IntrIDOp.isIntrinsicID()) {
1747 report(msg: "G_INTRINSIC first src operand must be an intrinsic ID", MI);
1748 break;
1749 }
1750
1751 if (!verifyGIntrinsicSideEffects(MI))
1752 break;
1753 if (!verifyGIntrinsicConvergence(MI))
1754 break;
1755
1756 break;
1757 }
1758 case TargetOpcode::G_SEXT_INREG: {
1759 if (!MI->getOperand(i: 2).isImm()) {
1760 report(msg: "G_SEXT_INREG expects an immediate operand #2", MI);
1761 break;
1762 }
1763
1764 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1765 int64_t Imm = MI->getOperand(i: 2).getImm();
1766 if (Imm <= 0)
1767 report(msg: "G_SEXT_INREG size must be >= 1", MI);
1768 if (Imm >= SrcTy.getScalarSizeInBits())
1769 report(msg: "G_SEXT_INREG size must be less than source bit width", MI);
1770 break;
1771 }
1772 case TargetOpcode::G_BSWAP: {
1773 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1774 if (DstTy.getScalarSizeInBits() % 16 != 0)
1775 report(msg: "G_BSWAP size must be a multiple of 16 bits", MI);
1776 break;
1777 }
1778 case TargetOpcode::G_VSCALE: {
1779 if (!MI->getOperand(i: 1).isCImm()) {
1780 report(msg: "G_VSCALE operand must be cimm", MI);
1781 break;
1782 }
1783 if (MI->getOperand(i: 1).getCImm()->isZero()) {
1784 report(msg: "G_VSCALE immediate cannot be zero", MI);
1785 break;
1786 }
1787 break;
1788 }
1789 case TargetOpcode::G_STEP_VECTOR: {
1790 if (!MI->getOperand(i: 1).isCImm()) {
1791 report(msg: "operand must be cimm", MI);
1792 break;
1793 }
1794
1795 if (!MI->getOperand(i: 1).getCImm()->getValue().isStrictlyPositive()) {
1796 report(msg: "step must be > 0", MI);
1797 break;
1798 }
1799
1800 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1801 if (!DstTy.isScalableVector()) {
1802 report(msg: "Destination type must be a scalable vector", MI);
1803 break;
1804 }
1805
1806 // <vscale x 2 x p0>
1807 if (!DstTy.getElementType().isScalar()) {
1808 report(msg: "Destination element type must be scalar", MI);
1809 break;
1810 }
1811
1812 if (MI->getOperand(i: 1).getCImm()->getBitWidth() !=
1813 DstTy.getElementType().getScalarSizeInBits()) {
1814 report(msg: "step bitwidth differs from result type element bitwidth", MI);
1815 break;
1816 }
1817 break;
1818 }
1819 case TargetOpcode::G_INSERT_SUBVECTOR: {
1820 const MachineOperand &Src0Op = MI->getOperand(i: 1);
1821 if (!Src0Op.isReg()) {
1822 report(msg: "G_INSERT_SUBVECTOR first source must be a register", MI);
1823 break;
1824 }
1825
1826 const MachineOperand &Src1Op = MI->getOperand(i: 2);
1827 if (!Src1Op.isReg()) {
1828 report(msg: "G_INSERT_SUBVECTOR second source must be a register", MI);
1829 break;
1830 }
1831
1832 const MachineOperand &IndexOp = MI->getOperand(i: 3);
1833 if (!IndexOp.isImm()) {
1834 report(msg: "G_INSERT_SUBVECTOR index must be an immediate", MI);
1835 break;
1836 }
1837
1838 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1839 LLT Src1Ty = MRI->getType(Reg: Src1Op.getReg());
1840
1841 if (!DstTy.isVector()) {
1842 report(msg: "Destination type must be a vector", MI);
1843 break;
1844 }
1845
1846 if (!Src1Ty.isVector()) {
1847 report(msg: "Second source must be a vector", MI);
1848 break;
1849 }
1850
1851 if (DstTy.getElementType() != Src1Ty.getElementType()) {
1852 report(msg: "Element type of vectors must be the same", MI);
1853 break;
1854 }
1855
1856 if (!DstTy.isScalable() && Src1Ty.isScalable()) {
1857 report(msg: "Cannot insert a scalable vector into a fixed length vector", MI);
1858 break;
1859 }
1860
1861 bool IsMixedFixedIntoScalable =
1862 DstTy.isScalableVector() && Src1Ty.isFixedVector();
1863
1864 if (!IsMixedFixedIntoScalable &&
1865 ElementCount::isKnownGT(LHS: Src1Ty.getElementCount(),
1866 RHS: DstTy.getElementCount())) {
1867 report(msg: "Second source must be smaller than destination vector", MI);
1868 break;
1869 }
1870
1871 uint64_t Idx = IndexOp.getImm();
1872 uint64_t Src1MinLen = Src1Ty.getElementCount().getKnownMinValue();
1873 if (IndexOp.getImm() % Src1MinLen != 0) {
1874 report(msg: "Index must be a multiple of the second source vector's "
1875 "minimum vector length",
1876 MI);
1877 break;
1878 }
1879
1880 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1881 if (Idx >= DstMinLen ||
1882 (!IsMixedFixedIntoScalable && Idx + Src1MinLen > DstMinLen)) {
1883 report(msg: "Subvector type and index must not cause insert to overrun the "
1884 "vector being inserted into",
1885 MI);
1886 break;
1887 }
1888
1889 break;
1890 }
1891 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1892 const MachineOperand &SrcOp = MI->getOperand(i: 1);
1893 if (!SrcOp.isReg()) {
1894 report(msg: "G_EXTRACT_SUBVECTOR first source must be a register", MI);
1895 break;
1896 }
1897
1898 const MachineOperand &IndexOp = MI->getOperand(i: 2);
1899 if (!IndexOp.isImm()) {
1900 report(msg: "G_EXTRACT_SUBVECTOR index must be an immediate", MI);
1901 break;
1902 }
1903
1904 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1905 LLT SrcTy = MRI->getType(Reg: SrcOp.getReg());
1906
1907 if (!DstTy.isVector()) {
1908 report(msg: "Destination type must be a vector", MI);
1909 break;
1910 }
1911
1912 if (!SrcTy.isVector()) {
1913 report(msg: "Source must be a vector", MI);
1914 break;
1915 }
1916
1917 if (DstTy.getElementType() != SrcTy.getElementType()) {
1918 report(msg: "Element type of vectors must be the same", MI);
1919 break;
1920 }
1921
1922 if (DstTy.isScalable() && !SrcTy.isScalable()) {
1923 report(msg: "Cannot extract a scalable vector from a fixed length vector", MI);
1924 break;
1925 }
1926
1927 if (ElementCount::isKnownGT(LHS: DstTy.getElementCount(),
1928 RHS: SrcTy.getElementCount())) {
1929 report(msg: "Destination vector must be smaller than source vector", MI);
1930 break;
1931 }
1932
1933 uint64_t Idx = IndexOp.getImm();
1934 uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();
1935 if (Idx % DstMinLen != 0) {
1936 report(msg: "Index must be a multiple of the destination vector's minimum "
1937 "vector length",
1938 MI);
1939 break;
1940 }
1941
1942 bool IsMixedFixedFromScalable =
1943 DstTy.isFixedVector() && SrcTy.isScalableVector();
1944 uint64_t SrcMinLen = SrcTy.getElementCount().getKnownMinValue();
1945 if (Idx >= SrcMinLen ||
1946 (!IsMixedFixedFromScalable && Idx + DstMinLen > SrcMinLen)) {
1947 report(msg: "Destination type and index must not cause extract to overrun the "
1948 "source vector",
1949 MI);
1950 break;
1951 }
1952
1953 break;
1954 }
1955 case TargetOpcode::G_SHUFFLE_VECTOR: {
1956 const MachineOperand &MaskOp = MI->getOperand(i: 3);
1957 if (!MaskOp.isShuffleMask()) {
1958 report(msg: "Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);
1959 break;
1960 }
1961
1962 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
1963 LLT Src0Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
1964 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
1965
1966 if (Src0Ty != Src1Ty)
1967 report(msg: "Source operands must be the same type", MI);
1968
1969 if (Src0Ty.getScalarType() != DstTy.getScalarType()) {
1970 report(msg: "G_SHUFFLE_VECTOR cannot change element type", MI);
1971 break;
1972 }
1973 if (!Src0Ty.isVector()) {
1974 report(msg: "G_SHUFFLE_VECTOR must have vector src", MI);
1975 break;
1976 }
1977 if (!DstTy.isVector()) {
1978 report(msg: "G_SHUFFLE_VECTOR must have vector dst", MI);
1979 break;
1980 }
1981
1982 // Don't check that all operands are vector because scalars are used in
1983 // place of 1 element vectors.
1984 int SrcNumElts = Src0Ty.getNumElements();
1985 int DstNumElts = DstTy.getNumElements();
1986
1987 ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();
1988
1989 if (static_cast<int>(MaskIdxes.size()) != DstNumElts)
1990 report(msg: "Wrong result type for shufflemask", MI);
1991
1992 for (int Idx : MaskIdxes) {
1993 if (Idx < 0)
1994 continue;
1995
1996 if (Idx >= 2 * SrcNumElts)
1997 report(msg: "Out of bounds shuffle index", MI);
1998 }
1999
2000 break;
2001 }
2002
2003 case TargetOpcode::G_SPLAT_VECTOR: {
2004 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2005 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2006
2007 if (!DstTy.isScalableVector()) {
2008 report(msg: "Destination type must be a scalable vector", MI);
2009 break;
2010 }
2011
2012 if (!SrcTy.isScalar() && !SrcTy.isPointer()) {
2013 report(msg: "Source type must be a scalar or pointer", MI);
2014 break;
2015 }
2016
2017 if (TypeSize::isKnownGT(LHS: DstTy.getElementType().getSizeInBits(),
2018 RHS: SrcTy.getSizeInBits())) {
2019 report(msg: "Element type of the destination must be the same size or smaller "
2020 "than the source type",
2021 MI);
2022 break;
2023 }
2024
2025 break;
2026 }
2027 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2028 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2029 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2030 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2031
2032 if (!DstTy.isScalar() && !DstTy.isPointer()) {
2033 report(msg: "Destination type must be a scalar or pointer", MI);
2034 break;
2035 }
2036
2037 if (!SrcTy.isVector()) {
2038 report(msg: "First source must be a vector", MI);
2039 break;
2040 }
2041
2042 auto TLI = MF->getSubtarget().getTargetLowering();
2043 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2044 report(msg: "Index type must match VectorIdxTy", MI);
2045 break;
2046 }
2047
2048 break;
2049 }
2050 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2051 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2052 LLT VecTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2053 LLT ScaTy = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2054 LLT IdxTy = MRI->getType(Reg: MI->getOperand(i: 3).getReg());
2055
2056 if (!DstTy.isVector()) {
2057 report(msg: "Destination type must be a vector", MI);
2058 break;
2059 }
2060
2061 if (VecTy != DstTy) {
2062 report(msg: "Destination type and vector type must match", MI);
2063 break;
2064 }
2065
2066 if (!ScaTy.isScalar() && !ScaTy.isPointer()) {
2067 report(msg: "Inserted element must be a scalar or pointer", MI);
2068 break;
2069 }
2070
2071 auto TLI = MF->getSubtarget().getTargetLowering();
2072 if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(DL: MF->getDataLayout())) {
2073 report(msg: "Index type must match VectorIdxTy", MI);
2074 break;
2075 }
2076
2077 break;
2078 }
2079 case TargetOpcode::G_DYN_STACKALLOC: {
2080 const MachineOperand &DstOp = MI->getOperand(i: 0);
2081 const MachineOperand &AllocOp = MI->getOperand(i: 1);
2082 const MachineOperand &AlignOp = MI->getOperand(i: 2);
2083
2084 if (!DstOp.isReg() || !MRI->getType(Reg: DstOp.getReg()).isPointer()) {
2085 report(msg: "dst operand 0 must be a pointer type", MI);
2086 break;
2087 }
2088
2089 if (!AllocOp.isReg() || !MRI->getType(Reg: AllocOp.getReg()).isScalar()) {
2090 report(msg: "src operand 1 must be a scalar reg type", MI);
2091 break;
2092 }
2093
2094 if (!AlignOp.isImm()) {
2095 report(msg: "src operand 2 must be an immediate type", MI);
2096 break;
2097 }
2098 break;
2099 }
2100 case TargetOpcode::G_MEMCPY_INLINE:
2101 case TargetOpcode::G_MEMCPY:
2102 case TargetOpcode::G_MEMMOVE: {
2103 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2104 if (MMOs.size() != 2) {
2105 report(msg: "memcpy/memmove must have 2 memory operands", MI);
2106 break;
2107 }
2108
2109 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) ||
2110 (MMOs[1]->isStore() || !MMOs[1]->isLoad())) {
2111 report(msg: "wrong memory operand types", MI);
2112 break;
2113 }
2114
2115 if (MMOs[0]->getSize() != MMOs[1]->getSize())
2116 report(msg: "inconsistent memory operand sizes", MI);
2117
2118 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2119 LLT SrcPtrTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2120
2121 if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) {
2122 report(msg: "memory instruction operand must be a pointer", MI);
2123 break;
2124 }
2125
2126 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2127 report(msg: "inconsistent store address space", MI);
2128 if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())
2129 report(msg: "inconsistent load address space", MI);
2130
2131 if (Opc != TargetOpcode::G_MEMCPY_INLINE)
2132 if (!MI->getOperand(i: 3).isImm() || (MI->getOperand(i: 3).getImm() & ~1LL))
2133 report(msg: "'tail' flag (operand 3) must be an immediate 0 or 1", MI);
2134
2135 break;
2136 }
2137 case TargetOpcode::G_BZERO:
2138 case TargetOpcode::G_MEMSET:
2139 case TargetOpcode::G_MEMSET_INLINE: {
2140 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
2141 std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset"
2142 : Opc == TargetOpcode::G_MEMSET_INLINE ? "memset_inline"
2143 : "bzero";
2144 if (MMOs.size() != 1) {
2145 report(Msg: Twine(Name, " must have 1 memory operand"), MI);
2146 break;
2147 }
2148
2149 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
2150 report(Msg: Twine(Name, " memory operand must be a store"), MI);
2151 break;
2152 }
2153
2154 LLT DstPtrTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2155 if (!DstPtrTy.isPointer()) {
2156 report(Msg: Twine(Name, " operand must be a pointer"), MI);
2157 break;
2158 }
2159
2160 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
2161 report(Msg: "inconsistent " + Twine(Name, " address space"), MI);
2162
2163 if (Opc != TargetOpcode::G_MEMSET_INLINE) {
2164 if (!MI->getOperand(i: MI->getNumOperands() - 1).isImm() ||
2165 (MI->getOperand(i: MI->getNumOperands() - 1).getImm() & ~1LL))
2166 report(msg: "'tail' flag (last operand) must be an immediate 0 or 1", MI);
2167 }
2168
2169 break;
2170 }
2171 case TargetOpcode::G_UBSANTRAP: {
2172 const MachineOperand &KindOp = MI->getOperand(i: 0);
2173 if (!MI->getOperand(i: 0).isImm()) {
2174 report(msg: "Crash kind must be an immediate", MO: &KindOp, MONum: 0);
2175 break;
2176 }
2177 int64_t Kind = MI->getOperand(i: 0).getImm();
2178 if (!isInt<8>(x: Kind))
2179 report(msg: "Crash kind must be 8 bit wide", MO: &KindOp, MONum: 0);
2180 break;
2181 }
2182 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
2183 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
2184 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2185 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2186 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2187 if (!DstTy.isScalar())
2188 report(msg: "Vector reduction requires a scalar destination type", MI);
2189 if (!Src1Ty.isScalar())
2190 report(msg: "Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI);
2191 if (!Src2Ty.isVector())
2192 report(msg: "Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);
2193 break;
2194 }
2195 case TargetOpcode::G_VECREDUCE_FADD:
2196 case TargetOpcode::G_VECREDUCE_FMUL:
2197 case TargetOpcode::G_VECREDUCE_FMAX:
2198 case TargetOpcode::G_VECREDUCE_FMIN:
2199 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
2200 case TargetOpcode::G_VECREDUCE_FMINIMUM:
2201 case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
2202 case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
2203 case TargetOpcode::G_VECREDUCE_ADD:
2204 case TargetOpcode::G_VECREDUCE_MUL:
2205 case TargetOpcode::G_VECREDUCE_AND:
2206 case TargetOpcode::G_VECREDUCE_OR:
2207 case TargetOpcode::G_VECREDUCE_XOR:
2208 case TargetOpcode::G_VECREDUCE_SMAX:
2209 case TargetOpcode::G_VECREDUCE_SMIN:
2210 case TargetOpcode::G_VECREDUCE_UMAX:
2211 case TargetOpcode::G_VECREDUCE_UMIN: {
2212 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2213 if (!DstTy.isScalar())
2214 report(msg: "Vector reduction requires a scalar destination type", MI);
2215 break;
2216 }
2217
2218 case TargetOpcode::G_SBFX:
2219 case TargetOpcode::G_UBFX: {
2220 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2221 if (DstTy.isVector()) {
2222 report(msg: "Bitfield extraction is not supported on vectors", MI);
2223 break;
2224 }
2225 break;
2226 }
2227 case TargetOpcode::G_SHL:
2228 case TargetOpcode::G_LSHR:
2229 case TargetOpcode::G_ASHR:
2230 case TargetOpcode::G_ROTR:
2231 case TargetOpcode::G_ROTL: {
2232 LLT Src1Ty = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2233 LLT Src2Ty = MRI->getType(Reg: MI->getOperand(i: 2).getReg());
2234 if (Src1Ty.isVector() != Src2Ty.isVector()) {
2235 report(msg: "Shifts and rotates require operands to be either all scalars or "
2236 "all vectors",
2237 MI);
2238 break;
2239 }
2240 break;
2241 }
2242 case TargetOpcode::G_LLROUND:
2243 case TargetOpcode::G_LROUND: {
2244 LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2245 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2246 if (!DstTy.isValid() || !SrcTy.isValid())
2247 break;
2248 if (SrcTy.isPointer() || DstTy.isPointer()) {
2249 StringRef Op = SrcTy.isPointer() ? "Source" : "Destination";
2250 report(Msg: Twine(Op, " operand must not be a pointer type"), MI);
2251 } else if (SrcTy.isScalar()) {
2252 verifyAllRegOpsScalar(MI: *MI, MRI: *MRI);
2253 break;
2254 } else if (SrcTy.isVector()) {
2255 verifyVectorElementMatch(Ty0: SrcTy, Ty1: DstTy, MI);
2256 break;
2257 }
2258 break;
2259 }
2260 case TargetOpcode::G_IS_FPCLASS: {
2261 LLT DestTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2262 LLT DestEltTy = DestTy.getScalarType();
2263 if (!DestEltTy.isScalar()) {
2264 report(msg: "Destination must be a scalar or vector of scalars", MI);
2265 break;
2266 }
2267 LLT SrcTy = MRI->getType(Reg: MI->getOperand(i: 1).getReg());
2268 LLT SrcEltTy = SrcTy.getScalarType();
2269 if (!SrcEltTy.isScalar()) {
2270 report(msg: "Source must be a scalar or vector of scalars", MI);
2271 break;
2272 }
2273 if (!verifyVectorElementMatch(Ty0: DestTy, Ty1: SrcTy, MI))
2274 break;
2275 const MachineOperand &TestMO = MI->getOperand(i: 2);
2276 if (!TestMO.isImm()) {
2277 report(msg: "floating-point class set (operand 2) must be an immediate", MI);
2278 break;
2279 }
2280 int64_t Test = TestMO.getImm();
2281 if (Test < 0 || Test > fcAllFlags) {
2282 report(msg: "Incorrect floating-point class set (operand 2)", MI);
2283 break;
2284 }
2285 break;
2286 }
2287 case TargetOpcode::G_PREFETCH: {
2288 const MachineOperand &AddrOp = MI->getOperand(i: 0);
2289 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer()) {
2290 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 0);
2291 break;
2292 }
2293 const MachineOperand &RWOp = MI->getOperand(i: 1);
2294 if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {
2295 report(msg: "rw operand must be an immediate 0-1", MO: &RWOp, MONum: 1);
2296 break;
2297 }
2298 const MachineOperand &LocalityOp = MI->getOperand(i: 2);
2299 if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {
2300 report(msg: "locality operand must be an immediate 0-3", MO: &LocalityOp, MONum: 2);
2301 break;
2302 }
2303 const MachineOperand &CacheTypeOp = MI->getOperand(i: 3);
2304 if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {
2305 report(msg: "cache type operand must be an immediate 0-1", MO: &CacheTypeOp, MONum: 3);
2306 break;
2307 }
2308 break;
2309 }
2310 case TargetOpcode::G_ASSERT_ALIGN: {
2311 if (MI->getOperand(i: 2).getImm() < 1)
2312 report(msg: "alignment immediate must be >= 1", MI);
2313 break;
2314 }
2315 case TargetOpcode::G_CONSTANT_POOL: {
2316 if (!MI->getOperand(i: 1).isCPI())
2317 report(msg: "Src operand 1 must be a constant pool index", MI);
2318 if (!MRI->getType(Reg: MI->getOperand(i: 0).getReg()).isPointer())
2319 report(msg: "Dst operand 0 must be a pointer", MI);
2320 break;
2321 }
2322 case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {
2323 const MachineOperand &AddrOp = MI->getOperand(i: 1);
2324 if (!AddrOp.isReg() || !MRI->getType(Reg: AddrOp.getReg()).isPointer())
2325 report(msg: "addr operand must be a pointer", MO: &AddrOp, MONum: 1);
2326 break;
2327 }
2328 case TargetOpcode::G_SMIN:
2329 case TargetOpcode::G_SMAX:
2330 case TargetOpcode::G_UMIN:
2331 case TargetOpcode::G_UMAX: {
2332 const LLT DstTy = MRI->getType(Reg: MI->getOperand(i: 0).getReg());
2333 if (DstTy.isPointerOrPointerVector())
2334 report(msg: "Generic smin/smax/umin/umax does not support pointer operands",
2335 MI);
2336 break;
2337 }
2338 default:
2339 break;
2340 }
2341}
2342
2343void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
2344 const MCInstrDesc &MCID = MI->getDesc();
2345 if (MI->getNumOperands() < MCID.getNumOperands()) {
2346 report(msg: "Too few operands", MI);
2347 OS << MCID.getNumOperands() << " operands expected, but "
2348 << MI->getNumOperands() << " given.\n";
2349 }
2350
2351 if (MI->getFlag(Flag: MachineInstr::NoConvergent) && !MCID.isConvergent())
2352 report(msg: "NoConvergent flag expected only on convergent instructions.", MI);
2353
2354 if (MI->isPHI()) {
2355 if (MF->getProperties().hasNoPHIs())
2356 report(msg: "Found PHI instruction with NoPHIs property set", MI);
2357
2358 if (FirstNonPHI)
2359 report(msg: "Found PHI instruction after non-PHI", MI);
2360 } else if (FirstNonPHI == nullptr)
2361 FirstNonPHI = MI;
2362
2363 // Check the tied operands.
2364 if (MI->isInlineAsm())
2365 verifyInlineAsm(MI);
2366
2367 // Check that unspillable terminators define a reg and have at most one use.
2368 if (TII->isUnspillableTerminator(MI)) {
2369 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
2370 report(msg: "Unspillable Terminator does not define a reg", MI);
2371 Register Def = MI->getOperand(i: 0).getReg();
2372 if (Def.isVirtual() && hasPHIs(MF: *MF) &&
2373 std::distance(first: MRI->use_nodbg_begin(RegNo: Def), last: MRI->use_nodbg_end()) > 1)
2374 report(msg: "Unspillable Terminator expected to have at most one use!", MI);
2375 }
2376
2377 // A fully-formed DBG_VALUE must have a location. Ignore partially formed
2378 // DBG_VALUEs: these are convenient to use in tests, but should never get
2379 // generated.
2380 if (MI->isDebugValue() && MI->getNumOperands() == 4)
2381 if (!MI->getDebugLoc())
2382 report(msg: "Missing DebugLoc for debug instruction", MI);
2383
2384 // Meta instructions should never be the subject of debug value tracking,
2385 // they don't create a value in the output program at all.
2386 if (MI->isMetaInstruction() && MI->peekDebugInstrNum())
2387 report(msg: "Metadata instruction should not have a value tracking number", MI);
2388
2389 // Check the MachineMemOperands for basic consistency.
2390 for (MachineMemOperand *Op : MI->memoperands()) {
2391 if (Op->isLoad() && !MI->mayLoad())
2392 report(msg: "Missing mayLoad flag", MI);
2393 if (Op->isStore() && !MI->mayStore())
2394 report(msg: "Missing mayStore flag", MI);
2395 }
2396
2397 // Debug values must not have a slot index.
2398 // Other instructions must have one, unless they are inside a bundle.
2399 if (LiveInts) {
2400 bool mapped = !LiveInts->isNotInMIMap(Instr: *MI);
2401 if (MI->isDebugOrPseudoInstr()) {
2402 if (mapped)
2403 report(msg: "Debug instruction has a slot index", MI);
2404 } else if (MI->isInsideBundle()) {
2405 if (mapped)
2406 report(msg: "Instruction inside bundle has a slot index", MI);
2407 } else {
2408 if (!mapped)
2409 report(msg: "Missing slot index", MI);
2410 }
2411 }
2412
2413 unsigned Opc = MCID.getOpcode();
2414 if (isPreISelGenericOpcode(Opcode: Opc) || isPreISelGenericOptimizationHint(Opcode: Opc)) {
2415 verifyPreISelGenericInstruction(MI);
2416 return;
2417 }
2418
2419 StringRef ErrorInfo;
2420 if (!TII->verifyInstruction(MI: *MI, ErrInfo&: ErrorInfo))
2421 report(msg: ErrorInfo.data(), MI);
2422
2423 // Verify properties of various specific instruction types
2424 switch (MI->getOpcode()) {
2425 case TargetOpcode::COPY: {
2426 const MachineOperand &DstOp = MI->getOperand(i: 0);
2427 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2428 const Register SrcReg = SrcOp.getReg();
2429 const Register DstReg = DstOp.getReg();
2430
2431 LLT DstTy = MRI->getType(Reg: DstReg);
2432 LLT SrcTy = MRI->getType(Reg: SrcReg);
2433 if (SrcTy.isValid() && DstTy.isValid()) {
2434 // If both types are valid, check that the types are the same.
2435 if (SrcTy != DstTy) {
2436 report(msg: "Copy Instruction is illegal with mismatching types", MI);
2437 OS << "Def = " << DstTy << ", Src = " << SrcTy << '\n';
2438 }
2439
2440 break;
2441 }
2442
2443 if (!SrcTy.isValid() && !DstTy.isValid())
2444 break;
2445
2446 // If we have only one valid type, this is likely a copy between a virtual
2447 // and physical register.
2448 TypeSize SrcSize = TypeSize::getZero();
2449 TypeSize DstSize = TypeSize::getZero();
2450 if (SrcReg.isPhysical() && DstTy.isValid()) {
2451 if (!hasPhysRegClassForType(TRI: *TRI, Reg: SrcReg, Ty: DstTy))
2452 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2453 } else {
2454 SrcSize = TRI->getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
2455 }
2456
2457 if (DstReg.isPhysical() && SrcTy.isValid()) {
2458 if (!hasPhysRegClassForType(TRI: *TRI, Reg: DstReg, Ty: SrcTy))
2459 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2460 } else {
2461 DstSize = TRI->getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2462 }
2463
2464 // The next two checks allow COPY between physical and virtual registers,
2465 // when the virtual register has a scalable size and the physical register
2466 // has a fixed size. These checks allow COPY between *potentially*
2467 // mismatched sizes. However, once RegisterBankSelection occurs,
2468 // MachineVerifier should be able to resolve a fixed size for the scalable
2469 // vector, and at that point this function will know for sure whether the
2470 // sizes are mismatched and correctly report a size mismatch.
2471 if (SrcReg.isPhysical() && DstReg.isVirtual() && DstSize.isScalable() &&
2472 !SrcSize.isScalable())
2473 break;
2474 if (SrcReg.isVirtual() && DstReg.isPhysical() && SrcSize.isScalable() &&
2475 !DstSize.isScalable())
2476 break;
2477
2478 if (SrcSize.isNonZero() && DstSize.isNonZero() && SrcSize != DstSize) {
2479 if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {
2480 report(msg: "Copy Instruction is illegal with mismatching sizes", MI);
2481 OS << "Def Size = " << DstSize << ", Src Size = " << SrcSize << '\n';
2482 }
2483 }
2484 break;
2485 }
2486 case TargetOpcode::COPY_LANEMASK: {
2487 const MachineOperand &DstOp = MI->getOperand(i: 0);
2488 const MachineOperand &SrcOp = MI->getOperand(i: 1);
2489 const MachineOperand &LaneMaskOp = MI->getOperand(i: 2);
2490 const Register SrcReg = SrcOp.getReg();
2491 const LaneBitmask LaneMask = LaneMaskOp.getLaneMask();
2492 LaneBitmask SrcMaxLaneMask = LaneBitmask::getAll();
2493
2494 if (DstOp.getSubReg())
2495 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &DstOp, MONum: 0);
2496
2497 if (SrcOp.getSubReg())
2498 report(msg: "COPY_LANEMASK must not use a subregister index", MO: &SrcOp, MONum: 1);
2499
2500 if (LaneMask.none())
2501 report(msg: "COPY_LANEMASK must read at least one lane", MI);
2502
2503 if (SrcReg.isPhysical()) {
2504 const TargetRegisterClass *SrcRC = TRI->getMinimalPhysRegClass(Reg: SrcReg);
2505 if (SrcRC)
2506 SrcMaxLaneMask = SrcRC->getLaneMask();
2507 } else {
2508 SrcMaxLaneMask = MRI->getMaxLaneMaskForVReg(Reg: SrcReg);
2509 }
2510
2511 // COPY_LANEMASK should be used only for partial copy. For full
2512 // copy, one should strictly use the COPY instruction.
2513 if (SrcMaxLaneMask == LaneMask)
2514 report(msg: "COPY_LANEMASK cannot be used to do full copy", MI);
2515
2516 // If LaneMask is greater than the SrcMaxLaneMask, it implies
2517 // COPY_LANEMASK is attempting to read from the lanes that
2518 // don't exists in the source register.
2519 if (SrcMaxLaneMask < LaneMask)
2520 report(msg: "COPY_LANEMASK attempts to read from the lanes that "
2521 "don't exist in the source register",
2522 MI);
2523
2524 break;
2525 }
2526 case TargetOpcode::STATEPOINT: {
2527 StatepointOpers SO(MI);
2528 if (!MI->getOperand(i: SO.getIDPos()).isImm() ||
2529 !MI->getOperand(i: SO.getNBytesPos()).isImm() ||
2530 !MI->getOperand(i: SO.getNCallArgsPos()).isImm()) {
2531 report(msg: "meta operands to STATEPOINT not constant!", MI);
2532 break;
2533 }
2534
2535 auto VerifyStackMapConstant = [&](unsigned Offset) {
2536 if (Offset >= MI->getNumOperands()) {
2537 report(msg: "stack map constant to STATEPOINT is out of range!", MI);
2538 return;
2539 }
2540 if (!MI->getOperand(i: Offset - 1).isImm() ||
2541 MI->getOperand(i: Offset - 1).getImm() != StackMaps::ConstantOp ||
2542 !MI->getOperand(i: Offset).isImm())
2543 report(msg: "stack map constant to STATEPOINT not well formed!", MI);
2544 };
2545 VerifyStackMapConstant(SO.getCCIdx());
2546 VerifyStackMapConstant(SO.getFlagsIdx());
2547 VerifyStackMapConstant(SO.getNumDeoptArgsIdx());
2548 VerifyStackMapConstant(SO.getNumGCPtrIdx());
2549 VerifyStackMapConstant(SO.getNumAllocaIdx());
2550 VerifyStackMapConstant(SO.getNumGcMapEntriesIdx());
2551
2552 // Verify that all explicit statepoint defs are tied to gc operands as
2553 // they are expected to be a relocation of gc operands.
2554 unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx();
2555 unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2;
2556 for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) {
2557 unsigned UseOpIdx;
2558 if (!MI->isRegTiedToUseOperand(DefOpIdx: Idx, UseOpIdx: &UseOpIdx)) {
2559 report(msg: "STATEPOINT defs expected to be tied", MI);
2560 break;
2561 }
2562 if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) {
2563 report(msg: "STATEPOINT def tied to non-gc operand", MI);
2564 break;
2565 }
2566 }
2567
2568 // TODO: verify we have properly encoded deopt arguments
2569 } break;
2570 case TargetOpcode::INSERT_SUBREG: {
2571 unsigned InsertedSize;
2572 if (unsigned SubIdx = MI->getOperand(i: 2).getSubReg())
2573 InsertedSize = TRI->getSubRegIdxSize(Idx: SubIdx);
2574 else
2575 InsertedSize = TRI->getRegSizeInBits(Reg: MI->getOperand(i: 2).getReg(), MRI: *MRI);
2576 unsigned SubRegSize = TRI->getSubRegIdxSize(Idx: MI->getOperand(i: 3).getImm());
2577 if (SubRegSize < InsertedSize) {
2578 report(msg: "INSERT_SUBREG expected inserted value to have equal or lesser "
2579 "size than the subreg it was inserted into", MI);
2580 break;
2581 }
2582 } break;
2583 case TargetOpcode::REG_SEQUENCE: {
2584 unsigned NumOps = MI->getNumOperands();
2585 if (!(NumOps & 1)) {
2586 report(msg: "Invalid number of operands for REG_SEQUENCE", MI);
2587 break;
2588 }
2589
2590 for (unsigned I = 1; I != NumOps; I += 2) {
2591 const MachineOperand &RegOp = MI->getOperand(i: I);
2592 const MachineOperand &SubRegOp = MI->getOperand(i: I + 1);
2593
2594 if (!RegOp.isReg())
2595 report(msg: "Invalid register operand for REG_SEQUENCE", MO: &RegOp, MONum: I);
2596
2597 if (!SubRegOp.isImm() || SubRegOp.getImm() == 0 ||
2598 SubRegOp.getImm() >= TRI->getNumSubRegIndices()) {
2599 report(msg: "Invalid subregister index operand for REG_SEQUENCE",
2600 MO: &SubRegOp, MONum: I + 1);
2601 }
2602 }
2603
2604 Register DstReg = MI->getOperand(i: 0).getReg();
2605 if (DstReg.isPhysical())
2606 report(msg: "REG_SEQUENCE does not support physical register results", MI);
2607
2608 if (MI->getOperand(i: 0).getSubReg())
2609 report(msg: "Invalid subreg result for REG_SEQUENCE", MI);
2610
2611 break;
2612 }
2613 }
2614}
2615
2616void
2617MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
2618 const MachineInstr *MI = MO->getParent();
2619 const MCInstrDesc &MCID = MI->getDesc();
2620 unsigned NumDefs = MCID.getNumDefs();
2621 if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)
2622 NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;
2623
2624 // The first MCID.NumDefs operands must be explicit register defines
2625 if (MONum < NumDefs) {
2626 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2627 if (!MO->isReg())
2628 report(msg: "Explicit definition must be a register", MO, MONum);
2629 else if (!MO->isDef() && !MCOI.isOptionalDef())
2630 report(msg: "Explicit definition marked as use", MO, MONum);
2631 else if (MO->isImplicit())
2632 report(msg: "Explicit definition marked as implicit", MO, MONum);
2633 } else if (MONum < MCID.getNumOperands()) {
2634 const MCOperandInfo &MCOI = MCID.operands()[MONum];
2635 // Don't check if it's the last operand in a variadic instruction. See,
2636 // e.g., LDM_RET in the arm back end. Check non-variadic operands only.
2637 bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1;
2638 if (!IsOptional) {
2639 if (MO->isReg()) {
2640 if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs())
2641 report(msg: "Explicit operand marked as def", MO, MONum);
2642 if (MO->isImplicit())
2643 report(msg: "Explicit operand marked as implicit", MO, MONum);
2644 }
2645
2646 // Check that an instruction has register operands only as expected.
2647 if (MCOI.OperandType == MCOI::OPERAND_REGISTER &&
2648 !MO->isReg() && !MO->isFI())
2649 report(msg: "Expected a register operand.", MO, MONum);
2650 if (MO->isReg()) {
2651 if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE ||
2652 (MCOI.OperandType == MCOI::OPERAND_PCREL &&
2653 !TII->isPCRelRegisterOperandLegal(MO: *MO)))
2654 report(msg: "Expected a non-register operand.", MO, MONum);
2655 }
2656 }
2657
2658 int TiedTo = MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::TIED_TO);
2659 if (TiedTo != -1) {
2660 if (!MO->isReg())
2661 report(msg: "Tied use must be a register", MO, MONum);
2662 else if (!MO->isTied())
2663 report(msg: "Operand should be tied", MO, MONum);
2664 else if (unsigned(TiedTo) != MI->findTiedOperandIdx(OpIdx: MONum))
2665 report(msg: "Tied def doesn't match MCInstrDesc", MO, MONum);
2666 else if (MO->getReg().isPhysical()) {
2667 const MachineOperand &MOTied = MI->getOperand(i: TiedTo);
2668 if (!MOTied.isReg())
2669 report(msg: "Tied counterpart must be a register", MO: &MOTied, MONum: TiedTo);
2670 else if (MOTied.getReg().isPhysical() &&
2671 MO->getReg() != MOTied.getReg())
2672 report(msg: "Tied physical registers must match.", MO: &MOTied, MONum: TiedTo);
2673 }
2674 } else if (MO->isReg() && MO->isTied())
2675 report(msg: "Explicit operand should not be tied", MO, MONum);
2676 } else if (!MI->isVariadic()) {
2677 // ARM adds %reg0 operands to indicate predicates. We'll allow that.
2678 if (!MO->isValidExcessOperand())
2679 report(msg: "Extra explicit operand on non-variadic instruction", MO, MONum);
2680 }
2681
2682 // Verify earlyClobber def operand
2683 if (MCID.getOperandConstraint(OpNum: MONum, Constraint: MCOI::EARLY_CLOBBER) != -1) {
2684 if (!MO->isReg())
2685 report(msg: "Early clobber must be a register", MI);
2686 if (!MO->isEarlyClobber())
2687 report(msg: "Missing earlyClobber flag", MI);
2688 }
2689
2690 switch (MO->getType()) {
2691 case MachineOperand::MO_Register: {
2692 // Verify debug flag on debug instructions. Check this first because reg0
2693 // indicates an undefined debug value.
2694 if (MI->isDebugInstr() && MO->isUse()) {
2695 if (!MO->isDebug())
2696 report(msg: "Register operand must be marked debug", MO, MONum);
2697 } else if (MO->isDebug()) {
2698 report(msg: "Register operand must not be marked debug", MO, MONum);
2699 }
2700
2701 const Register Reg = MO->getReg();
2702 if (!Reg)
2703 return;
2704 if (MRI->tracksLiveness() && !MI->isDebugInstr())
2705 checkLiveness(MO, MONum);
2706
2707 if (MO->isDef() && MO->isUndef() && !MO->getSubReg() &&
2708 MO->getReg().isVirtual()) // TODO: Apply to physregs too
2709 report(msg: "Undef virtual register def operands require a subregister", MO, MONum);
2710
2711 // Verify the consistency of tied operands.
2712 if (MO->isTied()) {
2713 unsigned OtherIdx = MI->findTiedOperandIdx(OpIdx: MONum);
2714 const MachineOperand &OtherMO = MI->getOperand(i: OtherIdx);
2715 if (!OtherMO.isReg())
2716 report(msg: "Must be tied to a register", MO, MONum);
2717 if (!OtherMO.isTied())
2718 report(msg: "Missing tie flags on tied operand", MO, MONum);
2719 if (MI->findTiedOperandIdx(OpIdx: OtherIdx) != MONum)
2720 report(msg: "Inconsistent tie links", MO, MONum);
2721 if (MONum < MCID.getNumDefs()) {
2722 if (OtherIdx < MCID.getNumOperands()) {
2723 if (-1 == MCID.getOperandConstraint(OpNum: OtherIdx, Constraint: MCOI::TIED_TO))
2724 report(msg: "Explicit def tied to explicit use without tie constraint",
2725 MO, MONum);
2726 } else {
2727 if (!OtherMO.isImplicit())
2728 report(msg: "Explicit def should be tied to implicit use", MO, MONum);
2729 }
2730 }
2731 }
2732
2733 // Verify two-address constraints after the twoaddressinstruction pass.
2734 // Both twoaddressinstruction pass and phi-node-elimination pass call
2735 // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification
2736 // after twoaddressinstruction pass not after phi-node-elimination pass. So
2737 // we shouldn't use the IsSSA as the condition, we should based on
2738 // TiedOpsRewritten property to verify two-address constraints, this
2739 // property will be set in twoaddressinstruction pass.
2740 unsigned DefIdx;
2741 if (MF->getProperties().hasTiedOpsRewritten() && MO->isUse() &&
2742 MI->isRegTiedToDefOperand(UseOpIdx: MONum, DefOpIdx: &DefIdx) &&
2743 Reg != MI->getOperand(i: DefIdx).getReg())
2744 report(msg: "Two-address instruction operands must be identical", MO, MONum);
2745
2746 // Check register classes.
2747 unsigned SubIdx = MO->getSubReg();
2748
2749 if (Reg.isPhysical()) {
2750 if (SubIdx) {
2751 report(msg: "Illegal subregister index for physical register", MO, MONum);
2752 return;
2753 }
2754 if (MONum < MCID.getNumOperands()) {
2755 if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum)) {
2756 if (!DRC->contains(Reg)) {
2757 report(msg: "Illegal physical register for instruction", MO, MONum);
2758 OS << printReg(Reg, TRI) << " is not a "
2759 << TRI->getRegClassName(Class: DRC) << " register.\n";
2760 }
2761 }
2762 }
2763 if (MO->isRenamable()) {
2764 if (MRI->isReserved(PhysReg: Reg)) {
2765 report(msg: "isRenamable set on reserved register", MO, MONum);
2766 return;
2767 }
2768 }
2769 } else {
2770 // Virtual register.
2771 const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);
2772 if (!RC) {
2773 // This is a generic virtual register.
2774
2775 // Do not allow undef uses for generic virtual registers. This ensures
2776 // getVRegDef can never fail and return null on a generic register.
2777 //
2778 // FIXME: This restriction should probably be broadened to all SSA
2779 // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still
2780 // run on the SSA function just before phi elimination.
2781 if (MO->isUndef())
2782 report(msg: "Generic virtual register use cannot be undef", MO, MONum);
2783
2784 // Debug value instruction is permitted to use undefined vregs.
2785 // This is a performance measure to skip the overhead of immediately
2786 // pruning unused debug operands. The final undef substitution occurs
2787 // when debug values are allocated in LDVImpl::handleDebugValue, so
2788 // these verifications always apply after this pass.
2789 if (isFunctionTracksDebugUserValues || !MO->isUse() ||
2790 !MI->isDebugValue() || !MRI->def_empty(RegNo: Reg)) {
2791 // If we're post-Select, we can't have gvregs anymore.
2792 if (isFunctionSelected) {
2793 report(msg: "Generic virtual register invalid in a Selected function",
2794 MO, MONum);
2795 return;
2796 }
2797
2798 // The gvreg must have a type and it must not have a SubIdx.
2799 LLT Ty = MRI->getType(Reg);
2800 if (!Ty.isValid()) {
2801 report(msg: "Generic virtual register must have a valid type", MO,
2802 MONum);
2803 return;
2804 }
2805
2806 const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);
2807 const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo();
2808
2809 // If we're post-RegBankSelect, the gvreg must have a bank.
2810 if (!RegBank && isFunctionRegBankSelected) {
2811 report(msg: "Generic virtual register must have a bank in a "
2812 "RegBankSelected function",
2813 MO, MONum);
2814 return;
2815 }
2816
2817 // Make sure the register fits into its register bank if any.
2818 if (RegBank && Ty.isValid() && !Ty.isScalableVector() &&
2819 RBI->getMaximumSize(RegBankID: RegBank->getID()) < Ty.getSizeInBits()) {
2820 report(msg: "Register bank is too small for virtual register", MO,
2821 MONum);
2822 OS << "Register bank " << RegBank->getName() << " too small("
2823 << RBI->getMaximumSize(RegBankID: RegBank->getID()) << ") to fit "
2824 << Ty.getSizeInBits() << "-bits\n";
2825 return;
2826 }
2827 }
2828
2829 if (SubIdx) {
2830 report(msg: "Generic virtual register does not allow subregister index", MO,
2831 MONum);
2832 return;
2833 }
2834
2835 // If this is a target specific instruction and this operand
2836 // has register class constraint, the virtual register must
2837 // comply to it.
2838 if (!isPreISelGenericOpcode(Opcode: MCID.getOpcode()) &&
2839 MONum < MCID.getNumOperands() && TII->getRegClass(MCID, OpNum: MONum)) {
2840 report(msg: "Virtual register does not match instruction constraint", MO,
2841 MONum);
2842 OS << "Expect register class "
2843 << TRI->getRegClassName(Class: TII->getRegClass(MCID, OpNum: MONum))
2844 << " but got nothing\n";
2845 return;
2846 }
2847
2848 break;
2849 }
2850 // Validate that SubIdx can be applied to the virtual register.
2851 if (!TRI->isSubRegValidForRegClass(RC, Idx: SubIdx)) {
2852 report(msg: "Invalid subregister index for virtual register", MO, MONum);
2853 OS << "Register class " << TRI->getRegClassName(Class: RC)
2854 << " does not support subreg index "
2855 << TRI->getSubRegIndexName(SubIdx) << '\n';
2856 return;
2857 }
2858 if (MONum >= MCID.getNumOperands())
2859 break;
2860 const TargetRegisterClass *DRC = TII->getRegClass(MCID, OpNum: MONum);
2861 if (!DRC)
2862 break;
2863
2864 // If SubIdx is used, verify that RC with SubIdx can be used for an
2865 // operand of class DRC. This is valid if for every register in RC, the
2866 // register obtained by applying SubIdx to it is in DRC.
2867 if (SubIdx && TRI->getMatchingSuperRegClass(A: RC, B: DRC, Idx: SubIdx) != RC) {
2868 report(msg: "Illegal virtual register for instruction", MO, MONum);
2869 OS << TRI->getRegClassName(Class: RC) << "." << TRI->getSubRegIndexName(SubIdx)
2870 << " cannot be used for " << TRI->getRegClassName(Class: DRC)
2871 << " operands.";
2872 }
2873
2874 // If no SubIdx is used, verify that RC is a sub-class of DRC.
2875 if (!SubIdx && !RC->hasSuperClassEq(RC: DRC)) {
2876 report(msg: "Illegal virtual register for instruction", MO, MONum);
2877 OS << "Expected a " << TRI->getRegClassName(Class: DRC)
2878 << " register, but got a " << TRI->getRegClassName(Class: RC)
2879 << " register\n";
2880 }
2881 }
2882 break;
2883 }
2884
2885 case MachineOperand::MO_RegisterMask:
2886 regMasks.push_back(Elt: MO->getRegMask());
2887 break;
2888
2889 case MachineOperand::MO_MachineBasicBlock:
2890 if (MI->isPHI() && !MO->getMBB()->isSuccessor(MBB: MI->getParent()))
2891 report(msg: "PHI operand is not in the CFG", MO, MONum);
2892 break;
2893
2894 case MachineOperand::MO_FrameIndex:
2895 if (LiveStks && LiveStks->hasInterval(Slot: MO->getIndex()) &&
2896 LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
2897 int FI = MO->getIndex();
2898 LiveInterval &LI = LiveStks->getInterval(Slot: FI);
2899 SlotIndex Idx = LiveInts->getInstructionIndex(Instr: *MI);
2900
2901 bool MayStore = MI->mayStore();
2902 bool MayLoad = MI->mayLoad();
2903 // For a memory-to-memory move, we need to check if the frame
2904 // index is used for storing or loading, by inspecting the
2905 // memory operands.
2906 if (MayStore && MayLoad) {
2907 for (const MachineMemOperand *MMO : MI->memoperands()) {
2908 const auto *Value = dyn_cast_if_present<FixedStackPseudoSourceValue>(
2909 Val: MMO->getPseudoValue());
2910 if (!Value || Value->getFrameIndex() != FI)
2911 continue;
2912
2913 if (MMO->isStore())
2914 MayLoad = false;
2915 else
2916 MayStore = false;
2917 break;
2918 }
2919 if (MayLoad == MayStore)
2920 report(msg: "Missing fixed stack memoperand.", MI);
2921 }
2922 if (MayLoad && !LI.liveAt(index: Idx.getRegSlot(EC: true))) {
2923 report(msg: "Instruction loads from dead spill slot", MO, MONum);
2924 OS << "Live stack: " << LI << '\n';
2925 }
2926 if (MayStore && !LI.liveAt(index: Idx.getRegSlot())) {
2927 report(msg: "Instruction stores to dead spill slot", MO, MONum);
2928 OS << "Live stack: " << LI << '\n';
2929 }
2930 }
2931 break;
2932
2933 case MachineOperand::MO_CFIIndex:
2934 if (MO->getCFIIndex() >= MF->getFrameInstructions().size())
2935 report(msg: "CFI instruction has invalid index", MO, MONum);
2936 break;
2937
2938 default:
2939 break;
2940 }
2941}
2942
2943void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,
2944 unsigned MONum, SlotIndex UseIdx,
2945 const LiveRange &LR,
2946 VirtRegOrUnit VRegOrUnit,
2947 LaneBitmask LaneMask) {
2948 const MachineInstr *MI = MO->getParent();
2949
2950 if (!LR.verify()) {
2951 report(msg: "invalid live range", MO, MONum);
2952 report_context_liverange(LR);
2953 report_context_vreg_regunit(VRegOrUnit);
2954 report_context(Pos: UseIdx);
2955 return;
2956 }
2957
2958 LiveQueryResult LRQ = LR.Query(Idx: UseIdx);
2959 bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut());
2960 // Check if we have a segment at the use, note however that we only need one
2961 // live subregister range, the others may be dead.
2962 if (!HasValue && LaneMask.none()) {
2963 report(msg: "No live segment at use", MO, MONum);
2964 report_context_liverange(LR);
2965 report_context_vreg_regunit(VRegOrUnit);
2966 report_context(Pos: UseIdx);
2967 }
2968 if (MO->isKill() && !LRQ.isKill()) {
2969 report(msg: "Live range continues after kill flag", MO, MONum);
2970 report_context_liverange(LR);
2971 report_context_vreg_regunit(VRegOrUnit);
2972 if (LaneMask.any())
2973 report_context_lanemask(LaneMask);
2974 report_context(Pos: UseIdx);
2975 }
2976}
2977
2978void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,
2979 unsigned MONum, SlotIndex DefIdx,
2980 const LiveRange &LR,
2981 VirtRegOrUnit VRegOrUnit,
2982 bool SubRangeCheck,
2983 LaneBitmask LaneMask) {
2984 if (!LR.verify()) {
2985 report(msg: "invalid live range", MO, MONum);
2986 report_context_liverange(LR);
2987 report_context_vreg_regunit(VRegOrUnit);
2988 if (LaneMask.any())
2989 report_context_lanemask(LaneMask);
2990 report_context(Pos: DefIdx);
2991 }
2992
2993 if (const VNInfo *VNI = LR.getVNInfoAt(Idx: DefIdx)) {
2994 // The LR can correspond to the whole reg and its def slot is not obliged
2995 // to be the same as the MO' def slot. E.g. when we check here "normal"
2996 // subreg MO but there is other EC subreg MO in the same instruction so the
2997 // whole reg has EC def slot and differs from the currently checked MO' def
2998 // slot. For example:
2999 // %0 [16e,32r:0) 0@16e L..3 [16e,32r:0) 0@16e L..C [16r,32r:0) 0@16r
3000 // Check that there is an early-clobber def of the same superregister
3001 // somewhere is performed in visitMachineFunctionAfter()
3002 if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) ||
3003 !SlotIndex::isSameInstr(A: VNI->def, B: DefIdx) ||
3004 (VNI->def != DefIdx &&
3005 (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) {
3006 report(msg: "Inconsistent valno->def", MO, MONum);
3007 report_context_liverange(LR);
3008 report_context_vreg_regunit(VRegOrUnit);
3009 if (LaneMask.any())
3010 report_context_lanemask(LaneMask);
3011 report_context(VNI: *VNI);
3012 report_context(Pos: DefIdx);
3013 }
3014 } else {
3015 report(msg: "No live segment at def", MO, MONum);
3016 report_context_liverange(LR);
3017 report_context_vreg_regunit(VRegOrUnit);
3018 if (LaneMask.any())
3019 report_context_lanemask(LaneMask);
3020 report_context(Pos: DefIdx);
3021 }
3022 // Check that, if the dead def flag is present, LiveInts agree.
3023 if (MO->isDead()) {
3024 LiveQueryResult LRQ = LR.Query(Idx: DefIdx);
3025 if (!LRQ.isDeadDef()) {
3026 assert(VRegOrUnit.isVirtualReg() && "Expecting a virtual register.");
3027 // A dead subreg def only tells us that the specific subreg is dead. There
3028 // could be other non-dead defs of other subregs, or we could have other
3029 // parts of the register being live through the instruction. So unless we
3030 // are checking liveness for a subrange it is ok for the live range to
3031 // continue, given that we have a dead def of a subregister.
3032 if (SubRangeCheck || MO->getSubReg() == 0) {
3033 report(msg: "Live range continues after dead def flag", MO, MONum);
3034 report_context_liverange(LR);
3035 report_context_vreg_regunit(VRegOrUnit);
3036 if (LaneMask.any())
3037 report_context_lanemask(LaneMask);
3038 }
3039 }
3040 }
3041}
3042
3043void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {
3044 const MachineInstr *MI = MO->getParent();
3045 const Register Reg = MO->getReg();
3046 const unsigned SubRegIdx = MO->getSubReg();
3047
3048 const LiveInterval *LI = nullptr;
3049 if (LiveInts && Reg.isVirtual()) {
3050 if (LiveInts->hasInterval(Reg)) {
3051 LI = &LiveInts->getInterval(Reg);
3052 if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&
3053 !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(VReg: Reg))
3054 report(msg: "Live interval for subreg operand has no subranges", MO, MONum);
3055 } else {
3056 report(msg: "Virtual register has no live interval", MO, MONum);
3057 }
3058 }
3059
3060 // Both use and def operands can read a register.
3061 if (MO->readsReg()) {
3062 if (MO->isKill())
3063 addRegWithSubRegs(RV&: regsKilled, Reg);
3064
3065 // Check that LiveVars knows this kill (unless we are inside a bundle, in
3066 // which case we have already checked that LiveVars knows any kills on the
3067 // bundle header instead).
3068 if (LiveVars && Reg.isVirtual() && MO->isKill() &&
3069 !MI->isBundledWithPred()) {
3070 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
3071 if (!is_contained(Range&: VI.Kills, Element: MI))
3072 report(msg: "Kill missing from LiveVariables", MO, MONum);
3073 }
3074
3075 // Check LiveInts liveness and kill.
3076 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3077 SlotIndex UseIdx;
3078 if (MI->isPHI()) {
3079 // PHI use occurs on the edge, so check for live out here instead.
3080 UseIdx = LiveInts->getMBBEndIdx(
3081 mbb: MI->getOperand(i: MONum + 1).getMBB()).getPrevSlot();
3082 } else {
3083 UseIdx = LiveInts->getInstructionIndex(Instr: *MI);
3084 }
3085 // Check the cached regunit intervals.
3086 if (Reg.isPhysical() && !isReserved(Reg)) {
3087 for (MCRegUnit Unit : TRI->regunits(Reg: Reg.asMCReg())) {
3088 if (MRI->isReservedRegUnit(Unit))
3089 continue;
3090 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3091 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LR, VRegOrUnit: VirtRegOrUnit(Unit));
3092 }
3093 }
3094
3095 if (Reg.isVirtual()) {
3096 // This is a virtual register interval.
3097 checkLivenessAtUse(MO, MONum, UseIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3098
3099 if (LI->hasSubRanges() && !MO->isDef()) {
3100 LaneBitmask MOMask = SubRegIdx != 0
3101 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3102 : MRI->getMaxLaneMaskForVReg(Reg);
3103 LaneBitmask LiveInMask;
3104 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3105 if ((MOMask & SR.LaneMask).none())
3106 continue;
3107 checkLivenessAtUse(MO, MONum, UseIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg),
3108 LaneMask: SR.LaneMask);
3109 LiveQueryResult LRQ = SR.Query(Idx: UseIdx);
3110 if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))
3111 LiveInMask |= SR.LaneMask;
3112 }
3113 // At least parts of the register has to be live at the use.
3114 if ((LiveInMask & MOMask).none()) {
3115 report(msg: "No live subrange at use", MO, MONum);
3116 report_context(LI: *LI);
3117 report_context(Pos: UseIdx);
3118 }
3119 // For PHIs all lanes should be live
3120 if (MI->isPHI() && LiveInMask != MOMask) {
3121 report(msg: "Not all lanes of PHI source live at use", MO, MONum);
3122 report_context(LI: *LI);
3123 report_context(Pos: UseIdx);
3124 }
3125 }
3126 }
3127 }
3128
3129 // Use of a dead register.
3130 if (!regsLive.count(V: Reg)) {
3131 if (Reg.isPhysical()) {
3132 // Reserved registers may be used even when 'dead'.
3133 bool Bad = !isReserved(Reg);
3134 // We are fine if just any subregister has a defined value.
3135 if (Bad) {
3136
3137 for (const MCPhysReg &SubReg : TRI->subregs(Reg)) {
3138 if (regsLive.count(V: SubReg)) {
3139 Bad = false;
3140 break;
3141 }
3142 }
3143 }
3144 // If there is an additional implicit-use of a super register we stop
3145 // here. By definition we are fine if the super register is not
3146 // (completely) dead, if the complete super register is dead we will
3147 // get a report for its operand.
3148 if (Bad) {
3149 for (const MachineOperand &MOP : MI->uses()) {
3150 if (!MOP.isReg() || !MOP.isImplicit())
3151 continue;
3152
3153 if (!MOP.getReg().isPhysical())
3154 continue;
3155
3156 if (MOP.getReg() != Reg &&
3157 all_of(Range: TRI->regunits(Reg), P: [&](const MCRegUnit RegUnit) {
3158 return llvm::is_contained(Range: TRI->regunits(Reg: MOP.getReg()),
3159 Element: RegUnit);
3160 }))
3161 Bad = false;
3162 }
3163 }
3164 if (Bad)
3165 report(msg: "Using an undefined physical register", MO, MONum);
3166 } else if (MRI->def_empty(RegNo: Reg)) {
3167 report(msg: "Reading virtual register without a def", MO, MONum);
3168 } else {
3169 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3170 // We don't know which virtual registers are live in, so only complain
3171 // if vreg was killed in this MBB. Otherwise keep track of vregs that
3172 // must be live in. PHI instructions are handled separately.
3173 if (MInfo.regsKilled.count(V: Reg))
3174 report(msg: "Using a killed virtual register", MO, MONum);
3175 else if (!MI->isPHI())
3176 MInfo.vregsLiveIn.insert(KV: std::make_pair(x: Reg, y&: MI));
3177 }
3178 }
3179 }
3180
3181 if (MO->isDef()) {
3182 // Register defined.
3183 // TODO: verify that earlyclobber ops are not used.
3184 if (MO->isDead())
3185 addRegWithSubRegs(RV&: regsDead, Reg);
3186 else
3187 addRegWithSubRegs(RV&: regsDefined, Reg);
3188
3189 // Verify SSA form.
3190 if (MRI->isSSA() && Reg.isVirtual()) {
3191 if (!MRI->hasOneDef(RegNo: Reg))
3192 report(msg: "Multiple virtual register defs in SSA form", MO, MONum);
3193 if (MO->getSubReg())
3194 report(msg: "Subreg def in SSA form", MO, MONum);
3195 }
3196
3197 // Check LiveInts for a live segment, but only for virtual registers.
3198 if (LiveInts && !LiveInts->isNotInMIMap(Instr: *MI)) {
3199 SlotIndex DefIdx = LiveInts->getInstructionIndex(Instr: *MI);
3200 DefIdx = DefIdx.getRegSlot(EC: MO->isEarlyClobber());
3201
3202 if (Reg.isVirtual()) {
3203 checkLivenessAtDef(MO, MONum, DefIdx, LR: *LI, VRegOrUnit: VirtRegOrUnit(Reg));
3204
3205 if (LI->hasSubRanges()) {
3206 LaneBitmask MOMask = SubRegIdx != 0
3207 ? TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx)
3208 : MRI->getMaxLaneMaskForVReg(Reg);
3209 for (const LiveInterval::SubRange &SR : LI->subranges()) {
3210 if ((SR.LaneMask & MOMask).none())
3211 continue;
3212 checkLivenessAtDef(MO, MONum, DefIdx, LR: SR, VRegOrUnit: VirtRegOrUnit(Reg), SubRangeCheck: true,
3213 LaneMask: SR.LaneMask);
3214 }
3215 }
3216 }
3217 }
3218 }
3219}
3220
3221// This function gets called after visiting all instructions in a bundle. The
3222// argument points to the bundle header.
3223// Normal stand-alone instructions are also considered 'bundles', and this
3224// function is called for all of them.
3225void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {
3226 BBInfo &MInfo = MBBInfoMap[MI->getParent()];
3227 set_union(S1&: MInfo.regsKilled, S2: regsKilled);
3228 set_subtract(S1&: regsLive, S2: regsKilled); regsKilled.clear();
3229 // Kill any masked registers.
3230 while (!regMasks.empty()) {
3231 const uint32_t *Mask = regMasks.pop_back_val();
3232 for (Register Reg : regsLive)
3233 if (Reg.isPhysical() &&
3234 MachineOperand::clobbersPhysReg(RegMask: Mask, PhysReg: Reg.asMCReg()))
3235 regsDead.push_back(Elt: Reg);
3236 }
3237 set_subtract(S1&: regsLive, S2: regsDead); regsDead.clear();
3238 set_union(S1&: regsLive, S2: regsDefined); regsDefined.clear();
3239}
3240
3241void
3242MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {
3243 MBBInfoMap[MBB].regsLiveOut = regsLive;
3244 regsLive.clear();
3245
3246 if (Indexes) {
3247 SlotIndex stop = Indexes->getMBBEndIdx(mbb: MBB);
3248 if (!(stop > lastIndex)) {
3249 report(msg: "Block ends before last instruction index", MBB);
3250 OS << "Block ends at " << stop << " last instruction was at " << lastIndex
3251 << '\n';
3252 }
3253 lastIndex = stop;
3254 }
3255}
3256
3257namespace {
3258// This implements a set of registers that serves as a filter: can filter other
3259// sets by passing through elements not in the filter and blocking those that
3260// are. Any filter implicitly includes the full set of physical registers upon
3261// creation, thus filtering them all out. The filter itself as a set only grows,
3262// and needs to be as efficient as possible.
3263struct VRegFilter {
3264 // Add elements to the filter itself. \pre Input set \p FromRegSet must have
3265 // no duplicates. Both virtual and physical registers are fine.
3266 template <typename RegSetT> void add(const RegSetT &FromRegSet) {
3267 SmallVector<Register, 0> VRegsBuffer;
3268 filterAndAdd(FromRegSet, VRegsBuffer);
3269 }
3270 // Filter \p FromRegSet through the filter and append passed elements into \p
3271 // ToVRegs. All elements appended are then added to the filter itself.
3272 // \returns true if anything changed.
3273 template <typename RegSetT>
3274 bool filterAndAdd(const RegSetT &FromRegSet,
3275 SmallVectorImpl<Register> &ToVRegs) {
3276 unsigned SparseUniverse = Sparse.size();
3277 unsigned NewSparseUniverse = SparseUniverse;
3278 unsigned NewDenseSize = Dense.size();
3279 size_t Begin = ToVRegs.size();
3280 for (Register Reg : FromRegSet) {
3281 if (!Reg.isVirtual())
3282 continue;
3283 unsigned Index = Reg.virtRegIndex();
3284 if (Index < SparseUniverseMax) {
3285 if (Index < SparseUniverse && Sparse.test(Idx: Index))
3286 continue;
3287 NewSparseUniverse = std::max(a: NewSparseUniverse, b: Index + 1);
3288 } else {
3289 if (Dense.count(V: Reg))
3290 continue;
3291 ++NewDenseSize;
3292 }
3293 ToVRegs.push_back(Elt: Reg);
3294 }
3295 size_t End = ToVRegs.size();
3296 if (Begin == End)
3297 return false;
3298 // Reserving space in sets once performs better than doing so continuously
3299 // and pays easily for double look-ups (even in Dense with SparseUniverseMax
3300 // tuned all the way down) and double iteration (the second one is over a
3301 // SmallVector, which is a lot cheaper compared to DenseSet or BitVector).
3302 Sparse.resize(N: NewSparseUniverse);
3303 Dense.reserve(Size: NewDenseSize);
3304 for (unsigned I = Begin; I < End; ++I) {
3305 Register Reg = ToVRegs[I];
3306 unsigned Index = Reg.virtRegIndex();
3307 if (Index < SparseUniverseMax)
3308 Sparse.set(Index);
3309 else
3310 Dense.insert(V: Reg);
3311 }
3312 return true;
3313 }
3314
3315private:
3316 static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8;
3317 // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyond
3318 // are tracked by Dense. The only purpose of the threshold and the Dense set
3319 // is to have a reasonably growing memory usage in pathological cases (large
3320 // number of very sparse VRegFilter instances live at the same time). In
3321 // practice even in the worst-by-execution time cases having all elements
3322 // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more
3323 // space efficient than if tracked by Dense. The threshold is set to keep the
3324 // worst-case memory usage within 2x of figures determined empirically for
3325 // "all Dense" scenario in such worst-by-execution-time cases.
3326 BitVector Sparse;
3327 DenseSet<Register> Dense;
3328};
3329
3330// Implements both a transfer function and a (binary, in-place) join operator
3331// for a dataflow over register sets with set union join and filtering transfer
3332// (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.
3333// Maintains out_b as its state, allowing for O(n) iteration over it at any
3334// time, where n is the size of the set (as opposed to O(U) where U is the
3335// universe). filter_b implicitly contains all physical registers at all times.
3336class FilteringVRegSet {
3337 VRegFilter Filter;
3338 SmallVector<Register, 0> VRegs;
3339
3340public:
3341 // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.
3342 // Both virtual and physical registers are fine.
3343 template <typename RegSetT> void addToFilter(const RegSetT &RS) {
3344 Filter.add(RS);
3345 }
3346 // Passes \p RS through the filter_b (transfer function) and adds what's left
3347 // to itself (out_b).
3348 template <typename RegSetT> bool add(const RegSetT &RS) {
3349 // Double-duty the Filter: to maintain VRegs a set (and the join operation
3350 // a set union) just add everything being added here to the Filter as well.
3351 return Filter.filterAndAdd(RS, VRegs);
3352 }
3353 using const_iterator = decltype(VRegs)::const_iterator;
3354 const_iterator begin() const { return VRegs.begin(); }
3355 const_iterator end() const { return VRegs.end(); }
3356 size_t size() const { return VRegs.size(); }
3357};
3358} // namespace
3359
3360// Calculate the largest possible vregsPassed sets. These are the registers that
3361// can pass through an MBB live, but may not be live every time. It is assumed
3362// that all vregsPassed sets are empty before the call.
3363void MachineVerifier::calcRegsPassed() {
3364 if (MF->empty())
3365 // ReversePostOrderTraversal doesn't handle empty functions.
3366 return;
3367
3368 for (const MachineBasicBlock *MB :
3369 ReversePostOrderTraversal<const MachineFunction *>(MF)) {
3370 FilteringVRegSet VRegs;
3371 BBInfo &Info = MBBInfoMap[MB];
3372 assert(Info.reachable);
3373
3374 VRegs.addToFilter(RS: Info.regsKilled);
3375 VRegs.addToFilter(RS: Info.regsLiveOut);
3376 for (const MachineBasicBlock *Pred : MB->predecessors()) {
3377 const BBInfo &PredInfo = MBBInfoMap[Pred];
3378 if (!PredInfo.reachable)
3379 continue;
3380
3381 VRegs.add(RS: PredInfo.regsLiveOut);
3382 VRegs.add(RS: PredInfo.vregsPassed);
3383 }
3384 Info.vregsPassed.reserve(Size: VRegs.size());
3385 Info.vregsPassed.insert_range(R&: VRegs);
3386 }
3387}
3388
3389// Calculate the set of virtual registers that must be passed through each basic
3390// block in order to satisfy the requirements of successor blocks. This is very
3391// similar to calcRegsPassed, only backwards.
3392void MachineVerifier::calcRegsRequired() {
3393 // First push live-in regs to predecessors' vregsRequired.
3394 SmallPtrSet<const MachineBasicBlock*, 8> todo;
3395 for (const auto &MBB : *MF) {
3396 BBInfo &MInfo = MBBInfoMap[&MBB];
3397 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3398 BBInfo &PInfo = MBBInfoMap[Pred];
3399 if (PInfo.addRequired(RM: MInfo.vregsLiveIn))
3400 todo.insert(Ptr: Pred);
3401 }
3402
3403 // Handle the PHI node.
3404 for (const MachineInstr &MI : MBB.phis()) {
3405 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3406 // Skip those Operands which are undef regs or not regs.
3407 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())
3408 continue;
3409
3410 // Get register and predecessor for one PHI edge.
3411 Register Reg = MI.getOperand(i).getReg();
3412 const MachineBasicBlock *Pred = MI.getOperand(i: i + 1).getMBB();
3413
3414 BBInfo &PInfo = MBBInfoMap[Pred];
3415 if (PInfo.addRequired(Reg))
3416 todo.insert(Ptr: Pred);
3417 }
3418 }
3419 }
3420
3421 // Iteratively push vregsRequired to predecessors. This will converge to the
3422 // same final state regardless of DenseSet iteration order.
3423 while (!todo.empty()) {
3424 const MachineBasicBlock *MBB = *todo.begin();
3425 todo.erase(Ptr: MBB);
3426 BBInfo &MInfo = MBBInfoMap[MBB];
3427 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
3428 if (Pred == MBB)
3429 continue;
3430 BBInfo &SInfo = MBBInfoMap[Pred];
3431 if (SInfo.addRequired(RS: MInfo.vregsRequired))
3432 todo.insert(Ptr: Pred);
3433 }
3434 }
3435}
3436
3437// Check PHI instructions at the beginning of MBB. It is assumed that
3438// calcRegsPassed has been run so BBInfo::isLiveOut is valid.
3439void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {
3440 BBInfo &MInfo = MBBInfoMap[&MBB];
3441
3442 SmallPtrSet<const MachineBasicBlock*, 8> seen;
3443 for (const MachineInstr &Phi : MBB) {
3444 if (!Phi.isPHI())
3445 break;
3446 seen.clear();
3447
3448 const MachineOperand &MODef = Phi.getOperand(i: 0);
3449 if (!MODef.isReg() || !MODef.isDef()) {
3450 report(msg: "Expected first PHI operand to be a register def", MO: &MODef, MONum: 0);
3451 continue;
3452 }
3453 if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||
3454 MODef.isEarlyClobber() || MODef.isDebug())
3455 report(msg: "Unexpected flag on PHI operand", MO: &MODef, MONum: 0);
3456 Register DefReg = MODef.getReg();
3457 if (!DefReg.isVirtual())
3458 report(msg: "Expected first PHI operand to be a virtual register", MO: &MODef, MONum: 0);
3459
3460 for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {
3461 const MachineOperand &MO0 = Phi.getOperand(i: I);
3462 if (!MO0.isReg()) {
3463 report(msg: "Expected PHI operand to be a register", MO: &MO0, MONum: I);
3464 continue;
3465 }
3466 if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||
3467 MO0.isDebug() || MO0.isTied())
3468 report(msg: "Unexpected flag on PHI operand", MO: &MO0, MONum: I);
3469
3470 const MachineOperand &MO1 = Phi.getOperand(i: I + 1);
3471 if (!MO1.isMBB()) {
3472 report(msg: "Expected PHI operand to be a basic block", MO: &MO1, MONum: I + 1);
3473 continue;
3474 }
3475
3476 const MachineBasicBlock &Pre = *MO1.getMBB();
3477 if (!Pre.isSuccessor(MBB: &MBB)) {
3478 report(msg: "PHI input is not a predecessor block", MO: &MO1, MONum: I + 1);
3479 continue;
3480 }
3481
3482 if (MInfo.reachable) {
3483 seen.insert(Ptr: &Pre);
3484 BBInfo &PrInfo = MBBInfoMap[&Pre];
3485 if (!MO0.isUndef() && PrInfo.reachable &&
3486 !PrInfo.isLiveOut(Reg: MO0.getReg()))
3487 report(msg: "PHI operand is not live-out from predecessor", MO: &MO0, MONum: I);
3488 }
3489 }
3490
3491 // Did we see all predecessors?
3492 if (MInfo.reachable) {
3493 for (MachineBasicBlock *Pred : MBB.predecessors()) {
3494 if (!seen.count(Ptr: Pred)) {
3495 report(msg: "Missing PHI operand", MI: &Phi);
3496 OS << printMBBReference(MBB: *Pred)
3497 << " is a predecessor according to the CFG.\n";
3498 }
3499 }
3500 }
3501 }
3502}
3503
3504static void
3505verifyConvergenceControl(const MachineFunction &MF, MachineDominatorTree &DT,
3506 std::function<void(const Twine &Message)> FailureCB,
3507 raw_ostream &OS) {
3508 MachineConvergenceVerifier CV;
3509 CV.initialize(OS: &OS, FailureCB, F: MF);
3510
3511 for (const auto &MBB : MF) {
3512 CV.visit(BB: MBB);
3513 for (const auto &MI : MBB.instrs())
3514 CV.visit(I: MI);
3515 }
3516
3517 if (CV.sawTokens()) {
3518 DT.recalculate(Func&: const_cast<MachineFunction &>(MF));
3519 CV.verify(DT);
3520 }
3521}
3522
3523void MachineVerifier::visitMachineFunctionAfter() {
3524 auto FailureCB = [this](const Twine &Message) {
3525 report(msg: Message.str().c_str(), MF);
3526 };
3527 verifyConvergenceControl(MF: *MF, DT, FailureCB, OS);
3528
3529 calcRegsPassed();
3530
3531 for (const MachineBasicBlock &MBB : *MF)
3532 checkPHIOps(MBB);
3533
3534 // Now check liveness info if available
3535 calcRegsRequired();
3536
3537 // Check for killed virtual registers that should be live out.
3538 for (const auto &MBB : *MF) {
3539 BBInfo &MInfo = MBBInfoMap[&MBB];
3540 for (Register VReg : MInfo.vregsRequired)
3541 if (MInfo.regsKilled.count(V: VReg)) {
3542 report(msg: "Virtual register killed in block, but needed live out.", MBB: &MBB);
3543 OS << "Virtual register " << printReg(Reg: VReg)
3544 << " is used after the block.\n";
3545 }
3546 }
3547
3548 if (!MF->empty()) {
3549 BBInfo &MInfo = MBBInfoMap[&MF->front()];
3550 for (Register VReg : MInfo.vregsRequired) {
3551 report(msg: "Virtual register defs don't dominate all uses.", MF);
3552 report_context_vreg(VReg);
3553 }
3554 }
3555
3556 if (LiveVars)
3557 verifyLiveVariables();
3558 if (LiveInts)
3559 verifyLiveIntervals();
3560
3561 // Check live-in list of each MBB. If a register is live into MBB, check
3562 // that the register is in regsLiveOut of each predecessor block. Since
3563 // this must come from a definition in the predecessor or its live-in
3564 // list, this will catch a live-through case where the predecessor does not
3565 // have the register in its live-in list. This currently only checks
3566 // registers that have no aliases, are not allocatable and are not
3567 // reserved, which could mean a condition code register for instance.
3568 if (MRI->tracksLiveness())
3569 for (const auto &MBB : *MF)
3570 for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) {
3571 MCRegister LiveInReg = P.PhysReg;
3572 bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid();
3573 if (hasAliases || isAllocatable(Reg: LiveInReg) || isReserved(Reg: LiveInReg))
3574 continue;
3575 for (const MachineBasicBlock *Pred : MBB.predecessors()) {
3576 BBInfo &PInfo = MBBInfoMap[Pred];
3577 if (!PInfo.regsLiveOut.count(V: LiveInReg)) {
3578 report(msg: "Live in register not found to be live out from predecessor.",
3579 MBB: &MBB);
3580 OS << TRI->getName(RegNo: LiveInReg) << " not found to be live out from "
3581 << printMBBReference(MBB: *Pred) << '\n';
3582 }
3583 }
3584 }
3585
3586 for (auto CSInfo : MF->getCallSitesInfo())
3587 if (!CSInfo.first->isCall())
3588 report(msg: "Call site info referencing instruction that is not call", MF);
3589
3590 // If there's debug-info, check that we don't have any duplicate value
3591 // tracking numbers.
3592 if (MF->getFunction().getSubprogram()) {
3593 DenseSet<unsigned> SeenNumbers;
3594 for (const auto &MBB : *MF) {
3595 for (const auto &MI : MBB) {
3596 if (auto Num = MI.peekDebugInstrNum()) {
3597 auto Result = SeenNumbers.insert(V: (unsigned)Num);
3598 if (!Result.second)
3599 report(msg: "Instruction has a duplicated value tracking number", MI: &MI);
3600 }
3601 }
3602 }
3603 }
3604}
3605
3606void MachineVerifier::verifyLiveVariables() {
3607 assert(LiveVars && "Don't call verifyLiveVariables without LiveVars");
3608 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3609 Register Reg = Register::index2VirtReg(Index: I);
3610 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
3611 for (const auto &MBB : *MF) {
3612 BBInfo &MInfo = MBBInfoMap[&MBB];
3613
3614 // Our vregsRequired should be identical to LiveVariables' AliveBlocks
3615 if (MInfo.vregsRequired.count(V: Reg)) {
3616 if (!VI.AliveBlocks.test(Idx: MBB.getNumber())) {
3617 report(msg: "LiveVariables: Block missing from AliveBlocks", MBB: &MBB);
3618 OS << "Virtual register " << printReg(Reg)
3619 << " must be live through the block.\n";
3620 }
3621 } else {
3622 if (VI.AliveBlocks.test(Idx: MBB.getNumber())) {
3623 report(msg: "LiveVariables: Block should not be in AliveBlocks", MBB: &MBB);
3624 OS << "Virtual register " << printReg(Reg)
3625 << " is not needed live through the block.\n";
3626 }
3627 }
3628 }
3629 }
3630}
3631
3632void MachineVerifier::verifyLiveIntervals() {
3633 assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
3634 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {
3635 Register Reg = Register::index2VirtReg(Index: I);
3636
3637 // Spilling and splitting may leave unused registers around. Skip them.
3638 if (MRI->reg_nodbg_empty(RegNo: Reg))
3639 continue;
3640
3641 if (!LiveInts->hasInterval(Reg)) {
3642 report(msg: "Missing live interval for virtual register", MF);
3643 OS << printReg(Reg, TRI) << " still has defs or uses\n";
3644 continue;
3645 }
3646
3647 const LiveInterval &LI = LiveInts->getInterval(Reg);
3648 assert(Reg == LI.reg() && "Invalid reg to interval mapping");
3649 verifyLiveInterval(LI);
3650 }
3651
3652 // Verify all the cached regunit intervals.
3653 for (MCRegUnit Unit : TRI->regunits())
3654 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))
3655 verifyLiveRange(*LR, VirtRegOrUnit(Unit));
3656}
3657
3658void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,
3659 const VNInfo *VNI,
3660 VirtRegOrUnit VRegOrUnit,
3661 LaneBitmask LaneMask) {
3662 if (VNI->isUnused())
3663 return;
3664
3665 const VNInfo *DefVNI = LR.getVNInfoAt(Idx: VNI->def);
3666
3667 if (!DefVNI) {
3668 report(msg: "Value not live at VNInfo def and not marked unused", MF);
3669 report_context(LR, VRegOrUnit, LaneMask);
3670 report_context(VNI: *VNI);
3671 return;
3672 }
3673
3674 if (DefVNI != VNI) {
3675 report(msg: "Live segment at def has different VNInfo", MF);
3676 report_context(LR, VRegOrUnit, LaneMask);
3677 report_context(VNI: *VNI);
3678 return;
3679 }
3680
3681 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: VNI->def);
3682 if (!MBB) {
3683 report(msg: "Invalid VNInfo definition index", MF);
3684 report_context(LR, VRegOrUnit, LaneMask);
3685 report_context(VNI: *VNI);
3686 return;
3687 }
3688
3689 if (VNI->isPHIDef()) {
3690 if (VNI->def != LiveInts->getMBBStartIdx(mbb: MBB)) {
3691 report(msg: "PHIDef VNInfo is not defined at MBB start", MBB);
3692 report_context(LR, VRegOrUnit, LaneMask);
3693 report_context(VNI: *VNI);
3694 }
3695 return;
3696 }
3697
3698 // Non-PHI def.
3699 const MachineInstr *MI = LiveInts->getInstructionFromIndex(index: VNI->def);
3700 if (!MI) {
3701 report(msg: "No instruction at VNInfo def index", MBB);
3702 report_context(LR, VRegOrUnit, LaneMask);
3703 report_context(VNI: *VNI);
3704 return;
3705 }
3706
3707 bool hasDef = false;
3708 bool isEarlyClobber = false;
3709 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3710 if (!MOI->isReg() || !MOI->isDef())
3711 continue;
3712 if (VRegOrUnit.isVirtualReg()) {
3713 if (MOI->getReg() != VRegOrUnit.asVirtualReg())
3714 continue;
3715 } else {
3716 if (!MOI->getReg().isPhysical() ||
3717 !TRI->hasRegUnit(Reg: MOI->getReg(), RegUnit: VRegOrUnit.asMCRegUnit()))
3718 continue;
3719 }
3720 if (LaneMask.any() &&
3721 (TRI->getSubRegIndexLaneMask(SubIdx: MOI->getSubReg()) & LaneMask).none())
3722 continue;
3723 hasDef = true;
3724 if (MOI->isEarlyClobber())
3725 isEarlyClobber = true;
3726 }
3727
3728 if (!hasDef) {
3729 report(msg: "Defining instruction does not modify register", MI);
3730 report_context(LR, VRegOrUnit, LaneMask);
3731 report_context(VNI: *VNI);
3732 }
3733
3734 // Early clobber defs begin at USE slots, but other defs must begin at
3735 // DEF slots.
3736 if (isEarlyClobber) {
3737 if (!VNI->def.isEarlyClobber()) {
3738 report(msg: "Early clobber def must be at an early-clobber slot", MBB);
3739 report_context(LR, VRegOrUnit, LaneMask);
3740 report_context(VNI: *VNI);
3741 }
3742 } else if (!VNI->def.isRegister()) {
3743 report(msg: "Non-PHI, non-early clobber def must be at a register slot", MBB);
3744 report_context(LR, VRegOrUnit, LaneMask);
3745 report_context(VNI: *VNI);
3746 }
3747}
3748
3749void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
3750 const LiveRange::const_iterator I,
3751 VirtRegOrUnit VRegOrUnit,
3752 LaneBitmask LaneMask) {
3753 const LiveRange::Segment &S = *I;
3754 const VNInfo *VNI = S.valno;
3755 assert(VNI && "Live segment has no valno");
3756
3757 if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(ValNo: VNI->id)) {
3758 report(msg: "Foreign valno in live segment", MF);
3759 report_context(LR, VRegOrUnit, LaneMask);
3760 report_context(S);
3761 report_context(VNI: *VNI);
3762 }
3763
3764 if (VNI->isUnused()) {
3765 report(msg: "Live segment valno is marked unused", MF);
3766 report_context(LR, VRegOrUnit, LaneMask);
3767 report_context(S);
3768 }
3769
3770 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(index: S.start);
3771 if (!MBB) {
3772 report(msg: "Bad start of live segment, no basic block", MF);
3773 report_context(LR, VRegOrUnit, LaneMask);
3774 report_context(S);
3775 return;
3776 }
3777 SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(mbb: MBB);
3778 if (S.start != MBBStartIdx && S.start != VNI->def) {
3779 report(msg: "Live segment must begin at MBB entry or valno def", MBB);
3780 report_context(LR, VRegOrUnit, LaneMask);
3781 report_context(S);
3782 }
3783
3784 const MachineBasicBlock *EndMBB =
3785 LiveInts->getMBBFromIndex(index: S.end.getPrevSlot());
3786 if (!EndMBB) {
3787 report(msg: "Bad end of live segment, no basic block", MF);
3788 report_context(LR, VRegOrUnit, LaneMask);
3789 report_context(S);
3790 return;
3791 }
3792
3793 // Checks for non-live-out segments.
3794 if (S.end != LiveInts->getMBBEndIdx(mbb: EndMBB)) {
3795 // RegUnit intervals are allowed dead phis.
3796 if (!VRegOrUnit.isVirtualReg() && VNI->isPHIDef() && S.start == VNI->def &&
3797 S.end == VNI->def.getDeadSlot())
3798 return;
3799
3800 // The live segment is ending inside EndMBB
3801 const MachineInstr *MI =
3802 LiveInts->getInstructionFromIndex(index: S.end.getPrevSlot());
3803 if (!MI) {
3804 report(msg: "Live segment doesn't end at a valid instruction", MBB: EndMBB);
3805 report_context(LR, VRegOrUnit, LaneMask);
3806 report_context(S);
3807 return;
3808 }
3809
3810 // The block slot must refer to a basic block boundary.
3811 if (S.end.isBlock()) {
3812 report(msg: "Live segment ends at B slot of an instruction", MBB: EndMBB);
3813 report_context(LR, VRegOrUnit, LaneMask);
3814 report_context(S);
3815 }
3816
3817 if (S.end.isDead()) {
3818 // Segment ends on the dead slot.
3819 // That means there must be a dead def.
3820 if (!SlotIndex::isSameInstr(A: S.start, B: S.end)) {
3821 report(msg: "Live segment ending at dead slot spans instructions", MBB: EndMBB);
3822 report_context(LR, VRegOrUnit, LaneMask);
3823 report_context(S);
3824 }
3825 }
3826
3827 // After tied operands are rewritten, a live segment can only end at an
3828 // early-clobber slot if it is being redefined by an early-clobber def.
3829 // TODO: Before tied operands are rewritten, a live segment can only end at
3830 // an early-clobber slot if the last use is tied to an early-clobber def.
3831 if (MF->getProperties().hasTiedOpsRewritten() && S.end.isEarlyClobber()) {
3832 if (I + 1 == LR.end() || (I + 1)->start != S.end) {
3833 report(msg: "Live segment ending at early clobber slot must be "
3834 "redefined by an EC def in the same instruction",
3835 MBB: EndMBB);
3836 report_context(LR, VRegOrUnit, LaneMask);
3837 report_context(S);
3838 }
3839 }
3840
3841 // The following checks only apply to virtual registers. Physreg liveness
3842 // is too weird to check.
3843 if (VRegOrUnit.isVirtualReg()) {
3844 // A live segment can end with either a redefinition, a kill flag on a
3845 // use, or a dead flag on a def.
3846 bool hasRead = false;
3847 bool hasSubRegDef = false;
3848 bool hasDeadDef = false;
3849 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
3850 if (!MOI->isReg() || MOI->getReg() != VRegOrUnit.asVirtualReg())
3851 continue;
3852 unsigned Sub = MOI->getSubReg();
3853 LaneBitmask SLM =
3854 Sub != 0 ? TRI->getSubRegIndexLaneMask(SubIdx: Sub) : LaneBitmask::getAll();
3855 if (MOI->isDef()) {
3856 if (Sub != 0) {
3857 hasSubRegDef = true;
3858 // An operand %0:sub0 reads %0:sub1..n. Invert the lane
3859 // mask for subregister defs. Read-undef defs will be handled by
3860 // readsReg below.
3861 SLM = ~SLM;
3862 }
3863 if (MOI->isDead())
3864 hasDeadDef = true;
3865 }
3866 if (LaneMask.any() && (LaneMask & SLM).none())
3867 continue;
3868 if (MOI->readsReg())
3869 hasRead = true;
3870 }
3871 if (S.end.isDead()) {
3872 // Make sure that the corresponding machine operand for a "dead" live
3873 // range has the dead flag. We cannot perform this check for subregister
3874 // liveranges as partially dead values are allowed.
3875 if (LaneMask.none() && !hasDeadDef) {
3876 report(
3877 msg: "Instruction ending live segment on dead slot has no dead flag",
3878 MI);
3879 report_context(LR, VRegOrUnit, LaneMask);
3880 report_context(S);
3881 }
3882 } else {
3883 if (!hasRead) {
3884 // When tracking subregister liveness, the main range must start new
3885 // values on partial register writes, even if there is no read.
3886 if (!MRI->shouldTrackSubRegLiveness(VReg: VRegOrUnit.asVirtualReg()) ||
3887 LaneMask.any() || !hasSubRegDef) {
3888 report(msg: "Instruction ending live segment doesn't read the register",
3889 MI);
3890 report_context(LR, VRegOrUnit, LaneMask);
3891 report_context(S);
3892 }
3893 }
3894 }
3895 }
3896 }
3897
3898 // Now check all the basic blocks in this live segment.
3899 MachineFunction::const_iterator MFI = MBB->getIterator();
3900 // Is this live segment the beginning of a non-PHIDef VN?
3901 if (S.start == VNI->def && !VNI->isPHIDef()) {
3902 // Not live-in to any blocks.
3903 if (MBB == EndMBB)
3904 return;
3905 // Skip this block.
3906 ++MFI;
3907 }
3908
3909 SmallVector<SlotIndex, 4> Undefs;
3910 if (LaneMask.any()) {
3911 LiveInterval &OwnerLI = LiveInts->getInterval(Reg: VRegOrUnit.asVirtualReg());
3912 OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, MRI: *MRI, Indexes: *Indexes);
3913 }
3914
3915 while (true) {
3916 assert(LiveInts->isLiveInToMBB(LR, &*MFI));
3917 // We don't know how to track physregs into a landing pad.
3918 if (!VRegOrUnit.isVirtualReg() && MFI->isEHPad()) {
3919 if (&*MFI == EndMBB)
3920 break;
3921 ++MFI;
3922 continue;
3923 }
3924
3925 // Is VNI a PHI-def in the current block?
3926 bool IsPHI = VNI->isPHIDef() &&
3927 VNI->def == LiveInts->getMBBStartIdx(mbb: &*MFI);
3928
3929 // Check that VNI is live-out of all predecessors.
3930 for (const MachineBasicBlock *Pred : MFI->predecessors()) {
3931 SlotIndex PEnd = LiveInts->getMBBEndIdx(mbb: Pred);
3932 // Predecessor of landing pad live-out on last call.
3933 if (MFI->isEHPad()) {
3934 for (const MachineInstr &MI : llvm::reverse(C: *Pred)) {
3935 if (MI.isCall()) {
3936 PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex();
3937 break;
3938 }
3939 }
3940 }
3941 const VNInfo *PVNI = LR.getVNInfoBefore(Idx: PEnd);
3942
3943 // All predecessors must have a live-out value. However for a phi
3944 // instruction with subregister intervals
3945 // only one of the subregisters (not necessarily the current one) needs to
3946 // be defined.
3947 if (!PVNI && (LaneMask.none() || !IsPHI)) {
3948 if (LiveRangeCalc::isJointlyDominated(MBB: Pred, Defs: Undefs, Indexes: *Indexes))
3949 continue;
3950 report(msg: "Register not marked live out of predecessor", MBB: Pred);
3951 report_context(LR, VRegOrUnit, LaneMask);
3952 report_context(VNI: *VNI);
3953 OS << " live into " << printMBBReference(MBB: *MFI) << '@'
3954 << LiveInts->getMBBStartIdx(mbb: &*MFI) << ", not live before " << PEnd
3955 << '\n';
3956 continue;
3957 }
3958
3959 // Only PHI-defs can take different predecessor values.
3960 if (!IsPHI && PVNI != VNI) {
3961 report(msg: "Different value live out of predecessor", MBB: Pred);
3962 report_context(LR, VRegOrUnit, LaneMask);
3963 OS << "Valno #" << PVNI->id << " live out of "
3964 << printMBBReference(MBB: *Pred) << '@' << PEnd << "\nValno #" << VNI->id
3965 << " live into " << printMBBReference(MBB: *MFI) << '@'
3966 << LiveInts->getMBBStartIdx(mbb: &*MFI) << '\n';
3967 }
3968 }
3969 if (&*MFI == EndMBB)
3970 break;
3971 ++MFI;
3972 }
3973}
3974
3975void MachineVerifier::verifyLiveRange(const LiveRange &LR,
3976 VirtRegOrUnit VRegOrUnit,
3977 LaneBitmask LaneMask) {
3978 for (const VNInfo *VNI : LR.valnos)
3979 verifyLiveRangeValue(LR, VNI, VRegOrUnit, LaneMask);
3980
3981 for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)
3982 verifyLiveRangeSegment(LR, I, VRegOrUnit, LaneMask);
3983}
3984
3985void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {
3986 Register Reg = LI.reg();
3987 assert(Reg.isVirtual());
3988 verifyLiveRange(LR: LI, VRegOrUnit: VirtRegOrUnit(Reg));
3989
3990 if (LI.hasSubRanges()) {
3991 LaneBitmask Mask;
3992 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
3993 for (const LiveInterval::SubRange &SR : LI.subranges()) {
3994 if ((Mask & SR.LaneMask).any()) {
3995 report(msg: "Lane masks of sub ranges overlap in live interval", MF);
3996 report_context(LI);
3997 }
3998 if ((SR.LaneMask & ~MaxMask).any()) {
3999 report(msg: "Subrange lanemask is invalid", MF);
4000 report_context(LI);
4001 }
4002 if (SR.empty()) {
4003 report(msg: "Subrange must not be empty", MF);
4004 report_context(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
4005 }
4006 Mask |= SR.LaneMask;
4007 verifyLiveRange(LR: SR, VRegOrUnit: VirtRegOrUnit(LI.reg()), LaneMask: SR.LaneMask);
4008 if (!LI.covers(Other: SR)) {
4009 report(msg: "A Subrange is not covered by the main range", MF);
4010 report_context(LI);
4011 }
4012 }
4013 }
4014
4015 // Check the LI only has one connected component.
4016 ConnectedVNInfoEqClasses ConEQ(*LiveInts);
4017 unsigned NumComp = ConEQ.Classify(LR: LI);
4018 if (NumComp > 1) {
4019 report(msg: "Multiple connected components in live interval", MF);
4020 report_context(LI);
4021 for (unsigned comp = 0; comp != NumComp; ++comp) {
4022 OS << comp << ": valnos";
4023 for (const VNInfo *I : LI.valnos)
4024 if (comp == ConEQ.getEqClass(VNI: I))
4025 OS << ' ' << I->id;
4026 OS << '\n';
4027 }
4028 }
4029}
4030
4031namespace {
4032
4033 // FrameSetup and FrameDestroy can have zero adjustment, so using a single
4034 // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the
4035 // value is zero.
4036 // We use a bool plus an integer to capture the stack state.
4037struct StackStateOfBB {
4038 StackStateOfBB() = default;
4039 StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup)
4040 : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),
4041 ExitIsSetup(ExitSetup) {}
4042
4043 // Can be negative, which means we are setting up a frame.
4044 int EntryValue = 0;
4045 int ExitValue = 0;
4046 bool EntryIsSetup = false;
4047 bool ExitIsSetup = false;
4048};
4049
4050} // end anonymous namespace
4051
4052/// Make sure on every path through the CFG, a FrameSetup <n> is always followed
4053/// by a FrameDestroy <n>, stack adjustments are identical on all
4054/// CFG edges to a merge point, and frame is destroyed at end of a return block.
4055void MachineVerifier::verifyStackFrame() {
4056 unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode();
4057 unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
4058 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
4059 return;
4060
4061 SmallVector<StackStateOfBB, 8> SPState;
4062 SPState.resize(N: MF->getNumBlockIDs());
4063 df_iterator_default_set<const MachineBasicBlock*> Reachable;
4064
4065 // Visit the MBBs in DFS order.
4066 for (df_ext_iterator<const MachineFunction *,
4067 df_iterator_default_set<const MachineBasicBlock *>>
4068 DFI = df_ext_begin(G: MF, S&: Reachable), DFE = df_ext_end(G: MF, S&: Reachable);
4069 DFI != DFE; ++DFI) {
4070 const MachineBasicBlock *MBB = *DFI;
4071
4072 StackStateOfBB BBState;
4073 // Check the exit state of the DFS stack predecessor.
4074 if (DFI.getPathLength() >= 2) {
4075 const MachineBasicBlock *StackPred = DFI.getPath(n: DFI.getPathLength() - 2);
4076 assert(Reachable.count(StackPred) &&
4077 "DFS stack predecessor is already visited.\n");
4078 BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;
4079 BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;
4080 BBState.ExitValue = BBState.EntryValue;
4081 BBState.ExitIsSetup = BBState.EntryIsSetup;
4082 }
4083
4084 if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) {
4085 report(msg: "Call frame size on entry does not match value computed from "
4086 "predecessor",
4087 MBB);
4088 OS << "Call frame size on entry " << MBB->getCallFrameSize()
4089 << " does not match value computed from predecessor "
4090 << -BBState.EntryValue << '\n';
4091 }
4092
4093 // Update stack state by checking contents of MBB.
4094 for (const auto &I : *MBB) {
4095 if (I.getOpcode() == FrameSetupOpcode) {
4096 if (BBState.ExitIsSetup)
4097 report(msg: "FrameSetup is after another FrameSetup", MI: &I);
4098 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4099 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4100 "instruction.", MI: &I);
4101 BBState.ExitValue -= TII->getFrameTotalSize(I);
4102 BBState.ExitIsSetup = true;
4103 }
4104
4105 if (I.getOpcode() == FrameDestroyOpcode) {
4106 int Size = TII->getFrameTotalSize(I);
4107 if (!BBState.ExitIsSetup)
4108 report(msg: "FrameDestroy is not after a FrameSetup", MI: &I);
4109 int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :
4110 BBState.ExitValue;
4111 if (BBState.ExitIsSetup && AbsSPAdj != Size) {
4112 report(msg: "FrameDestroy <n> is after FrameSetup <m>", MI: &I);
4113 OS << "FrameDestroy <" << Size << "> is after FrameSetup <"
4114 << AbsSPAdj << ">.\n";
4115 }
4116 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())
4117 report(msg: "AdjustsStack not set in presence of a frame pseudo "
4118 "instruction.", MI: &I);
4119 BBState.ExitValue += Size;
4120 BBState.ExitIsSetup = false;
4121 }
4122 }
4123 SPState[MBB->getNumber()] = BBState;
4124
4125 // Make sure the exit state of any predecessor is consistent with the entry
4126 // state.
4127 for (const MachineBasicBlock *Pred : MBB->predecessors()) {
4128 if (Reachable.count(Ptr: Pred) &&
4129 (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue ||
4130 SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {
4131 report(msg: "The exit stack state of a predecessor is inconsistent.", MBB);
4132 OS << "Predecessor " << printMBBReference(MBB: *Pred) << " has exit state ("
4133 << SPState[Pred->getNumber()].ExitValue << ", "
4134 << SPState[Pred->getNumber()].ExitIsSetup << "), while "
4135 << printMBBReference(MBB: *MBB) << " has entry state ("
4136 << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";
4137 }
4138 }
4139
4140 // Make sure the entry state of any successor is consistent with the exit
4141 // state.
4142 for (const MachineBasicBlock *Succ : MBB->successors()) {
4143 if (Reachable.count(Ptr: Succ) &&
4144 (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue ||
4145 SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {
4146 report(msg: "The entry stack state of a successor is inconsistent.", MBB);
4147 OS << "Successor " << printMBBReference(MBB: *Succ) << " has entry state ("
4148 << SPState[Succ->getNumber()].EntryValue << ", "
4149 << SPState[Succ->getNumber()].EntryIsSetup << "), while "
4150 << printMBBReference(MBB: *MBB) << " has exit state ("
4151 << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";
4152 }
4153 }
4154
4155 // Make sure a basic block with return ends with zero stack adjustment.
4156 if (!MBB->empty() && MBB->back().isReturn()) {
4157 if (BBState.ExitIsSetup)
4158 report(msg: "A return block ends with a FrameSetup.", MBB);
4159 if (BBState.ExitValue)
4160 report(msg: "A return block ends with a nonzero stack adjustment.", MBB);
4161 }
4162 }
4163}
4164
4165void MachineVerifier::verifyStackProtector() {
4166 const MachineFrameInfo &MFI = MF->getFrameInfo();
4167 if (!MFI.hasStackProtectorIndex())
4168 return;
4169 // Only applicable when the offsets of frame objects have been determined,
4170 // which is indicated by a non-zero stack size.
4171 if (!MFI.getStackSize())
4172 return;
4173 const TargetFrameLowering &TFI = *MF->getSubtarget().getFrameLowering();
4174 bool StackGrowsDown =
4175 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
4176 unsigned FI = MFI.getStackProtectorIndex();
4177 int64_t SPStart = MFI.getObjectOffset(ObjectIdx: FI);
4178 int64_t SPEnd = SPStart + MFI.getObjectSize(ObjectIdx: FI);
4179 for (unsigned I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
4180 if (I == FI)
4181 continue;
4182 if (MFI.isDeadObjectIndex(ObjectIdx: I))
4183 continue;
4184 // FIXME: Skip non-default stack objects, as some targets may place them
4185 // above the stack protector. This is a workaround for the fact that
4186 // backends such as AArch64 may place SVE stack objects *above* the stack
4187 // protector.
4188 if (MFI.getStackID(ObjectIdx: I) != TargetStackID::Default)
4189 continue;
4190 // Skip variable-sized objects because they do not have a fixed offset.
4191 if (MFI.isVariableSizedObjectIndex(ObjectIdx: I))
4192 continue;
4193 // FIXME: Skip spill slots which may be allocated above the stack protector.
4194 // Ideally this would only skip callee-saved registers, but we don't have
4195 // that information here. For example, spill-slots used for scavenging are
4196 // not described in CalleeSavedInfo.
4197 if (MFI.isSpillSlotObjectIndex(ObjectIdx: I))
4198 continue;
4199 int64_t ObjStart = MFI.getObjectOffset(ObjectIdx: I);
4200 int64_t ObjEnd = ObjStart + MFI.getObjectSize(ObjectIdx: I);
4201 if (SPStart < ObjEnd && ObjStart < SPEnd) {
4202 report(msg: "Stack protector overlaps with another stack object", MF);
4203 break;
4204 }
4205 if ((StackGrowsDown && SPStart <= ObjStart) ||
4206 (!StackGrowsDown && SPStart >= ObjStart)) {
4207 report(msg: "Stack protector is not the top-most object on the stack", MF);
4208 break;
4209 }
4210 }
4211}
4212