1//===-- llvm/CodeGen/GlobalISel/Legalizer.cpp -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file implements the LegalizerHelper class to legalize individual
10/// instructions and the LegalizePass wrapper pass for the primary
11/// legalization.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/CodeGen/GlobalISel/Legalizer.h"
16#include "llvm/ADT/PostOrderIterator.h"
17#include "llvm/Analysis/OptimizationRemarkEmitter.h"
18#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
19#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
20#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
21#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
22#include "llvm/CodeGen/GlobalISel/GISelWorkList.h"
23#include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
24#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
25#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
26#include "llvm/CodeGen/GlobalISel/Utils.h"
27#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
28#include "llvm/CodeGen/TargetPassConfig.h"
29#include "llvm/CodeGen/TargetSubtargetInfo.h"
30#include "llvm/Support/Debug.h"
31#include "llvm/Support/Error.h"
32
33#define DEBUG_TYPE "legalizer"
34
35using namespace llvm;
36
37static cl::opt<bool>
38 EnableCSEInLegalizer("enable-cse-in-legalizer",
39 cl::desc("Should enable CSE in Legalizer"),
40 cl::Optional, cl::init(Val: false));
41
42// This is a temporary hack, should be removed soon.
43static cl::opt<bool> AllowGInsertAsArtifact(
44 "allow-ginsert-as-artifact",
45 cl::desc("Allow G_INSERT to be considered an artifact. Hack around AMDGPU "
46 "test infinite loops."),
47 cl::Optional, cl::init(Val: true));
48
49enum class DebugLocVerifyLevel {
50 None,
51 Legalizations,
52 LegalizationsAndArtifactCombiners,
53};
54#ifndef NDEBUG
55static cl::opt<DebugLocVerifyLevel> VerifyDebugLocs(
56 "verify-legalizer-debug-locs",
57 cl::desc("Verify that debug locations are handled"),
58 cl::values(
59 clEnumValN(DebugLocVerifyLevel::None, "none", "No verification"),
60 clEnumValN(DebugLocVerifyLevel::Legalizations, "legalizations",
61 "Verify legalizations"),
62 clEnumValN(DebugLocVerifyLevel::LegalizationsAndArtifactCombiners,
63 "legalizations+artifactcombiners",
64 "Verify legalizations and artifact combines")),
65 cl::init(DebugLocVerifyLevel::Legalizations));
66#else
67// Always disable it for release builds by preventing the observer from being
68// installed.
69static const DebugLocVerifyLevel VerifyDebugLocs = DebugLocVerifyLevel::None;
70#endif
71
72char Legalizer::ID = 0;
73INITIALIZE_PASS_BEGIN(Legalizer, DEBUG_TYPE,
74 "Legalize the Machine IR a function's Machine IR", false,
75 false)
76INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
77INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
78INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
79INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
80INITIALIZE_PASS_END(Legalizer, DEBUG_TYPE,
81 "Legalize the Machine IR a function's Machine IR", false,
82 false)
83
84Legalizer::Legalizer() : MachineFunctionPass(ID) { }
85
86void Legalizer::getAnalysisUsage(AnalysisUsage &AU) const {
87 AU.addRequired<LibcallLoweringInfoWrapper>();
88 AU.addRequired<TargetPassConfig>();
89 AU.addRequired<GISelCSEAnalysisWrapperPass>();
90 AU.addPreserved<GISelCSEAnalysisWrapperPass>();
91 AU.addRequired<GISelValueTrackingAnalysisLegacy>();
92 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
93 getSelectionDAGFallbackAnalysisUsage(AU);
94 MachineFunctionPass::getAnalysisUsage(AU);
95}
96
97void Legalizer::init(MachineFunction &MF) {
98}
99
100static bool isArtifact(const MachineInstr &MI) {
101 switch (MI.getOpcode()) {
102 default:
103 return false;
104 case TargetOpcode::G_TRUNC:
105 case TargetOpcode::G_ZEXT:
106 case TargetOpcode::G_ANYEXT:
107 case TargetOpcode::G_SEXT:
108 case TargetOpcode::G_MERGE_VALUES:
109 case TargetOpcode::G_UNMERGE_VALUES:
110 case TargetOpcode::G_CONCAT_VECTORS:
111 case TargetOpcode::G_BUILD_VECTOR:
112 case TargetOpcode::G_EXTRACT:
113 return true;
114 case TargetOpcode::G_INSERT:
115 return AllowGInsertAsArtifact;
116 }
117}
118using InstListTy = GISelWorkList<256>;
119using ArtifactListTy = GISelWorkList<128>;
120
121namespace {
122class LegalizerWorkListManager : public GISelChangeObserver {
123 InstListTy &InstList;
124 ArtifactListTy &ArtifactList;
125#ifndef NDEBUG
126 SmallVector<MachineInstr *, 4> NewMIs;
127#endif
128
129public:
130 LegalizerWorkListManager(InstListTy &Insts, ArtifactListTy &Arts)
131 : InstList(Insts), ArtifactList(Arts) {}
132
133 void createdOrChangedInstr(MachineInstr &MI) {
134 // Only legalize pre-isel generic instructions.
135 // Legalization process could generate Target specific pseudo
136 // instructions with generic types. Don't record them
137 if (isPreISelGenericOpcode(Opcode: MI.getOpcode())) {
138 if (isArtifact(MI))
139 ArtifactList.insert(I: &MI);
140 else
141 InstList.insert(I: &MI);
142 }
143 }
144
145 void createdInstr(MachineInstr &MI) override {
146 LLVM_DEBUG(NewMIs.push_back(&MI));
147 createdOrChangedInstr(MI);
148 }
149
150 void printNewInstrs() {
151 LLVM_DEBUG({
152 for (const auto *MI : NewMIs)
153 dbgs() << ".. .. New MI: " << *MI;
154 NewMIs.clear();
155 });
156 }
157
158 void erasingInstr(MachineInstr &MI) override {
159 LLVM_DEBUG(dbgs() << ".. .. Erasing: " << MI);
160 InstList.remove(I: &MI);
161 ArtifactList.remove(I: &MI);
162 }
163
164 void changingInstr(MachineInstr &MI) override {
165 LLVM_DEBUG(dbgs() << ".. .. Changing MI: " << MI);
166 }
167
168 void changedInstr(MachineInstr &MI) override {
169 // When insts change, we want to revisit them to legalize them again.
170 // We'll consider them the same as created.
171 LLVM_DEBUG(dbgs() << ".. .. Changed MI: " << MI);
172 createdOrChangedInstr(MI);
173 }
174};
175} // namespace
176
177Legalizer::MFResult Legalizer::legalizeMachineFunction(
178 MachineFunction &MF, const LegalizerInfo &LI,
179 ArrayRef<GISelChangeObserver *> AuxObservers,
180 LostDebugLocObserver &LocObserver, MachineIRBuilder &MIRBuilder,
181 const LibcallLoweringInfo *Libcalls, GISelValueTracking *VT) {
182 MIRBuilder.setMF(MF);
183 MachineRegisterInfo &MRI = MF.getRegInfo();
184
185 // Populate worklists.
186 InstListTy InstList;
187 ArtifactListTy ArtifactList;
188 ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);
189 // Perform legalization bottom up so we can DCE as we legalize.
190 // Traverse BB in RPOT and within each basic block, add insts top down,
191 // so when we pop_back_val in the legalization process, we traverse bottom-up.
192 for (auto *MBB : RPOT) {
193 if (MBB->empty())
194 continue;
195 for (MachineInstr &MI : *MBB) {
196 // Only legalize pre-isel generic instructions: others don't have types
197 // and are assumed to be legal.
198 if (!isPreISelGenericOpcode(Opcode: MI.getOpcode()))
199 continue;
200 if (isArtifact(MI))
201 ArtifactList.deferred_insert(I: &MI);
202 else
203 InstList.deferred_insert(I: &MI);
204 }
205 }
206 ArtifactList.finalize();
207 InstList.finalize();
208
209 // This observer keeps the worklists updated.
210 LegalizerWorkListManager WorkListObserver(InstList, ArtifactList);
211 // We want both WorkListObserver as well as all the auxiliary observers (e.g.
212 // CSEInfo) to observe all changes. Use the wrapper observer.
213 GISelObserverWrapper WrapperObserver(&WorkListObserver);
214 for (GISelChangeObserver *Observer : AuxObservers)
215 WrapperObserver.addObserver(O: Observer);
216
217 // Now install the observer as the delegate to MF.
218 // This will keep all the observers notified about new insertions/deletions.
219 RAIIMFObsDelInstaller Installer(MF, WrapperObserver);
220 LegalizerHelper Helper(MF, LI, WrapperObserver, MIRBuilder, Libcalls, VT);
221 LegalizationArtifactCombiner ArtCombiner(MIRBuilder, MRI, LI, VT);
222 bool Changed = false;
223 SmallVector<MachineInstr *, 128> RetryList;
224 do {
225 LLVM_DEBUG(dbgs() << "=== New Iteration ===\n");
226 assert(RetryList.empty() && "Expected no instructions in RetryList");
227 unsigned NumArtifacts = ArtifactList.size();
228 while (!InstList.empty()) {
229 MachineInstr &MI = *InstList.pop_back_val();
230 assert(isPreISelGenericOpcode(MI.getOpcode()) &&
231 "Expecting generic opcode");
232 if (isTriviallyDead(MI, MRI)) {
233 salvageDebugInfo(MRI, MI);
234 eraseInstr(MI, MRI, LocObserver: &LocObserver);
235 continue;
236 }
237
238 // Do the legalization for this instruction.
239 auto Res = Helper.legalizeInstrStep(MI, LocObserver);
240 // Error out if we couldn't legalize this instruction. We may want to
241 // fall back to DAG ISel instead in the future.
242 if (Res == LegalizerHelper::UnableToLegalize) {
243 // Move illegal artifacts to RetryList instead of aborting because
244 // legalizing InstList may generate artifacts that allow
245 // ArtifactCombiner to combine away them.
246 if (isArtifact(MI)) {
247 LLVM_DEBUG(dbgs() << ".. Not legalized, moving to artifacts retry\n");
248 assert(NumArtifacts == 0 &&
249 "Artifacts are only expected in instruction list starting the "
250 "second iteration, but each iteration starting second must "
251 "start with an empty artifacts list");
252 (void)NumArtifacts;
253 RetryList.push_back(Elt: &MI);
254 continue;
255 }
256 Helper.MIRBuilder.stopObservingChanges();
257 return {.Changed: Changed, .FailedOn: &MI};
258 }
259 WorkListObserver.printNewInstrs();
260 LocObserver.checkpoint();
261 Changed |= Res == LegalizerHelper::Legalized;
262 }
263 // Try to combine the instructions in RetryList again if there
264 // are new artifacts. If not, stop legalizing.
265 if (!RetryList.empty()) {
266 if (!ArtifactList.empty()) {
267 while (!RetryList.empty())
268 ArtifactList.insert(I: RetryList.pop_back_val());
269 } else {
270 LLVM_DEBUG(dbgs() << "No new artifacts created, not retrying!\n");
271 Helper.MIRBuilder.stopObservingChanges();
272 return {.Changed: Changed, .FailedOn: RetryList.front()};
273 }
274 }
275 LocObserver.checkpoint();
276 while (!ArtifactList.empty()) {
277 MachineInstr &MI = *ArtifactList.pop_back_val();
278 assert(isPreISelGenericOpcode(MI.getOpcode()) &&
279 "Expecting generic opcode");
280 if (isTriviallyDead(MI, MRI)) {
281 salvageDebugInfo(MRI, MI);
282 eraseInstr(MI, MRI, LocObserver: &LocObserver);
283 continue;
284 }
285 SmallVector<MachineInstr *, 4> DeadInstructions;
286 LLVM_DEBUG(dbgs() << "Trying to combine: " << MI);
287 if (ArtCombiner.tryCombineInstruction(MI, DeadInsts&: DeadInstructions,
288 WrapperObserver)) {
289 WorkListObserver.printNewInstrs();
290 eraseInstrs(DeadInstrs: DeadInstructions, MRI, LocObserver: &LocObserver);
291 LocObserver.checkpoint(
292 CheckDebugLocs: VerifyDebugLocs ==
293 DebugLocVerifyLevel::LegalizationsAndArtifactCombiners);
294 Changed = true;
295 continue;
296 }
297 // If this was not an artifact (that could be combined away), this might
298 // need special handling. Add it to InstList, so when it's processed
299 // there, it has to be legal or specially handled.
300 else {
301 LLVM_DEBUG(dbgs() << ".. Not combined, moving to instructions list\n");
302 InstList.insert(I: &MI);
303 }
304 }
305 } while (!InstList.empty());
306
307 return {.Changed: Changed, /*FailedOn*/ nullptr};
308}
309
310bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
311 // If the ISel pipeline failed, do not bother running that pass.
312 if (MF.getProperties().hasFailedISel())
313 return false;
314 LLVM_DEBUG(dbgs() << "Legalize Machine IR for: " << MF.getName() << '\n');
315 init(MF);
316 const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
317 GISelCSEAnalysisWrapper &Wrapper =
318 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
319 MachineOptimizationRemarkEmitter MORE(MF, /*MBFI=*/nullptr);
320
321 std::unique_ptr<MachineIRBuilder> MIRBuilder;
322 GISelCSEInfo *CSEInfo = nullptr;
323 bool EnableCSE = EnableCSEInLegalizer.getNumOccurrences()
324 ? EnableCSEInLegalizer
325 : TPC.isGISelCSEEnabled();
326 if (EnableCSE) {
327 MIRBuilder = std::make_unique<CSEMIRBuilder>();
328 CSEInfo = &Wrapper.get(CSEOpt: TPC.getCSEConfig());
329 MIRBuilder->setCSEInfo(CSEInfo);
330 } else
331 MIRBuilder = std::make_unique<MachineIRBuilder>();
332
333 SmallVector<GISelChangeObserver *, 1> AuxObservers;
334 if (EnableCSE && CSEInfo) {
335 // We want CSEInfo in addition to WorkListObserver to observe all changes.
336 AuxObservers.push_back(Elt: CSEInfo);
337 }
338 assert(!CSEInfo || !errorToBool(CSEInfo->verify()));
339 LostDebugLocObserver LocObserver(DEBUG_TYPE);
340 if (VerifyDebugLocs > DebugLocVerifyLevel::None)
341 AuxObservers.push_back(Elt: &LocObserver);
342
343 const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
344
345 const LibcallLoweringInfo &Libcalls =
346 getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
347 M: *MF.getFunction().getParent(), Subtarget);
348
349 // This allows Known Bits Analysis in the legalizer.
350 GISelValueTracking *VT =
351 &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
352
353 const LegalizerInfo &LI = *Subtarget.getLegalizerInfo();
354 MFResult Result = legalizeMachineFunction(MF, LI, AuxObservers, LocObserver,
355 MIRBuilder&: *MIRBuilder, Libcalls: &Libcalls, VT);
356
357 if (Result.FailedOn) {
358 reportGISelFailure(MF, MORE, PassName: "gisel-legalize",
359 Msg: "unable to legalize instruction", MI: *Result.FailedOn);
360 return false;
361 }
362
363 if (LocObserver.getNumLostDebugLocs()) {
364 MachineOptimizationRemarkMissed R("gisel-legalize", "LostDebugLoc",
365 MF.getFunction().getSubprogram(),
366 /*MBB=*/&*MF.begin());
367 R << "lost "
368 << ore::NV("NumLostDebugLocs", LocObserver.getNumLostDebugLocs())
369 << " debug locations during pass";
370 reportGISelWarning(MF, MORE, R);
371 // Example remark:
372 // --- !Missed
373 // Pass: gisel-legalize
374 // Name: GISelFailure
375 // DebugLoc: { File: '.../legalize-urem.mir', Line: 1, Column: 0 }
376 // Function: test_urem_s32
377 // Args:
378 // - String: 'lost '
379 // - NumLostDebugLocs: '1'
380 // - String: ' debug locations during pass'
381 // ...
382 }
383
384 // If for some reason CSE was not enabled, make sure that we invalidate the
385 // CSEInfo object (as we currently declare that the analysis is preserved).
386 // The next time get on the wrapper is called, it will force it to recompute
387 // the analysis.
388 if (!EnableCSE)
389 Wrapper.setComputed(false);
390 return Result.Changed;
391}
392