1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "WebAssembly.h"
10#include "WebAssemblyAsmPrinter.h"
11#include "WebAssemblyExceptionInfo.h"
12#include "WebAssemblyTargetMachine.h"
13#include "llvm/CodeGen/AtomicExpand.h"
14#include "llvm/CodeGen/FuncletLayout.h"
15#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
16#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
17#include "llvm/CodeGen/GlobalISel/Legalizer.h"
18#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
19#include "llvm/CodeGen/IndirectBrExpand.h"
20#include "llvm/CodeGen/MachineBlockPlacement.h"
21#include "llvm/CodeGen/MachineCopyPropagation.h"
22#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
23#include "llvm/CodeGen/PatchableFunction.h"
24#include "llvm/CodeGen/PostRAMachineSink.h"
25#include "llvm/CodeGen/PostRASchedulerList.h"
26#include "llvm/CodeGen/RegisterCoalescerPass.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/CodeGen/UnreachableBlockElim.h"
30#include "llvm/IR/PassInstrumentation.h"
31#include "llvm/MC/MCStreamer.h"
32#include "llvm/Passes/CodeGenPassBuilder.h"
33#include "llvm/Passes/PassBuilder.h"
34#include "llvm/Support/CodeGen.h"
35#include "llvm/Support/Error.h"
36#include "llvm/Target/CGPassBuilderOption.h"
37#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
38
39using namespace llvm;
40
41namespace WebAssembly {
42extern cl::opt<bool> WasmDisableExplicitLocals;
43} // namespace WebAssembly
44
45using llvm::WebAssembly::WasmDisableExplicitLocals;
46
47namespace {
48
49class WebAssemblyCodeGenPassBuilder : public CodeGenPassBuilder {
50 using Base = CodeGenPassBuilder;
51
52 WebAssemblyTargetMachine &getTM() const {
53 return static_cast<WebAssemblyTargetMachine &>(TM);
54 }
55
56public:
57 explicit WebAssemblyCodeGenPassBuilder(WebAssemblyTargetMachine &TM,
58 const CGPassBuilderOption &Opts,
59 PassInstrumentationCallbacks *PIC)
60 : CodeGenPassBuilder(TM, Opts, PIC) {
61 disablePass<MachineLateInstrsCleanupPass, MachineCopyPropagationPass,
62 PostRAMachineSinkingPass, PostRASchedulerPass,
63 FuncletLayoutPass, StackMapLivenessPass, PatchableFunctionPass,
64 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass,
65 MachineBlockPlacementPass>();
66
67 // Currently RegisterCoalesce degrades wasm debug info quality by a
68 // significant margin. As a quick fix, disable this for -O1, which is often
69 // used for debugging large applications. Disabling this increases code size
70 // of Emscripten core benchmarks by ~5%, which is acceptable for -O1, which
71 // is usually not used for production builds.
72 // TODO Investigate why RegisterCoalesce degrades debug info quality and fix
73 // it properly
74 if (getOptLevel() == CodeGenOptLevel::Less)
75 disablePass<RegisterCoalescerPass>();
76 }
77
78 void addIRPasses(PassManagerWrapper &PMW) override;
79 void addISelPrepare(PassManagerWrapper &PMW) override;
80
81 Error addInstSelector(PassManagerWrapper &PMW) override;
82
83 Error addIRTranslator(PassManagerWrapper &PMW) override;
84 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
85 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
86 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
87 Error addRegBankSelect(PassManagerWrapper &PMW) override;
88 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
89
90 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
91 Expected<bool>
92 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
93 void addPreEmitPass(PassManagerWrapper &PMW) override;
94 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
95 void addAsmPrinter(PassManagerWrapper &PMW) override;
96 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
97};
98
99void WebAssemblyCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
100 // Add signatures to prototype-less function declarations
101 flushFPMsToMPM(PMW);
102 addModulePass(Pass: WebAssemblyAddMissingPrototypesPass(), PMW);
103
104 // Lower .llvm.global_dtors into .llvm.global_ctors with __cxa_atexit calls.
105 addModulePass(Pass: LowerGlobalDtorsPass(), PMW);
106
107 // Fix function bitcasts, as WebAssembly requires caller and callee signatures
108 // to match.
109 addModulePass(Pass: WebAssemblyFixFunctionBitcastsPass(), PMW);
110
111 // Optimize "returned" function attributes.
112 if (getOptLevel() != CodeGenOptLevel::None)
113 addFunctionPass(Pass: WebAssemblyOptimizeReturnedPass(), PMW);
114
115 // Wasm SjLj shares Emscripten SjLj's transformation, so it is handled here.
116 // Without Wasm or Emscripten EH, this also lowers invokes.
117 flushFPMsToMPM(PMW);
118 addModulePass(Pass: WebAssemblyLowerEmscriptenEHSjLjPass(), PMW);
119
120 // Expand indirectbr instructions to switches.
121 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
122
123 // Try to expand `vecreduce_{and, or}` into `{any, all}_true`.
124 addFunctionPass(Pass: WebAssemblyReduceToAnyAllTruePass(getTM()), PMW);
125
126 Base::addIRPasses(PMW);
127}
128
129void WebAssemblyCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
130 // We need to move reference type allocas to WASM_ADDRESS_SPACE_VAR so that
131 // loads and stores are promoted to local.gets/local.sets.
132 addFunctionPass(Pass: WebAssemblyRefTypeMem2LocalPass(), PMW);
133 // Lower atomics and TLS if necessary
134 flushFPMsToMPM(PMW);
135 addModulePass(Pass: WebAssemblyCoalesceFeaturesAndStripAtomicsPass(getTM()), PMW);
136
137 // This is a no-op if atomics are not used in the module
138 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
139
140 Base::addISelPrepare(PMW);
141}
142
143Error WebAssemblyCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
144 addMachineFunctionPass(Pass: WebAssemblyISelDAGToDAGPass(getTM(), getOptLevel()),
145 PMW);
146
147 // Run the argument-move pass immediately after the ScheduleDAG scheduler
148 // so that we can fix up the ARGUMENT instructions before anything else
149 // sees them in the wrong place.
150 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
151
152 // Set the p2align operands. This information is present during ISel, however
153 // it's inconvenient to collect. Collect it now, and update the immediate
154 // operands.
155 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
156
157 // Eliminate range checks and add default targets to br_table instructions.
158 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
159
160 // unreachable is terminator, non-terminator instruction after it is not
161 // allowed.
162 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
163
164 return Error::success();
165}
166
167Error WebAssemblyCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
168 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
169 return Error::success();
170}
171
172void WebAssemblyCodeGenPassBuilder::addPreLegalizeMachineIR(
173 PassManagerWrapper &PMW) {
174 if (getOptLevel() != CodeGenOptLevel::None)
175 addMachineFunctionPass(Pass: WebAssemblyPreLegalizerCombinerPass(), PMW);
176}
177
178Error WebAssemblyCodeGenPassBuilder::addLegalizeMachineIR(
179 PassManagerWrapper &PMW) {
180 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
181 return Error::success();
182}
183
184void WebAssemblyCodeGenPassBuilder::addPreRegBankSelect(
185 PassManagerWrapper &PMW) {
186 if (getOptLevel() != CodeGenOptLevel::None)
187 addMachineFunctionPass(Pass: WebAssemblyPostLegalizerCombinerPass(), PMW);
188}
189
190Error WebAssemblyCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
191 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
192 return Error::success();
193}
194
195Error WebAssemblyCodeGenPassBuilder::addGlobalInstructionSelect(
196 PassManagerWrapper &PMW) {
197 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
198
199 if (isGlobalISelAbortEnabled()) {
200 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
201 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
202 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
203 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
204 }
205
206 return Error::success();
207}
208
209Error WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteFast(
210 PassManagerWrapper &PMW) {
211 return Error::success();
212}
213
214Expected<bool> WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
215 PassManagerWrapper &PMW) {
216 return false;
217}
218
219void WebAssemblyCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
220 Base::addPreEmitPass(PMW);
221
222 // Nullify DBG_VALUE_LISTs that we cannot handle.
223 addMachineFunctionPass(Pass: WebAssemblyNullifyDebugValueListsPass(), PMW);
224
225 // Remove any unreachable blocks that may be left floating around.
226 // Rare, but possible. Needed for WebAssemblyFixIrreducibleControlFlow.
227 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
228
229 // Eliminate multiple-entry loops.
230 addMachineFunctionPass(Pass: WebAssemblyFixIrreducibleControlFlowPass(), PMW);
231
232 // Do various transformations for exception handling.
233 // Every CFG-changing optimizations should come before this.
234 addMachineFunctionPass(Pass: WebAssemblyLateEHPreparePass(), PMW);
235
236 // Now that we have a prologue and epilogue and all frame indices are
237 // rewritten, eliminate SP and FP. This allows them to be stackified,
238 // colored, and numbered with the rest of the registers.
239 addMachineFunctionPass(Pass: WebAssemblyReplacePhysRegsPass(), PMW);
240
241 // Preparations and optimizations related to register stackification.
242 if (getOptLevel() != CodeGenOptLevel::None) {
243 // Depend on LiveIntervals and perform some optimizations on it.
244 addMachineFunctionPass(Pass: WebAssemblyOptimizeLiveIntervalsPass(), PMW);
245
246 // Prepare memory intrinsic calls for register stackifying.
247 addMachineFunctionPass(Pass: WebAssemblyMemIntrinsicResultsPass(), PMW);
248 }
249
250 // Mark registers as representing wasm's value stack. This is a key
251 // code-compression technique in WebAssembly. We run this pass (and
252 // MemIntrinsicResults above) very late, so that it sees as much code as
253 // possible, including code emitted by PEI and expanded by late tail
254 // duplication.
255 addMachineFunctionPass(Pass: WebAssemblyRegStackifyPass(getOptLevel()), PMW);
256
257 if (getOptLevel() != CodeGenOptLevel::None) {
258 // Run the register coloring pass to reduce the total number of registers.
259 // This runs after stackification so that it doesn't consider registers
260 // that become stackified.
261 addMachineFunctionPass(Pass: WebAssemblyRegColoringPass(), PMW);
262 }
263
264 // Sort the blocks of the CFG into topological order, a prerequisite for
265 // BLOCK and LOOP markers.
266 addMachineFunctionPass(Pass: WebAssemblyCFGSortPass(), PMW);
267
268 // Insert BLOCK and LOOP markers.
269 addMachineFunctionPass(Pass: WebAssemblyCFGStackifyPass(), PMW);
270
271 // Insert explicit local.get and local.set operators.
272 if (!WasmDisableExplicitLocals)
273 addMachineFunctionPass(Pass: WebAssemblyExplicitLocalsPass(), PMW);
274
275 // Lower br_unless into br_if.
276 addMachineFunctionPass(Pass: WebAssemblyLowerBrUnlessPass(), PMW);
277
278 // Perform the very last peephole optimizations on the code.
279 if (getOptLevel() != CodeGenOptLevel::None)
280 addMachineFunctionPass(Pass: WebAssemblyPeepholePass(), PMW);
281
282 // Create a mapping from LLVM CodeGen virtual registers to wasm registers.
283 addMachineFunctionPass(Pass: WebAssemblyRegNumberingPass(), PMW);
284
285 // Fix debug_values whose defs have been stackified.
286 if (!WasmDisableExplicitLocals)
287 addMachineFunctionPass(Pass: WebAssemblyDebugFixupPass(), PMW);
288
289 // Collect information to prepare for MC lowering / asm printing.
290 flushFPMsToMPM(PMW);
291 addModulePass(Pass: WebAssemblyMCLowerPrePass(), PMW);
292}
293
294void WebAssemblyCodeGenPassBuilder::addAsmPrinterBegin(
295 PassManagerWrapper &PMW) {
296 addModulePass(Pass: WebAssemblyAsmPrinterBeginPass(), PMW, /*Force=*/true);
297}
298
299void WebAssemblyCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
300 addMachineFunctionPass(Pass: WebAssemblyAsmPrinterPass(), PMW);
301}
302
303void WebAssemblyCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
304 addModulePass(Pass: WebAssemblyAsmPrinterEndPass(), PMW);
305}
306
307} // namespace
308
309void WebAssemblyTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
310#define GET_PASS_REGISTRY "WebAssemblyPassRegistry.def"
311#include "llvm/Passes/TargetPassRegistry.inc"
312}
313
314Error WebAssemblyTargetMachine::buildCodeGenPipeline(
315 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
316 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
317 const CGPassBuilderOption &Opt, MCContext &Ctx,
318 PassInstrumentationCallbacks *PIC) {
319 auto CGPB = WebAssemblyCodeGenPassBuilder(*this, Opt, PIC);
320 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
321}
322