1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "WebAssembly.h"
10#include "WebAssemblyAsmPrinter.h"
11#include "WebAssemblyExceptionInfo.h"
12#include "WebAssemblyTargetMachine.h"
13#include "llvm/CodeGen/AtomicExpand.h"
14#include "llvm/CodeGen/FuncletLayout.h"
15#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
16#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
17#include "llvm/CodeGen/GlobalISel/Legalizer.h"
18#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
19#include "llvm/CodeGen/IndirectBrExpand.h"
20#include "llvm/CodeGen/MachineBlockPlacement.h"
21#include "llvm/CodeGen/MachineCopyPropagation.h"
22#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
23#include "llvm/CodeGen/PatchableFunction.h"
24#include "llvm/CodeGen/PostRAMachineSink.h"
25#include "llvm/CodeGen/PostRASchedulerList.h"
26#include "llvm/CodeGen/RegisterCoalescerPass.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/CodeGen/UnreachableBlockElim.h"
30#include "llvm/CodeGen/WasmEHPrepare.h"
31#include "llvm/IR/PassInstrumentation.h"
32#include "llvm/MC/MCStreamer.h"
33#include "llvm/Passes/CodeGenPassBuilder.h"
34#include "llvm/Passes/PassBuilder.h"
35#include "llvm/Support/CodeGen.h"
36#include "llvm/Support/Error.h"
37#include "llvm/Target/CGPassBuilderOption.h"
38#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
39#include "llvm/Transforms/Utils/LowerInvoke.h"
40
41using namespace llvm;
42
43namespace WebAssembly {
44extern cl::opt<bool> WasmDisableExplicitLocals;
45extern cl::opt<bool> WasmEnableEmSjLj;
46extern cl::opt<bool> WasmEnableSjLj;
47} // namespace WebAssembly
48
49using llvm::WebAssembly::WasmDisableExplicitLocals;
50using llvm::WebAssembly::WasmEnableEmSjLj;
51using llvm::WebAssembly::WasmEnableSjLj;
52
53namespace {
54
55class WebAssemblyCodeGenPassBuilder : public CodeGenPassBuilder {
56 using Base = CodeGenPassBuilder;
57
58 WebAssemblyTargetMachine &getTM() const {
59 return static_cast<WebAssemblyTargetMachine &>(TM);
60 }
61
62public:
63 explicit WebAssemblyCodeGenPassBuilder(WebAssemblyTargetMachine &TM,
64 const CGPassBuilderOption &Opts,
65 PassInstrumentationCallbacks *PIC)
66 : CodeGenPassBuilder(TM, Opts, PIC) {
67 disablePass<MachineLateInstrsCleanupPass, MachineCopyPropagationPass,
68 PostRAMachineSinkingPass, PostRASchedulerPass,
69 FuncletLayoutPass, StackMapLivenessPass, PatchableFunctionPass,
70 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass,
71 MachineBlockPlacementPass>();
72
73 // Currently RegisterCoalesce degrades wasm debug info quality by a
74 // significant margin. As a quick fix, disable this for -O1, which is often
75 // used for debugging large applications. Disabling this increases code size
76 // of Emscripten core benchmarks by ~5%, which is acceptable for -O1, which
77 // is usually not used for production builds.
78 // TODO Investigate why RegisterCoalesce degrades debug info quality and fix
79 // it properly
80 if (getOptLevel() == CodeGenOptLevel::Less)
81 disablePass<RegisterCoalescerPass>();
82 }
83
84 void addIRPasses(PassManagerWrapper &PMW) override;
85 void addISelPrepare(PassManagerWrapper &PMW) override;
86
87 Error addInstSelector(PassManagerWrapper &PMW) override;
88
89 Error addIRTranslator(PassManagerWrapper &PMW) override;
90 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
91 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
92 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
93 Error addRegBankSelect(PassManagerWrapper &PMW) override;
94 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
95
96 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
97 Expected<bool>
98 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
99 void addPreEmitPass(PassManagerWrapper &PMW) override;
100 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
101 void addAsmPrinter(PassManagerWrapper &PMW) override;
102 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
103};
104
105void WebAssemblyCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
106 // Add signatures to prototype-less function declarations
107 flushFPMsToMPM(PMW);
108 addModulePass(Pass: WebAssemblyAddMissingPrototypesPass(), PMW);
109
110 // Lower .llvm.global_dtors into .llvm.global_ctors with __cxa_atexit calls.
111 addModulePass(Pass: LowerGlobalDtorsPass(), PMW);
112
113 // Fix function bitcasts, as WebAssembly requires caller and callee signatures
114 // to match.
115 addModulePass(Pass: WebAssemblyFixFunctionBitcastsPass(), PMW);
116
117 // Optimize "returned" function attributes.
118 if (getOptLevel() != CodeGenOptLevel::None)
119 addFunctionPass(Pass: WebAssemblyOptimizeReturnedPass(), PMW);
120
121 // If exception handling is not enabled and setjmp/longjmp handling is
122 // enabled, we lower invokes into calls and delete unreachable landingpad
123 // blocks. Lowering invokes when there is no EH support is done in
124 // TargetPassConfig::addPassesToHandleExceptions, but that runs after these IR
125 // passes and Emscripten SjLj handling expects all invokes to be lowered
126 // before.
127 bool EnableEmEH = TM.Options.ExceptionModel == ExceptionHandling::Emscripten;
128 bool EnableWasmEH = TM.Options.ExceptionModel == ExceptionHandling::Wasm;
129 if (!EnableEmEH && !EnableWasmEH) {
130 addFunctionPass(Pass: LowerInvokePass(), PMW);
131 // The lower invoke pass may create unreachable code. Remove it in order not
132 // to process dead blocks in setjmp/longjmp handling.
133 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
134 }
135
136 // Handle exceptions and setjmp/longjmp if enabled. Unlike Wasm EH preparation
137 // done in WasmEHPrepare pass, Wasm SjLj preparation shares libraries and
138 // transformation algorithms with Emscripten SjLj, so we run
139 // LowerEmscriptenEHSjLj pass also when Wasm SjLj is enabled.
140 if (EnableEmEH || WasmEnableEmSjLj || WasmEnableSjLj) {
141 flushFPMsToMPM(PMW);
142 addModulePass(Pass: WebAssemblyLowerEmscriptenEHSjLjPass(EnableEmEH), PMW);
143 }
144
145 // Expand indirectbr instructions to switches.
146 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
147
148 // Try to expand `vecreduce_{and, or}` into `{any, all}_true`.
149 addFunctionPass(Pass: WebAssemblyReduceToAnyAllTruePass(getTM()), PMW);
150
151 Base::addIRPasses(PMW);
152}
153
154void WebAssemblyCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
155 if (TM.Options.ExceptionModel == ExceptionHandling::Wasm)
156 addFunctionPass(Pass: WasmEHPreparePass(), PMW);
157
158 // We need to move reference type allocas to WASM_ADDRESS_SPACE_VAR so that
159 // loads and stores are promoted to local.gets/local.sets.
160 addFunctionPass(Pass: WebAssemblyRefTypeMem2LocalPass(), PMW);
161 // Lower atomics and TLS if necessary
162 flushFPMsToMPM(PMW);
163 addModulePass(Pass: WebAssemblyCoalesceFeaturesAndStripAtomicsPass(getTM()), PMW);
164
165 // This is a no-op if atomics are not used in the module
166 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
167
168 Base::addISelPrepare(PMW);
169}
170
171Error WebAssemblyCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
172 addMachineFunctionPass(Pass: WebAssemblyISelDAGToDAGPass(getTM(), getOptLevel()),
173 PMW);
174
175 // Run the argument-move pass immediately after the ScheduleDAG scheduler
176 // so that we can fix up the ARGUMENT instructions before anything else
177 // sees them in the wrong place.
178 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
179
180 // Set the p2align operands. This information is present during ISel, however
181 // it's inconvenient to collect. Collect it now, and update the immediate
182 // operands.
183 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
184
185 // Eliminate range checks and add default targets to br_table instructions.
186 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
187
188 // unreachable is terminator, non-terminator instruction after it is not
189 // allowed.
190 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
191
192 return Error::success();
193}
194
195Error WebAssemblyCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
196 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
197 return Error::success();
198}
199
200void WebAssemblyCodeGenPassBuilder::addPreLegalizeMachineIR(
201 PassManagerWrapper &PMW) {
202 if (getOptLevel() != CodeGenOptLevel::None)
203 addMachineFunctionPass(Pass: WebAssemblyPreLegalizerCombinerPass(), PMW);
204}
205
206Error WebAssemblyCodeGenPassBuilder::addLegalizeMachineIR(
207 PassManagerWrapper &PMW) {
208 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
209 return Error::success();
210}
211
212void WebAssemblyCodeGenPassBuilder::addPreRegBankSelect(
213 PassManagerWrapper &PMW) {
214 if (getOptLevel() != CodeGenOptLevel::None)
215 addMachineFunctionPass(Pass: WebAssemblyPostLegalizerCombinerPass(), PMW);
216}
217
218Error WebAssemblyCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
219 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
220 return Error::success();
221}
222
223Error WebAssemblyCodeGenPassBuilder::addGlobalInstructionSelect(
224 PassManagerWrapper &PMW) {
225 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
226
227 if (isGlobalISelAbortEnabled()) {
228 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
229 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
230 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
231 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
232 }
233
234 return Error::success();
235}
236
237Error WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteFast(
238 PassManagerWrapper &PMW) {
239 return Error::success();
240}
241
242Expected<bool> WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
243 PassManagerWrapper &PMW) {
244 return false;
245}
246
247void WebAssemblyCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
248 Base::addPreEmitPass(PMW);
249
250 // Nullify DBG_VALUE_LISTs that we cannot handle.
251 addMachineFunctionPass(Pass: WebAssemblyNullifyDebugValueListsPass(), PMW);
252
253 // Remove any unreachable blocks that may be left floating around.
254 // Rare, but possible. Needed for WebAssemblyFixIrreducibleControlFlow.
255 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
256
257 // Eliminate multiple-entry loops.
258 addMachineFunctionPass(Pass: WebAssemblyFixIrreducibleControlFlowPass(), PMW);
259
260 // Do various transformations for exception handling.
261 // Every CFG-changing optimizations should come before this.
262 if (TM.Options.ExceptionModel == ExceptionHandling::Wasm)
263 addMachineFunctionPass(Pass: WebAssemblyLateEHPreparePass(), PMW);
264
265 // Now that we have a prologue and epilogue and all frame indices are
266 // rewritten, eliminate SP and FP. This allows them to be stackified,
267 // colored, and numbered with the rest of the registers.
268 addMachineFunctionPass(Pass: WebAssemblyReplacePhysRegsPass(), PMW);
269
270 // Preparations and optimizations related to register stackification.
271 if (getOptLevel() != CodeGenOptLevel::None) {
272 // Depend on LiveIntervals and perform some optimizations on it.
273 addMachineFunctionPass(Pass: WebAssemblyOptimizeLiveIntervalsPass(), PMW);
274
275 // Prepare memory intrinsic calls for register stackifying.
276 addMachineFunctionPass(Pass: WebAssemblyMemIntrinsicResultsPass(), PMW);
277 }
278
279 // Mark registers as representing wasm's value stack. This is a key
280 // code-compression technique in WebAssembly. We run this pass (and
281 // MemIntrinsicResults above) very late, so that it sees as much code as
282 // possible, including code emitted by PEI and expanded by late tail
283 // duplication.
284 addMachineFunctionPass(Pass: WebAssemblyRegStackifyPass(getOptLevel()), PMW);
285
286 if (getOptLevel() != CodeGenOptLevel::None) {
287 // Run the register coloring pass to reduce the total number of registers.
288 // This runs after stackification so that it doesn't consider registers
289 // that become stackified.
290 addMachineFunctionPass(Pass: WebAssemblyRegColoringPass(), PMW);
291 }
292
293 // Sort the blocks of the CFG into topological order, a prerequisite for
294 // BLOCK and LOOP markers.
295 addMachineFunctionPass(Pass: WebAssemblyCFGSortPass(), PMW);
296
297 // Insert BLOCK and LOOP markers.
298 addMachineFunctionPass(Pass: WebAssemblyCFGStackifyPass(), PMW);
299
300 // Insert explicit local.get and local.set operators.
301 if (!WasmDisableExplicitLocals)
302 addMachineFunctionPass(Pass: WebAssemblyExplicitLocalsPass(), PMW);
303
304 // Lower br_unless into br_if.
305 addMachineFunctionPass(Pass: WebAssemblyLowerBrUnlessPass(), PMW);
306
307 // Perform the very last peephole optimizations on the code.
308 if (getOptLevel() != CodeGenOptLevel::None)
309 addMachineFunctionPass(Pass: WebAssemblyPeepholePass(), PMW);
310
311 // Create a mapping from LLVM CodeGen virtual registers to wasm registers.
312 addMachineFunctionPass(Pass: WebAssemblyRegNumberingPass(), PMW);
313
314 // Fix debug_values whose defs have been stackified.
315 if (!WasmDisableExplicitLocals)
316 addMachineFunctionPass(Pass: WebAssemblyDebugFixupPass(), PMW);
317
318 // Collect information to prepare for MC lowering / asm printing.
319 flushFPMsToMPM(PMW);
320 addModulePass(Pass: WebAssemblyMCLowerPrePass(), PMW);
321}
322
323void WebAssemblyCodeGenPassBuilder::addAsmPrinterBegin(
324 PassManagerWrapper &PMW) {
325 addModulePass(Pass: WebAssemblyAsmPrinterBeginPass(), PMW, /*Force=*/true);
326}
327
328void WebAssemblyCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
329 addMachineFunctionPass(Pass: WebAssemblyAsmPrinterPass(), PMW);
330}
331
332void WebAssemblyCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
333 addModulePass(Pass: WebAssemblyAsmPrinterEndPass(), PMW);
334}
335
336} // namespace
337
338void WebAssemblyTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
339#define GET_PASS_REGISTRY "WebAssemblyPassRegistry.def"
340#include "llvm/Passes/TargetPassRegistry.inc"
341}
342
343Error WebAssemblyTargetMachine::buildCodeGenPipeline(
344 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
345 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
346 const CGPassBuilderOption &Opt, MCContext &Ctx,
347 PassInstrumentationCallbacks *PIC) {
348 auto CGPB = WebAssemblyCodeGenPassBuilder(*this, Opt, PIC);
349 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
350}
351