1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "WebAssembly.h"
10#include "WebAssemblyAsmPrinter.h"
11#include "WebAssemblyExceptionInfo.h"
12#include "WebAssemblyTargetMachine.h"
13#include "llvm/CodeGen/AtomicExpand.h"
14#include "llvm/CodeGen/FuncletLayout.h"
15#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
16#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
17#include "llvm/CodeGen/GlobalISel/Legalizer.h"
18#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
19#include "llvm/CodeGen/IndirectBrExpand.h"
20#include "llvm/CodeGen/MachineBlockPlacement.h"
21#include "llvm/CodeGen/MachineCopyPropagation.h"
22#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
23#include "llvm/CodeGen/PatchableFunction.h"
24#include "llvm/CodeGen/PostRAMachineSink.h"
25#include "llvm/CodeGen/PostRASchedulerList.h"
26#include "llvm/CodeGen/RegisterCoalescerPass.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/CodeGen/UnreachableBlockElim.h"
30#include "llvm/IR/PassInstrumentation.h"
31#include "llvm/MC/MCStreamer.h"
32#include "llvm/Passes/CodeGenPassBuilder.h"
33#include "llvm/Passes/PassBuilder.h"
34#include "llvm/Support/CodeGen.h"
35#include "llvm/Support/Error.h"
36#include "llvm/Target/CGPassBuilderOption.h"
37#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
38#include "llvm/Transforms/Utils/LowerInvoke.h"
39
40using namespace llvm;
41
42namespace WebAssembly {
43extern cl::opt<bool> WasmDisableExplicitLocals;
44extern cl::opt<bool> WasmEnableEH;
45extern cl::opt<bool> WasmEnableEmSjLj;
46extern cl::opt<bool> WasmEnableSjLj;
47} // namespace WebAssembly
48
49using llvm::WebAssembly::WasmDisableExplicitLocals;
50using llvm::WebAssembly::WasmEnableEH;
51using llvm::WebAssembly::WasmEnableEmSjLj;
52using llvm::WebAssembly::WasmEnableSjLj;
53
54namespace {
55
56class WebAssemblyCodeGenPassBuilder : public CodeGenPassBuilder {
57 using Base = CodeGenPassBuilder;
58
59 WebAssemblyTargetMachine &getTM() const {
60 return static_cast<WebAssemblyTargetMachine &>(TM);
61 }
62
63public:
64 explicit WebAssemblyCodeGenPassBuilder(WebAssemblyTargetMachine &TM,
65 const CGPassBuilderOption &Opts,
66 PassInstrumentationCallbacks *PIC)
67 : CodeGenPassBuilder(TM, Opts, PIC) {
68 disablePass<MachineLateInstrsCleanupPass, MachineCopyPropagationPass,
69 PostRAMachineSinkingPass, PostRASchedulerPass,
70 FuncletLayoutPass, StackMapLivenessPass, PatchableFunctionPass,
71 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass,
72 MachineBlockPlacementPass>();
73
74 // Currently RegisterCoalesce degrades wasm debug info quality by a
75 // significant margin. As a quick fix, disable this for -O1, which is often
76 // used for debugging large applications. Disabling this increases code size
77 // of Emscripten core benchmarks by ~5%, which is acceptable for -O1, which
78 // is usually not used for production builds.
79 // TODO Investigate why RegisterCoalesce degrades debug info quality and fix
80 // it properly
81 if (getOptLevel() == CodeGenOptLevel::Less)
82 disablePass<RegisterCoalescerPass>();
83 }
84
85 void addIRPasses(PassManagerWrapper &PMW) override;
86 void addISelPrepare(PassManagerWrapper &PMW) override;
87
88 Error addInstSelector(PassManagerWrapper &PMW) override;
89
90 Error addIRTranslator(PassManagerWrapper &PMW) override;
91 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
92 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
93 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
94 Error addRegBankSelect(PassManagerWrapper &PMW) override;
95 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
96
97 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
98 Expected<bool>
99 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
100 void addPreEmitPass(PassManagerWrapper &PMW) override;
101 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
102 void addAsmPrinter(PassManagerWrapper &PMW) override;
103 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
104};
105
106void WebAssemblyCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
107 // Add signatures to prototype-less function declarations
108 flushFPMsToMPM(PMW);
109 addModulePass(Pass: WebAssemblyAddMissingPrototypesPass(), PMW);
110
111 // Lower .llvm.global_dtors into .llvm.global_ctors with __cxa_atexit calls.
112 addModulePass(Pass: LowerGlobalDtorsPass(), PMW);
113
114 // Fix function bitcasts, as WebAssembly requires caller and callee signatures
115 // to match.
116 addModulePass(Pass: WebAssemblyFixFunctionBitcastsPass(), PMW);
117
118 // Optimize "returned" function attributes.
119 if (getOptLevel() != CodeGenOptLevel::None)
120 addFunctionPass(Pass: WebAssemblyOptimizeReturnedPass(), PMW);
121
122 // If exception handling is not enabled and setjmp/longjmp handling is
123 // enabled, we lower invokes into calls and delete unreachable landingpad
124 // blocks. Lowering invokes when there is no EH support is done in
125 // TargetPassConfig::addPassesToHandleExceptions, but that runs after these IR
126 // passes and Emscripten SjLj handling expects all invokes to be lowered
127 // before.
128 bool EnableEmEH = TM.Options.ExceptionModel == ExceptionHandling::Emscripten;
129 if (!EnableEmEH && !WasmEnableEH) {
130 addFunctionPass(Pass: LowerInvokePass(), PMW);
131 // The lower invoke pass may create unreachable code. Remove it in order not
132 // to process dead blocks in setjmp/longjmp handling.
133 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
134 }
135
136 // Handle exceptions and setjmp/longjmp if enabled. Unlike Wasm EH preparation
137 // done in WasmEHPrepare pass, Wasm SjLj preparation shares libraries and
138 // transformation algorithms with Emscripten SjLj, so we run
139 // LowerEmscriptenEHSjLj pass also when Wasm SjLj is enabled.
140 if (EnableEmEH || WasmEnableEmSjLj || WasmEnableSjLj) {
141 flushFPMsToMPM(PMW);
142 addModulePass(Pass: WebAssemblyLowerEmscriptenEHSjLjPass(EnableEmEH), PMW);
143 }
144
145 // Expand indirectbr instructions to switches.
146 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
147
148 // Try to expand `vecreduce_{and, or}` into `{any, all}_true`.
149 addFunctionPass(Pass: WebAssemblyReduceToAnyAllTruePass(getTM()), PMW);
150
151 Base::addIRPasses(PMW);
152}
153
154void WebAssemblyCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
155 // We need to move reference type allocas to WASM_ADDRESS_SPACE_VAR so that
156 // loads and stores are promoted to local.gets/local.sets.
157 addFunctionPass(Pass: WebAssemblyRefTypeMem2LocalPass(), PMW);
158 // Lower atomics and TLS if necessary
159 flushFPMsToMPM(PMW);
160 addModulePass(Pass: WebAssemblyCoalesceFeaturesAndStripAtomicsPass(getTM()), PMW);
161
162 // This is a no-op if atomics are not used in the module
163 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
164
165 Base::addISelPrepare(PMW);
166}
167
168Error WebAssemblyCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
169 addMachineFunctionPass(Pass: WebAssemblyISelDAGToDAGPass(getTM(), getOptLevel()),
170 PMW);
171
172 // Run the argument-move pass immediately after the ScheduleDAG scheduler
173 // so that we can fix up the ARGUMENT instructions before anything else
174 // sees them in the wrong place.
175 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
176
177 // Set the p2align operands. This information is present during ISel, however
178 // it's inconvenient to collect. Collect it now, and update the immediate
179 // operands.
180 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
181
182 // Eliminate range checks and add default targets to br_table instructions.
183 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
184
185 // unreachable is terminator, non-terminator instruction after it is not
186 // allowed.
187 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
188
189 return Error::success();
190}
191
192Error WebAssemblyCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
193 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
194 return Error::success();
195}
196
197void WebAssemblyCodeGenPassBuilder::addPreLegalizeMachineIR(
198 PassManagerWrapper &PMW) {
199 if (getOptLevel() != CodeGenOptLevel::None)
200 addMachineFunctionPass(Pass: WebAssemblyPreLegalizerCombinerPass(), PMW);
201}
202
203Error WebAssemblyCodeGenPassBuilder::addLegalizeMachineIR(
204 PassManagerWrapper &PMW) {
205 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
206 return Error::success();
207}
208
209void WebAssemblyCodeGenPassBuilder::addPreRegBankSelect(
210 PassManagerWrapper &PMW) {
211 if (getOptLevel() != CodeGenOptLevel::None)
212 addMachineFunctionPass(Pass: WebAssemblyPostLegalizerCombinerPass(), PMW);
213}
214
215Error WebAssemblyCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
216 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
217 return Error::success();
218}
219
220Error WebAssemblyCodeGenPassBuilder::addGlobalInstructionSelect(
221 PassManagerWrapper &PMW) {
222 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
223
224 if (isGlobalISelAbortEnabled()) {
225 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
226 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
227 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
228 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
229 }
230
231 return Error::success();
232}
233
234Error WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteFast(
235 PassManagerWrapper &PMW) {
236 return Error::success();
237}
238
239Expected<bool> WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
240 PassManagerWrapper &PMW) {
241 return false;
242}
243
244void WebAssemblyCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
245 Base::addPreEmitPass(PMW);
246
247 // Nullify DBG_VALUE_LISTs that we cannot handle.
248 addMachineFunctionPass(Pass: WebAssemblyNullifyDebugValueListsPass(), PMW);
249
250 // Remove any unreachable blocks that may be left floating around.
251 // Rare, but possible. Needed for WebAssemblyFixIrreducibleControlFlow.
252 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
253
254 // Eliminate multiple-entry loops.
255 addMachineFunctionPass(Pass: WebAssemblyFixIrreducibleControlFlowPass(), PMW);
256
257 // Do various transformations for exception handling.
258 // Every CFG-changing optimizations should come before this.
259 if (TM.Options.ExceptionModel == ExceptionHandling::Wasm)
260 addMachineFunctionPass(Pass: WebAssemblyLateEHPreparePass(), PMW);
261
262 // Now that we have a prologue and epilogue and all frame indices are
263 // rewritten, eliminate SP and FP. This allows them to be stackified,
264 // colored, and numbered with the rest of the registers.
265 addMachineFunctionPass(Pass: WebAssemblyReplacePhysRegsPass(), PMW);
266
267 // Preparations and optimizations related to register stackification.
268 if (getOptLevel() != CodeGenOptLevel::None) {
269 // Depend on LiveIntervals and perform some optimizations on it.
270 addMachineFunctionPass(Pass: WebAssemblyOptimizeLiveIntervalsPass(), PMW);
271
272 // Prepare memory intrinsic calls for register stackifying.
273 addMachineFunctionPass(Pass: WebAssemblyMemIntrinsicResultsPass(), PMW);
274 }
275
276 // Mark registers as representing wasm's value stack. This is a key
277 // code-compression technique in WebAssembly. We run this pass (and
278 // MemIntrinsicResults above) very late, so that it sees as much code as
279 // possible, including code emitted by PEI and expanded by late tail
280 // duplication.
281 addMachineFunctionPass(Pass: WebAssemblyRegStackifyPass(getOptLevel()), PMW);
282
283 if (getOptLevel() != CodeGenOptLevel::None) {
284 // Run the register coloring pass to reduce the total number of registers.
285 // This runs after stackification so that it doesn't consider registers
286 // that become stackified.
287 addMachineFunctionPass(Pass: WebAssemblyRegColoringPass(), PMW);
288 }
289
290 // Sort the blocks of the CFG into topological order, a prerequisite for
291 // BLOCK and LOOP markers.
292 addMachineFunctionPass(Pass: WebAssemblyCFGSortPass(), PMW);
293
294 // Insert BLOCK and LOOP markers.
295 addMachineFunctionPass(Pass: WebAssemblyCFGStackifyPass(), PMW);
296
297 // Insert explicit local.get and local.set operators.
298 if (!WasmDisableExplicitLocals)
299 addMachineFunctionPass(Pass: WebAssemblyExplicitLocalsPass(), PMW);
300
301 // Lower br_unless into br_if.
302 addMachineFunctionPass(Pass: WebAssemblyLowerBrUnlessPass(), PMW);
303
304 // Perform the very last peephole optimizations on the code.
305 if (getOptLevel() != CodeGenOptLevel::None)
306 addMachineFunctionPass(Pass: WebAssemblyPeepholePass(), PMW);
307
308 // Create a mapping from LLVM CodeGen virtual registers to wasm registers.
309 addMachineFunctionPass(Pass: WebAssemblyRegNumberingPass(), PMW);
310
311 // Fix debug_values whose defs have been stackified.
312 if (!WasmDisableExplicitLocals)
313 addMachineFunctionPass(Pass: WebAssemblyDebugFixupPass(), PMW);
314
315 // Collect information to prepare for MC lowering / asm printing.
316 flushFPMsToMPM(PMW);
317 addModulePass(Pass: WebAssemblyMCLowerPrePass(), PMW);
318}
319
320void WebAssemblyCodeGenPassBuilder::addAsmPrinterBegin(
321 PassManagerWrapper &PMW) {
322 addModulePass(Pass: WebAssemblyAsmPrinterBeginPass(), PMW, /*Force=*/true);
323}
324
325void WebAssemblyCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
326 addMachineFunctionPass(Pass: WebAssemblyAsmPrinterPass(), PMW);
327}
328
329void WebAssemblyCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
330 addModulePass(Pass: WebAssemblyAsmPrinterEndPass(), PMW);
331}
332
333} // namespace
334
335void WebAssemblyTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
336#define GET_PASS_REGISTRY "WebAssemblyPassRegistry.def"
337#include "llvm/Passes/TargetPassRegistry.inc"
338}
339
340Error WebAssemblyTargetMachine::buildCodeGenPipeline(
341 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
342 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
343 const CGPassBuilderOption &Opt, MCContext &Ctx,
344 PassInstrumentationCallbacks *PIC) {
345 auto CGPB = WebAssemblyCodeGenPassBuilder(*this, Opt, PIC);
346 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
347}
348