1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "WebAssembly.h"
10#include "WebAssemblyAsmPrinter.h"
11#include "WebAssemblyExceptionInfo.h"
12#include "WebAssemblyTargetMachine.h"
13#include "llvm/CodeGen/AtomicExpand.h"
14#include "llvm/CodeGen/FuncletLayout.h"
15#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
16#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
17#include "llvm/CodeGen/GlobalISel/Legalizer.h"
18#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
19#include "llvm/CodeGen/IndirectBrExpand.h"
20#include "llvm/CodeGen/MachineBlockPlacement.h"
21#include "llvm/CodeGen/MachineCopyPropagation.h"
22#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
23#include "llvm/CodeGen/PatchableFunction.h"
24#include "llvm/CodeGen/PostRAMachineSink.h"
25#include "llvm/CodeGen/PostRASchedulerList.h"
26#include "llvm/CodeGen/RegisterCoalescerPass.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/CodeGen/UnreachableBlockElim.h"
30#include "llvm/IR/PassInstrumentation.h"
31#include "llvm/MC/MCStreamer.h"
32#include "llvm/Passes/CodeGenPassBuilder.h"
33#include "llvm/Passes/PassBuilder.h"
34#include "llvm/Support/CodeGen.h"
35#include "llvm/Support/Error.h"
36#include "llvm/Target/CGPassBuilderOption.h"
37#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
38#include "llvm/Transforms/Utils/LowerInvoke.h"
39
40using namespace llvm;
41
42namespace WebAssembly {
43extern cl::opt<bool> WasmDisableExplicitLocals;
44extern cl::opt<bool> WasmEnableEH;
45extern cl::opt<bool> WasmEnableEmEH;
46extern cl::opt<bool> WasmEnableEmSjLj;
47extern cl::opt<bool> WasmEnableSjLj;
48} // namespace WebAssembly
49
50using llvm::WebAssembly::WasmDisableExplicitLocals;
51using llvm::WebAssembly::WasmEnableEH;
52using llvm::WebAssembly::WasmEnableEmEH;
53using llvm::WebAssembly::WasmEnableEmSjLj;
54using llvm::WebAssembly::WasmEnableSjLj;
55
56namespace {
57
58class WebAssemblyCodeGenPassBuilder : public CodeGenPassBuilder {
59 using Base = CodeGenPassBuilder;
60
61 WebAssemblyTargetMachine &getTM() const {
62 return static_cast<WebAssemblyTargetMachine &>(TM);
63 }
64
65public:
66 explicit WebAssemblyCodeGenPassBuilder(WebAssemblyTargetMachine &TM,
67 const CGPassBuilderOption &Opts,
68 PassInstrumentationCallbacks *PIC)
69 : CodeGenPassBuilder(TM, Opts, PIC) {
70 disablePass<MachineLateInstrsCleanupPass, MachineCopyPropagationPass,
71 PostRAMachineSinkingPass, PostRASchedulerPass,
72 FuncletLayoutPass, StackMapLivenessPass, PatchableFunctionPass,
73 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass,
74 MachineBlockPlacementPass>();
75
76 // Currently RegisterCoalesce degrades wasm debug info quality by a
77 // significant margin. As a quick fix, disable this for -O1, which is often
78 // used for debugging large applications. Disabling this increases code size
79 // of Emscripten core benchmarks by ~5%, which is acceptable for -O1, which
80 // is usually not used for production builds.
81 // TODO Investigate why RegisterCoalesce degrades debug info quality and fix
82 // it properly
83 if (getOptLevel() == CodeGenOptLevel::Less)
84 disablePass<RegisterCoalescerPass>();
85 }
86
87 void addIRPasses(PassManagerWrapper &PMW) override;
88 void addISelPrepare(PassManagerWrapper &PMW) override;
89
90 Error addInstSelector(PassManagerWrapper &PMW) override;
91
92 Error addIRTranslator(PassManagerWrapper &PMW) override;
93 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
94 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
95 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
96 Error addRegBankSelect(PassManagerWrapper &PMW) override;
97 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
98
99 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
100 Expected<bool>
101 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
102 void addPreEmitPass(PassManagerWrapper &PMW) override;
103 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
104 void addAsmPrinter(PassManagerWrapper &PMW) override;
105 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
106};
107
108void WebAssemblyCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
109 // Add signatures to prototype-less function declarations
110 flushFPMsToMPM(PMW);
111 addModulePass(Pass: WebAssemblyAddMissingPrototypesPass(), PMW);
112
113 // Lower .llvm.global_dtors into .llvm.global_ctors with __cxa_atexit calls.
114 addModulePass(Pass: LowerGlobalDtorsPass(), PMW);
115
116 // Fix function bitcasts, as WebAssembly requires caller and callee signatures
117 // to match.
118 addModulePass(Pass: WebAssemblyFixFunctionBitcastsPass(), PMW);
119
120 // Optimize "returned" function attributes.
121 if (getOptLevel() != CodeGenOptLevel::None)
122 addFunctionPass(Pass: WebAssemblyOptimizeReturnedPass(), PMW);
123
124 // If exception handling is not enabled and setjmp/longjmp handling is
125 // enabled, we lower invokes into calls and delete unreachable landingpad
126 // blocks. Lowering invokes when there is no EH support is done in
127 // TargetPassConfig::addPassesToHandleExceptions, but that runs after these IR
128 // passes and Emscripten SjLj handling expects all invokes to be lowered
129 // before.
130 if (!WasmEnableEmEH && !WasmEnableEH) {
131 addFunctionPass(Pass: LowerInvokePass(), PMW);
132 // The lower invoke pass may create unreachable code. Remove it in order not
133 // to process dead blocks in setjmp/longjmp handling.
134 addFunctionPass(Pass: UnreachableBlockElimPass(), PMW);
135 }
136
137 // Handle exceptions and setjmp/longjmp if enabled. Unlike Wasm EH preparation
138 // done in WasmEHPrepare pass, Wasm SjLj preparation shares libraries and
139 // transformation algorithms with Emscripten SjLj, so we run
140 // LowerEmscriptenEHSjLj pass also when Wasm SjLj is enabled.
141 if (WasmEnableEmEH || WasmEnableEmSjLj || WasmEnableSjLj) {
142 flushFPMsToMPM(PMW);
143 addModulePass(Pass: WebAssemblyLowerEmscriptenEHSjLjPass(), PMW);
144 }
145
146 // Expand indirectbr instructions to switches.
147 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
148
149 // Try to expand `vecreduce_{and, or}` into `{any, all}_true`.
150 addFunctionPass(Pass: WebAssemblyReduceToAnyAllTruePass(getTM()), PMW);
151
152 Base::addIRPasses(PMW);
153}
154
155void WebAssemblyCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
156 // We need to move reference type allocas to WASM_ADDRESS_SPACE_VAR so that
157 // loads and stores are promoted to local.gets/local.sets.
158 addFunctionPass(Pass: WebAssemblyRefTypeMem2LocalPass(), PMW);
159 // Lower atomics and TLS if necessary
160 flushFPMsToMPM(PMW);
161 addModulePass(Pass: WebAssemblyCoalesceFeaturesAndStripAtomicsPass(getTM()), PMW);
162
163 // This is a no-op if atomics are not used in the module
164 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
165
166 Base::addISelPrepare(PMW);
167}
168
169Error WebAssemblyCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
170 addMachineFunctionPass(Pass: WebAssemblyISelDAGToDAGPass(getTM(), getOptLevel()),
171 PMW);
172
173 // Run the argument-move pass immediately after the ScheduleDAG scheduler
174 // so that we can fix up the ARGUMENT instructions before anything else
175 // sees them in the wrong place.
176 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
177
178 // Set the p2align operands. This information is present during ISel, however
179 // it's inconvenient to collect. Collect it now, and update the immediate
180 // operands.
181 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
182
183 // Eliminate range checks and add default targets to br_table instructions.
184 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
185
186 // unreachable is terminator, non-terminator instruction after it is not
187 // allowed.
188 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
189
190 return Error::success();
191}
192
193Error WebAssemblyCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
194 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
195 return Error::success();
196}
197
198void WebAssemblyCodeGenPassBuilder::addPreLegalizeMachineIR(
199 PassManagerWrapper &PMW) {
200 if (getOptLevel() != CodeGenOptLevel::None)
201 addMachineFunctionPass(Pass: WebAssemblyPreLegalizerCombinerPass(), PMW);
202}
203
204Error WebAssemblyCodeGenPassBuilder::addLegalizeMachineIR(
205 PassManagerWrapper &PMW) {
206 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
207 return Error::success();
208}
209
210void WebAssemblyCodeGenPassBuilder::addPreRegBankSelect(
211 PassManagerWrapper &PMW) {
212 if (getOptLevel() != CodeGenOptLevel::None)
213 addMachineFunctionPass(Pass: WebAssemblyPostLegalizerCombinerPass(), PMW);
214}
215
216Error WebAssemblyCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
217 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
218 return Error::success();
219}
220
221Error WebAssemblyCodeGenPassBuilder::addGlobalInstructionSelect(
222 PassManagerWrapper &PMW) {
223 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
224
225 if (isGlobalISelAbortEnabled()) {
226 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
227 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
228 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
229 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
230 }
231
232 return Error::success();
233}
234
235Error WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteFast(
236 PassManagerWrapper &PMW) {
237 return Error::success();
238}
239
240Expected<bool> WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
241 PassManagerWrapper &PMW) {
242 return false;
243}
244
245void WebAssemblyCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
246 Base::addPreEmitPass(PMW);
247
248 // Nullify DBG_VALUE_LISTs that we cannot handle.
249 addMachineFunctionPass(Pass: WebAssemblyNullifyDebugValueListsPass(), PMW);
250
251 // Remove any unreachable blocks that may be left floating around.
252 // Rare, but possible. Needed for WebAssemblyFixIrreducibleControlFlow.
253 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
254
255 // Eliminate multiple-entry loops.
256 addMachineFunctionPass(Pass: WebAssemblyFixIrreducibleControlFlowPass(), PMW);
257
258 // Do various transformations for exception handling.
259 // Every CFG-changing optimizations should come before this.
260 if (TM.Options.ExceptionModel == ExceptionHandling::Wasm)
261 addMachineFunctionPass(Pass: WebAssemblyLateEHPreparePass(), PMW);
262
263 // Now that we have a prologue and epilogue and all frame indices are
264 // rewritten, eliminate SP and FP. This allows them to be stackified,
265 // colored, and numbered with the rest of the registers.
266 addMachineFunctionPass(Pass: WebAssemblyReplacePhysRegsPass(), PMW);
267
268 // Preparations and optimizations related to register stackification.
269 if (getOptLevel() != CodeGenOptLevel::None) {
270 // Depend on LiveIntervals and perform some optimizations on it.
271 addMachineFunctionPass(Pass: WebAssemblyOptimizeLiveIntervalsPass(), PMW);
272
273 // Prepare memory intrinsic calls for register stackifying.
274 addMachineFunctionPass(Pass: WebAssemblyMemIntrinsicResultsPass(), PMW);
275 }
276
277 // Mark registers as representing wasm's value stack. This is a key
278 // code-compression technique in WebAssembly. We run this pass (and
279 // MemIntrinsicResults above) very late, so that it sees as much code as
280 // possible, including code emitted by PEI and expanded by late tail
281 // duplication.
282 addMachineFunctionPass(Pass: WebAssemblyRegStackifyPass(getOptLevel()), PMW);
283
284 if (getOptLevel() != CodeGenOptLevel::None) {
285 // Run the register coloring pass to reduce the total number of registers.
286 // This runs after stackification so that it doesn't consider registers
287 // that become stackified.
288 addMachineFunctionPass(Pass: WebAssemblyRegColoringPass(), PMW);
289 }
290
291 // Sort the blocks of the CFG into topological order, a prerequisite for
292 // BLOCK and LOOP markers.
293 addMachineFunctionPass(Pass: WebAssemblyCFGSortPass(), PMW);
294
295 // Insert BLOCK and LOOP markers.
296 addMachineFunctionPass(Pass: WebAssemblyCFGStackifyPass(), PMW);
297
298 // Insert explicit local.get and local.set operators.
299 if (!WasmDisableExplicitLocals)
300 addMachineFunctionPass(Pass: WebAssemblyExplicitLocalsPass(), PMW);
301
302 // Lower br_unless into br_if.
303 addMachineFunctionPass(Pass: WebAssemblyLowerBrUnlessPass(), PMW);
304
305 // Perform the very last peephole optimizations on the code.
306 if (getOptLevel() != CodeGenOptLevel::None)
307 addMachineFunctionPass(Pass: WebAssemblyPeepholePass(), PMW);
308
309 // Create a mapping from LLVM CodeGen virtual registers to wasm registers.
310 addMachineFunctionPass(Pass: WebAssemblyRegNumberingPass(), PMW);
311
312 // Fix debug_values whose defs have been stackified.
313 if (!WasmDisableExplicitLocals)
314 addMachineFunctionPass(Pass: WebAssemblyDebugFixupPass(), PMW);
315
316 // Collect information to prepare for MC lowering / asm printing.
317 flushFPMsToMPM(PMW);
318 addModulePass(Pass: WebAssemblyMCLowerPrePass(), PMW);
319}
320
321void WebAssemblyCodeGenPassBuilder::addAsmPrinterBegin(
322 PassManagerWrapper &PMW) {
323 addModulePass(Pass: WebAssemblyAsmPrinterBeginPass(), PMW, /*Force=*/true);
324}
325
326void WebAssemblyCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
327 addMachineFunctionPass(Pass: WebAssemblyAsmPrinterPass(), PMW);
328}
329
330void WebAssemblyCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
331 addModulePass(Pass: WebAssemblyAsmPrinterEndPass(), PMW);
332}
333
334} // namespace
335
336void WebAssemblyTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
337#define GET_PASS_REGISTRY "WebAssemblyPassRegistry.def"
338#include "llvm/Passes/TargetPassRegistry.inc"
339}
340
341Error WebAssemblyTargetMachine::buildCodeGenPipeline(
342 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
343 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
344 const CGPassBuilderOption &Opt, MCContext &Ctx,
345 PassInstrumentationCallbacks *PIC) {
346 auto CGPB = WebAssemblyCodeGenPassBuilder(*this, Opt, PIC);
347 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
348}
349