1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "WebAssembly.h"
10#include "WebAssemblyAsmPrinter.h"
11#include "WebAssemblyExceptionInfo.h"
12#include "WebAssemblyTargetMachine.h"
13#include "llvm/CodeGen/AtomicExpand.h"
14#include "llvm/CodeGen/FuncletLayout.h"
15#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
16#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
17#include "llvm/CodeGen/GlobalISel/Legalizer.h"
18#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
19#include "llvm/CodeGen/IndirectBrExpand.h"
20#include "llvm/CodeGen/MachineBlockPlacement.h"
21#include "llvm/CodeGen/MachineCopyPropagation.h"
22#include "llvm/CodeGen/MachineLateInstrsCleanup.h"
23#include "llvm/CodeGen/PatchableFunction.h"
24#include "llvm/CodeGen/PostRAMachineSink.h"
25#include "llvm/CodeGen/PostRASchedulerList.h"
26#include "llvm/CodeGen/RegisterCoalescerPass.h"
27#include "llvm/CodeGen/RemoveLoadsIntoFakeUses.h"
28#include "llvm/CodeGen/ShrinkWrap.h"
29#include "llvm/CodeGen/UnreachableBlockElim.h"
30#include "llvm/CodeGen/WasmEHPrepare.h"
31#include "llvm/CodeGen/WinEHPrepare.h"
32#include "llvm/IR/PassInstrumentation.h"
33#include "llvm/MC/MCStreamer.h"
34#include "llvm/Passes/CodeGenPassBuilder.h"
35#include "llvm/Passes/PassBuilder.h"
36#include "llvm/Support/CodeGen.h"
37#include "llvm/Support/Error.h"
38#include "llvm/Target/CGPassBuilderOption.h"
39#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
40
41using namespace llvm;
42
43namespace WebAssembly {
44extern cl::opt<bool> WasmDisableExplicitLocals;
45} // namespace WebAssembly
46
47using llvm::WebAssembly::WasmDisableExplicitLocals;
48
49namespace {
50
51class WebAssemblyCodeGenPassBuilder : public CodeGenPassBuilder {
52 using Base = CodeGenPassBuilder;
53
54 WebAssemblyTargetMachine &getTM() const {
55 return static_cast<WebAssemblyTargetMachine &>(TM);
56 }
57
58public:
59 explicit WebAssemblyCodeGenPassBuilder(WebAssemblyTargetMachine &TM,
60 const CGPassBuilderOption &Opts,
61 PassInstrumentationCallbacks *PIC)
62 : CodeGenPassBuilder(TM, Opts, PIC) {
63 disablePass<MachineLateInstrsCleanupPass, MachineCopyPropagationPass,
64 PostRAMachineSinkingPass, PostRASchedulerPass,
65 FuncletLayoutPass, StackMapLivenessPass, PatchableFunctionPass,
66 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass,
67 MachineBlockPlacementPass>();
68
69 // Currently RegisterCoalesce degrades wasm debug info quality by a
70 // significant margin. As a quick fix, disable this for -O1, which is often
71 // used for debugging large applications. Disabling this increases code size
72 // of Emscripten core benchmarks by ~5%, which is acceptable for -O1, which
73 // is usually not used for production builds.
74 // TODO Investigate why RegisterCoalesce degrades debug info quality and fix
75 // it properly
76 if (getOptLevel() == CodeGenOptLevel::Less)
77 disablePass<RegisterCoalescerPass>();
78 }
79
80 void addIRPasses(PassManagerWrapper &PMW) override;
81 void addPassesToHandleExceptions(PassManagerWrapper &PMW) override;
82 void addISelPrepare(PassManagerWrapper &PMW) override;
83
84 Error addInstSelector(PassManagerWrapper &PMW) override;
85
86 Error addIRTranslator(PassManagerWrapper &PMW) override;
87 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
88 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
89 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
90 Error addRegBankSelect(PassManagerWrapper &PMW) override;
91 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
92
93 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
94 Expected<bool>
95 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
96 void addPreEmitPass(PassManagerWrapper &PMW) override;
97 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
98 void addAsmPrinter(PassManagerWrapper &PMW) override;
99 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
100};
101
102void WebAssemblyCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
103 // Add signatures to prototype-less function declarations
104 flushFPMsToMPM(PMW);
105 addModulePass(Pass: WebAssemblyAddMissingPrototypesPass(), PMW);
106
107 // Lower .llvm.global_dtors into .llvm.global_ctors with __cxa_atexit calls.
108 addModulePass(Pass: LowerGlobalDtorsPass(), PMW);
109
110 // Fix function bitcasts, as WebAssembly requires caller and callee signatures
111 // to match.
112 addModulePass(Pass: WebAssemblyFixFunctionBitcastsPass(), PMW);
113
114 // Optimize "returned" function attributes.
115 if (getOptLevel() != CodeGenOptLevel::None)
116 addFunctionPass(Pass: WebAssemblyOptimizeReturnedPass(), PMW);
117
118 // Wasm SjLj shares Emscripten SjLj's transformation, so it is handled here.
119 // Without Wasm or Emscripten EH, this also lowers invokes.
120 flushFPMsToMPM(PMW);
121 addModulePass(Pass: WebAssemblyLowerEmscriptenEHSjLjPass(), PMW);
122
123 // Expand indirectbr instructions to switches.
124 addFunctionPass(Pass: IndirectBrExpandPass(TM), PMW);
125
126 // Try to expand `vecreduce_{and, or}` into `{any, all}_true`.
127 addFunctionPass(Pass: WebAssemblyReduceToAnyAllTruePass(getTM()), PMW);
128
129 Base::addIRPasses(PMW);
130}
131
132void WebAssemblyCodeGenPassBuilder::addPassesToHandleExceptions(
133 PassManagerWrapper &PMW) {
134 // Wasm EH needs WinEHPrepare to remove the catchswitch PHIs SelectionDAG
135 // cannot lower.
136 addFunctionPass(Pass: WinEHPreparePass(), PMW);
137 addFunctionPass(Pass: WasmEHPreparePass(), PMW);
138}
139
140void WebAssemblyCodeGenPassBuilder::addISelPrepare(PassManagerWrapper &PMW) {
141 // We need to move reference type allocas to WASM_ADDRESS_SPACE_VAR so that
142 // loads and stores are promoted to local.gets/local.sets.
143 addFunctionPass(Pass: WebAssemblyRefTypeMem2LocalPass(), PMW);
144 // Lower atomics and TLS if necessary
145 flushFPMsToMPM(PMW);
146 addModulePass(Pass: WebAssemblyCoalesceFeaturesAndStripAtomicsPass(getTM()), PMW);
147
148 // This is a no-op if atomics are not used in the module
149 addFunctionPass(Pass: AtomicExpandPass(TM), PMW);
150
151 Base::addISelPrepare(PMW);
152}
153
154Error WebAssemblyCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
155 addMachineFunctionPass(Pass: WebAssemblyISelDAGToDAGPass(getTM(), getOptLevel()),
156 PMW);
157
158 // Run the argument-move pass immediately after the ScheduleDAG scheduler
159 // so that we can fix up the ARGUMENT instructions before anything else
160 // sees them in the wrong place.
161 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
162
163 // Set the p2align operands. This information is present during ISel, however
164 // it's inconvenient to collect. Collect it now, and update the immediate
165 // operands.
166 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
167
168 // Eliminate range checks and add default targets to br_table instructions.
169 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
170
171 // unreachable is terminator, non-terminator instruction after it is not
172 // allowed.
173 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
174
175 return Error::success();
176}
177
178Error WebAssemblyCodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
179 addMachineFunctionPass(Pass: IRTranslatorPass(getOptLevel()), PMW);
180 return Error::success();
181}
182
183void WebAssemblyCodeGenPassBuilder::addPreLegalizeMachineIR(
184 PassManagerWrapper &PMW) {
185 if (getOptLevel() != CodeGenOptLevel::None)
186 addMachineFunctionPass(Pass: WebAssemblyPreLegalizerCombinerPass(), PMW);
187}
188
189Error WebAssemblyCodeGenPassBuilder::addLegalizeMachineIR(
190 PassManagerWrapper &PMW) {
191 addMachineFunctionPass(Pass: LegalizerPass(), PMW);
192 return Error::success();
193}
194
195void WebAssemblyCodeGenPassBuilder::addPreRegBankSelect(
196 PassManagerWrapper &PMW) {
197 if (getOptLevel() != CodeGenOptLevel::None)
198 addMachineFunctionPass(Pass: WebAssemblyPostLegalizerCombinerPass(), PMW);
199}
200
201Error WebAssemblyCodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
202 addMachineFunctionPass(Pass: RegBankSelectPass(), PMW);
203 return Error::success();
204}
205
206Error WebAssemblyCodeGenPassBuilder::addGlobalInstructionSelect(
207 PassManagerWrapper &PMW) {
208 addMachineFunctionPass(Pass: InstructionSelectPass(getOptLevel()), PMW);
209
210 if (isGlobalISelAbortEnabled()) {
211 addMachineFunctionPass(Pass: WebAssemblyArgumentMovePass(), PMW);
212 addMachineFunctionPass(Pass: WebAssemblySetP2AlignOperandsPass(), PMW);
213 addMachineFunctionPass(Pass: WebAssemblyFixBrTableDefaultsPass(), PMW);
214 addMachineFunctionPass(Pass: WebAssemblyCleanCodeAfterTrapPass(), PMW);
215 }
216
217 return Error::success();
218}
219
220Error WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteFast(
221 PassManagerWrapper &PMW) {
222 return Error::success();
223}
224
225Expected<bool> WebAssemblyCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
226 PassManagerWrapper &PMW) {
227 return false;
228}
229
230void WebAssemblyCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
231 Base::addPreEmitPass(PMW);
232
233 // Nullify DBG_VALUE_LISTs that we cannot handle.
234 addMachineFunctionPass(Pass: WebAssemblyNullifyDebugValueListsPass(), PMW);
235
236 // Remove any unreachable blocks that may be left floating around.
237 // Rare, but possible. Needed for WebAssemblyFixIrreducibleControlFlow.
238 addMachineFunctionPass(Pass: UnreachableMachineBlockElimPass(), PMW);
239
240 // Eliminate multiple-entry loops.
241 addMachineFunctionPass(Pass: WebAssemblyFixIrreducibleControlFlowPass(), PMW);
242
243 // Do various transformations for exception handling.
244 // Every CFG-changing optimizations should come before this.
245 addMachineFunctionPass(Pass: WebAssemblyLateEHPreparePass(), PMW);
246
247 // Now that we have a prologue and epilogue and all frame indices are
248 // rewritten, eliminate SP and FP. This allows them to be stackified,
249 // colored, and numbered with the rest of the registers.
250 addMachineFunctionPass(Pass: WebAssemblyReplacePhysRegsPass(), PMW);
251
252 // Preparations and optimizations related to register stackification.
253 if (getOptLevel() != CodeGenOptLevel::None) {
254 // Depend on LiveIntervals and perform some optimizations on it.
255 addMachineFunctionPass(Pass: WebAssemblyOptimizeLiveIntervalsPass(), PMW);
256
257 // Prepare memory intrinsic calls for register stackifying.
258 addMachineFunctionPass(Pass: WebAssemblyMemIntrinsicResultsPass(), PMW);
259 }
260
261 // Mark registers as representing wasm's value stack. This is a key
262 // code-compression technique in WebAssembly. We run this pass (and
263 // MemIntrinsicResults above) very late, so that it sees as much code as
264 // possible, including code emitted by PEI and expanded by late tail
265 // duplication.
266 addMachineFunctionPass(Pass: WebAssemblyRegStackifyPass(getOptLevel()), PMW);
267
268 if (getOptLevel() != CodeGenOptLevel::None) {
269 // Run the register coloring pass to reduce the total number of registers.
270 // This runs after stackification so that it doesn't consider registers
271 // that become stackified.
272 addMachineFunctionPass(Pass: WebAssemblyRegColoringPass(), PMW);
273 }
274
275 // Sort the blocks of the CFG into topological order, a prerequisite for
276 // BLOCK and LOOP markers.
277 addMachineFunctionPass(Pass: WebAssemblyCFGSortPass(), PMW);
278
279 // Insert BLOCK and LOOP markers.
280 addMachineFunctionPass(Pass: WebAssemblyCFGStackifyPass(), PMW);
281
282 // Insert explicit local.get and local.set operators.
283 if (!WasmDisableExplicitLocals)
284 addMachineFunctionPass(Pass: WebAssemblyExplicitLocalsPass(), PMW);
285
286 // Lower br_unless into br_if.
287 addMachineFunctionPass(Pass: WebAssemblyLowerBrUnlessPass(), PMW);
288
289 // Perform the very last peephole optimizations on the code.
290 if (getOptLevel() != CodeGenOptLevel::None)
291 addMachineFunctionPass(Pass: WebAssemblyPeepholePass(), PMW);
292
293 // Create a mapping from LLVM CodeGen virtual registers to wasm registers.
294 addMachineFunctionPass(Pass: WebAssemblyRegNumberingPass(), PMW);
295
296 // Fix debug_values whose defs have been stackified.
297 if (!WasmDisableExplicitLocals)
298 addMachineFunctionPass(Pass: WebAssemblyDebugFixupPass(), PMW);
299
300 // Collect information to prepare for MC lowering / asm printing.
301 flushFPMsToMPM(PMW);
302 addModulePass(Pass: WebAssemblyMCLowerPrePass(), PMW);
303}
304
305void WebAssemblyCodeGenPassBuilder::addAsmPrinterBegin(
306 PassManagerWrapper &PMW) {
307 addModulePass(Pass: WebAssemblyAsmPrinterBeginPass(), PMW, /*Force=*/true);
308}
309
310void WebAssemblyCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
311 addMachineFunctionPass(Pass: WebAssemblyAsmPrinterPass(), PMW);
312}
313
314void WebAssemblyCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
315 addModulePass(Pass: WebAssemblyAsmPrinterEndPass(), PMW);
316}
317
318} // namespace
319
320void WebAssemblyTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB){
321#define GET_PASS_REGISTRY "WebAssemblyPassRegistry.def"
322#include "llvm/Passes/TargetPassRegistry.inc"
323}
324
325Error WebAssemblyTargetMachine::buildCodeGenPipeline(
326 ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out,
327 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
328 const CGPassBuilderOption &Opt, MCContext &Ctx,
329 PassInstrumentationCallbacks *PIC) {
330 auto CGPB = WebAssemblyCodeGenPassBuilder(*this, Opt, PIC);
331 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
332}
333