1//===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Transforms/Coroutines/CoroEarly.h"
10#include "CoroInternal.h"
11#include "llvm/IR/DIBuilder.h"
12#include "llvm/IR/Function.h"
13#include "llvm/IR/IRBuilder.h"
14#include "llvm/IR/InstIterator.h"
15#include "llvm/IR/Module.h"
16
17using namespace llvm;
18
19#define DEBUG_TYPE "coro-early"
20
21namespace {
22// Created on demand if the coro-early pass has work to do.
23class Lowerer : public coro::LowererBase {
24 IRBuilder<> Builder;
25 PointerType *const AnyResumeFnPtrTy;
26 Constant *NoopCoro = nullptr;
27
28 void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind);
29 void lowerCoroPromise(CoroPromiseInst *Intrin);
30 void lowerCoroDone(IntrinsicInst *II);
31 void lowerCoroNoop(IntrinsicInst *II);
32
33public:
34 Lowerer(Module &M)
35 : LowererBase(M), Builder(Context),
36 AnyResumeFnPtrTy(FunctionType::get(Result: Type::getVoidTy(C&: Context), Params: Int8Ptr,
37 /*isVarArg=*/false)
38 ->getPointerTo()) {}
39 void lowerEarlyIntrinsics(Function &F);
40};
41}
42
43// Replace a direct call to coro.resume or coro.destroy with an indirect call to
44// an address returned by coro.subfn.addr intrinsic. This is done so that
45// CGPassManager recognizes devirtualization when CoroElide pass replaces a call
46// to coro.subfn.addr with an appropriate function address.
47void Lowerer::lowerResumeOrDestroy(CallBase &CB,
48 CoroSubFnInst::ResumeKind Index) {
49 Value *ResumeAddr = makeSubFnCall(Arg: CB.getArgOperand(i: 0), Index, InsertPt: &CB);
50 CB.setCalledOperand(ResumeAddr);
51 CB.setCallingConv(CallingConv::Fast);
52}
53
54// Coroutine promise field is always at the fixed offset from the beginning of
55// the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset
56// to a passed pointer to move from coroutine frame to coroutine promise and
57// vice versa. Since we don't know exactly which coroutine frame it is, we build
58// a coroutine frame mock up starting with two function pointers, followed by a
59// properly aligned coroutine promise field.
60// TODO: Handle the case when coroutine promise alloca has align override.
61void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
62 Value *Operand = Intrin->getArgOperand(i: 0);
63 Align Alignment = Intrin->getAlignment();
64 Type *Int8Ty = Builder.getInt8Ty();
65
66 auto *SampleStruct =
67 StructType::get(Context, Elements: {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
68 const DataLayout &DL = TheModule.getDataLayout();
69 int64_t Offset = alignTo(
70 Size: DL.getStructLayout(Ty: SampleStruct)->getElementOffset(Idx: 2), A: Alignment);
71 if (Intrin->isFromPromise())
72 Offset = -Offset;
73
74 Builder.SetInsertPoint(Intrin);
75 Value *Replacement =
76 Builder.CreateConstInBoundsGEP1_32(Ty: Int8Ty, Ptr: Operand, Idx0: Offset);
77
78 Intrin->replaceAllUsesWith(V: Replacement);
79 Intrin->eraseFromParent();
80}
81
82// When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in
83// the coroutine frame (it is UB to resume from a final suspend point).
84// The llvm.coro.done intrinsic is used to check whether a coroutine is
85// suspended at the final suspend point or not.
86void Lowerer::lowerCoroDone(IntrinsicInst *II) {
87 Value *Operand = II->getArgOperand(i: 0);
88
89 // ResumeFnAddr is the first pointer sized element of the coroutine frame.
90 static_assert(coro::Shape::SwitchFieldIndex::Resume == 0,
91 "resume function not at offset zero");
92 auto *FrameTy = Int8Ptr;
93 PointerType *FramePtrTy = FrameTy->getPointerTo();
94
95 Builder.SetInsertPoint(II);
96 auto *BCI = Builder.CreateBitCast(V: Operand, DestTy: FramePtrTy);
97 auto *Load = Builder.CreateLoad(Ty: FrameTy, Ptr: BCI);
98 auto *Cond = Builder.CreateICmpEQ(LHS: Load, RHS: NullPtr);
99
100 II->replaceAllUsesWith(V: Cond);
101 II->eraseFromParent();
102}
103
104static void buildDebugInfoForNoopResumeDestroyFunc(Function *NoopFn) {
105 Module &M = *NoopFn->getParent();
106 if (M.debug_compile_units().empty())
107 return;
108
109 DICompileUnit *CU = *M.debug_compile_units_begin();
110 DIBuilder DB(M, /*AllowUnresolved*/ false, CU);
111 std::array<Metadata *, 2> Params{nullptr, nullptr};
112 auto *SubroutineType =
113 DB.createSubroutineType(ParameterTypes: DB.getOrCreateTypeArray(Elements: Params));
114 StringRef Name = NoopFn->getName();
115 auto *SP = DB.createFunction(
116 Scope: CU, /*Name=*/Name, /*LinkageName=*/Name, /*File=*/ CU->getFile(),
117 /*LineNo=*/0, Ty: SubroutineType, /*ScopeLine=*/0, Flags: DINode::FlagArtificial,
118 SPFlags: DISubprogram::SPFlagDefinition);
119 NoopFn->setSubprogram(SP);
120 DB.finalize();
121}
122
123void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
124 if (!NoopCoro) {
125 LLVMContext &C = Builder.getContext();
126 Module &M = *II->getModule();
127
128 // Create a noop.frame struct type.
129 StructType *FrameTy = StructType::create(Context&: C, Name: "NoopCoro.Frame");
130 auto *FramePtrTy = FrameTy->getPointerTo();
131 auto *FnTy = FunctionType::get(Result: Type::getVoidTy(C), Params: FramePtrTy,
132 /*isVarArg=*/false);
133 auto *FnPtrTy = FnTy->getPointerTo();
134 FrameTy->setBody(Elements: {FnPtrTy, FnPtrTy});
135
136 // Create a Noop function that does nothing.
137 Function *NoopFn =
138 Function::Create(Ty: FnTy, Linkage: GlobalValue::LinkageTypes::PrivateLinkage,
139 N: "__NoopCoro_ResumeDestroy", M: &M);
140 NoopFn->setCallingConv(CallingConv::Fast);
141 buildDebugInfoForNoopResumeDestroyFunc(NoopFn);
142 auto *Entry = BasicBlock::Create(Context&: C, Name: "entry", Parent: NoopFn);
143 ReturnInst::Create(C, InsertAtEnd: Entry);
144
145 // Create a constant struct for the frame.
146 Constant* Values[] = {NoopFn, NoopFn};
147 Constant* NoopCoroConst = ConstantStruct::get(T: FrameTy, V: Values);
148 NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
149 GlobalVariable::PrivateLinkage, NoopCoroConst,
150 "NoopCoro.Frame.Const");
151 cast<GlobalVariable>(Val: NoopCoro)->setNoSanitizeMetadata();
152 }
153
154 Builder.SetInsertPoint(II);
155 auto *NoopCoroVoidPtr = Builder.CreateBitCast(V: NoopCoro, DestTy: Int8Ptr);
156 II->replaceAllUsesWith(V: NoopCoroVoidPtr);
157 II->eraseFromParent();
158}
159
160// Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate,
161// as CoroSplit assumes there is exactly one coro.begin. After CoroSplit,
162// NoDuplicate attribute will be removed from coro.begin otherwise, it will
163// interfere with inlining.
164static void setCannotDuplicate(CoroIdInst *CoroId) {
165 for (User *U : CoroId->users())
166 if (auto *CB = dyn_cast<CoroBeginInst>(Val: U))
167 CB->setCannotDuplicate();
168}
169
170void Lowerer::lowerEarlyIntrinsics(Function &F) {
171 CoroIdInst *CoroId = nullptr;
172 SmallVector<CoroFreeInst *, 4> CoroFrees;
173 bool HasCoroSuspend = false;
174 for (Instruction &I : llvm::make_early_inc_range(Range: instructions(F))) {
175 auto *CB = dyn_cast<CallBase>(Val: &I);
176 if (!CB)
177 continue;
178
179 switch (CB->getIntrinsicID()) {
180 default:
181 continue;
182 case Intrinsic::coro_free:
183 CoroFrees.push_back(Elt: cast<CoroFreeInst>(Val: &I));
184 break;
185 case Intrinsic::coro_suspend:
186 // Make sure that final suspend point is not duplicated as CoroSplit
187 // pass expects that there is at most one final suspend point.
188 if (cast<CoroSuspendInst>(Val: &I)->isFinal())
189 CB->setCannotDuplicate();
190 HasCoroSuspend = true;
191 break;
192 case Intrinsic::coro_end_async:
193 case Intrinsic::coro_end:
194 // Make sure that fallthrough coro.end is not duplicated as CoroSplit
195 // pass expects that there is at most one fallthrough coro.end.
196 if (cast<AnyCoroEndInst>(Val: &I)->isFallthrough())
197 CB->setCannotDuplicate();
198 break;
199 case Intrinsic::coro_noop:
200 lowerCoroNoop(II: cast<IntrinsicInst>(Val: &I));
201 break;
202 case Intrinsic::coro_id:
203 if (auto *CII = cast<CoroIdInst>(Val: &I)) {
204 if (CII->getInfo().isPreSplit()) {
205 assert(F.isPresplitCoroutine() &&
206 "The frontend uses Swtich-Resumed ABI should emit "
207 "\"presplitcoroutine\" attribute for the coroutine.");
208 setCannotDuplicate(CII);
209 CII->setCoroutineSelf();
210 CoroId = cast<CoroIdInst>(Val: &I);
211 }
212 }
213 break;
214 case Intrinsic::coro_id_retcon:
215 case Intrinsic::coro_id_retcon_once:
216 case Intrinsic::coro_id_async:
217 F.setPresplitCoroutine();
218 break;
219 case Intrinsic::coro_resume:
220 lowerResumeOrDestroy(CB&: *CB, Index: CoroSubFnInst::ResumeIndex);
221 break;
222 case Intrinsic::coro_destroy:
223 lowerResumeOrDestroy(CB&: *CB, Index: CoroSubFnInst::DestroyIndex);
224 break;
225 case Intrinsic::coro_promise:
226 lowerCoroPromise(Intrin: cast<CoroPromiseInst>(Val: &I));
227 break;
228 case Intrinsic::coro_done:
229 lowerCoroDone(II: cast<IntrinsicInst>(Val: &I));
230 break;
231 }
232 }
233
234 // Make sure that all CoroFree reference the coro.id intrinsic.
235 // Token type is not exposed through coroutine C/C++ builtins to plain C, so
236 // we allow specifying none and fixing it up here.
237 if (CoroId)
238 for (CoroFreeInst *CF : CoroFrees)
239 CF->setArgOperand(i: 0, v: CoroId);
240
241 // Coroutine suspention could potentially lead to any argument modified
242 // outside of the function, hence arguments should not have noalias
243 // attributes.
244 if (HasCoroSuspend)
245 for (Argument &A : F.args())
246 if (A.hasNoAliasAttr())
247 A.removeAttr(Kind: Attribute::NoAlias);
248}
249
250static bool declaresCoroEarlyIntrinsics(const Module &M) {
251 return coro::declaresIntrinsics(
252 M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
253 "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
254 "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
255 "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
256 "llvm.coro.suspend"});
257}
258
259PreservedAnalyses CoroEarlyPass::run(Module &M, ModuleAnalysisManager &) {
260 if (!declaresCoroEarlyIntrinsics(M))
261 return PreservedAnalyses::all();
262
263 Lowerer L(M);
264 for (auto &F : M)
265 L.lowerEarlyIntrinsics(F);
266
267 PreservedAnalyses PA;
268 PA.preserveSet<CFGAnalyses>();
269 return PA;
270}
271