1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Stmt nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGDebugInfo.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "CodeGenPGO.h"
18#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Expr.h"
21#include "clang/AST/Stmt.h"
22#include "clang/AST/StmtSYCL.h"
23#include "clang/AST/StmtVisitor.h"
24#include "clang/Basic/Builtins.h"
25#include "clang/Basic/DiagnosticSema.h"
26#include "clang/Basic/PrettyStackTrace.h"
27#include "clang/Basic/SourceManager.h"
28#include "clang/Basic/TargetInfo.h"
29#include "clang/CodeGenUtils/StmtUtils.h"
30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/DenseMap.h"
32#include "llvm/ADT/SmallSet.h"
33#include "llvm/ADT/StringExtras.h"
34#include "llvm/IR/Assumptions.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/InlineAsm.h"
37#include "llvm/IR/Intrinsics.h"
38#include "llvm/IR/MDBuilder.h"
39#include "llvm/Support/SaveAndRestore.h"
40#include <optional>
41
42using namespace clang;
43using namespace CodeGen;
44
45//===----------------------------------------------------------------------===//
46// Statement Emission
47//===----------------------------------------------------------------------===//
48
49void CodeGenFunction::EmitStopPoint(const Stmt *S) {
50 if (CGDebugInfo *DI = getDebugInfo()) {
51 SourceLocation Loc;
52 Loc = S->getBeginLoc();
53 DI->EmitLocation(Builder, Loc);
54
55 LastStopPoint = Loc;
56 }
57}
58
59void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
60 assert(S && "Null statement?");
61 PGO->setCurrentStmt(S);
62
63 // These statements have their own debug info handling.
64 if (EmitSimpleStmt(S, Attrs))
65 return;
66
67 // Check if we are generating unreachable code.
68 if (!HaveInsertPoint()) {
69 // If so, and the statement doesn't contain a label, then we do not need to
70 // generate actual code. This is safe because (1) the current point is
71 // unreachable, so we don't need to execute the code, and (2) we've already
72 // handled the statements which update internal data structures (like the
73 // local variable map) which could be used by subsequent statements.
74 if (!ContainsLabel(S)) {
75 // Verify that any decl statements were handled as simple, they may be in
76 // scope of subsequent reachable statements.
77 assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
78 PGO->markStmtMaybeUsed(S);
79 return;
80 }
81
82 // Otherwise, make a new block to hold the code.
83 EnsureInsertPoint();
84 }
85
86 // Generate a stoppoint if we are emitting debug info.
87 EmitStopPoint(S);
88
89 // Ignore all OpenMP directives except for simd if OpenMP with Simd is
90 // enabled.
91 if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
92 if (const auto *D = dyn_cast<OMPExecutableDirective>(Val: S)) {
93 EmitSimpleOMPExecutableDirective(D: *D);
94 return;
95 }
96 }
97
98 switch (S->getStmtClass()) {
99 case Stmt::NoStmtClass:
100 case Stmt::CXXCatchStmtClass:
101 case Stmt::SEHExceptStmtClass:
102 case Stmt::SEHFinallyStmtClass:
103 case Stmt::MSDependentExistsStmtClass:
104 case Stmt::UnresolvedSYCLKernelCallStmtClass:
105 llvm_unreachable("invalid statement class to emit generically");
106 case Stmt::NullStmtClass:
107 case Stmt::CompoundStmtClass:
108 case Stmt::DeclStmtClass:
109 case Stmt::LabelStmtClass:
110 case Stmt::AttributedStmtClass:
111 case Stmt::GotoStmtClass:
112 case Stmt::BreakStmtClass:
113 case Stmt::ContinueStmtClass:
114 case Stmt::DefaultStmtClass:
115 case Stmt::CaseStmtClass:
116 case Stmt::DeferStmtClass:
117 case Stmt::SEHLeaveStmtClass:
118 case Stmt::SYCLKernelCallStmtClass:
119 llvm_unreachable("should have emitted these statements as simple");
120
121#define STMT(Type, Base)
122#define ABSTRACT_STMT(Op)
123#define EXPR(Type, Base) \
124 case Stmt::Type##Class:
125#include "clang/AST/StmtNodes.inc"
126 {
127 // Remember the block we came in on.
128 llvm::BasicBlock *incoming = Builder.GetInsertBlock();
129 assert(incoming && "expression emission must have an insertion point");
130
131 EmitIgnoredExpr(E: cast<Expr>(Val: S));
132
133 llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
134 assert(outgoing && "expression emission cleared block!");
135
136 // The expression emitters assume (reasonably!) that the insertion
137 // point is always set. To maintain that, the call-emission code
138 // for noreturn functions has to enter a new block with no
139 // predecessors. We want to kill that block and mark the current
140 // insertion point unreachable in the common case of a call like
141 // "exit();". Since expression emission doesn't otherwise create
142 // blocks with no predecessors, we can just test for that.
143 // However, we must be careful not to do this to our incoming
144 // block, because *statement* emission does sometimes create
145 // reachable blocks which will have no predecessors until later in
146 // the function. This occurs with, e.g., labels that are not
147 // reachable by fallthrough.
148 if (incoming != outgoing && outgoing->use_empty()) {
149 outgoing->eraseFromParent();
150 Builder.ClearInsertionPoint();
151 }
152 break;
153 }
154
155 case Stmt::IndirectGotoStmtClass:
156 EmitIndirectGotoStmt(S: cast<IndirectGotoStmt>(Val: *S)); break;
157
158 case Stmt::IfStmtClass: EmitIfStmt(S: cast<IfStmt>(Val: *S)); break;
159 case Stmt::WhileStmtClass: EmitWhileStmt(S: cast<WhileStmt>(Val: *S), Attrs); break;
160 case Stmt::DoStmtClass: EmitDoStmt(S: cast<DoStmt>(Val: *S), Attrs); break;
161 case Stmt::ForStmtClass: EmitForStmt(S: cast<ForStmt>(Val: *S), Attrs); break;
162
163 case Stmt::ReturnStmtClass: EmitReturnStmt(S: cast<ReturnStmt>(Val: *S)); break;
164
165 case Stmt::SwitchStmtClass: EmitSwitchStmt(S: cast<SwitchStmt>(Val: *S)); break;
166 case Stmt::GCCAsmStmtClass: // Intentional fall-through.
167 case Stmt::MSAsmStmtClass: EmitAsmStmt(S: cast<AsmStmt>(Val: *S)); break;
168 case Stmt::CoroutineBodyStmtClass:
169 EmitCoroutineBody(S: cast<CoroutineBodyStmt>(Val: *S));
170 break;
171 case Stmt::CoreturnStmtClass:
172 EmitCoreturnStmt(S: cast<CoreturnStmt>(Val: *S));
173 break;
174 case Stmt::CapturedStmtClass: {
175 const CapturedStmt *CS = cast<CapturedStmt>(Val: S);
176 EmitCapturedStmt(S: *CS, K: CS->getCapturedRegionKind());
177 }
178 break;
179 case Stmt::ObjCAtTryStmtClass:
180 EmitObjCAtTryStmt(S: cast<ObjCAtTryStmt>(Val: *S));
181 break;
182 case Stmt::ObjCAtCatchStmtClass:
183 llvm_unreachable(
184 "@catch statements should be handled by EmitObjCAtTryStmt");
185 case Stmt::ObjCAtFinallyStmtClass:
186 llvm_unreachable(
187 "@finally statements should be handled by EmitObjCAtTryStmt");
188 case Stmt::ObjCAtThrowStmtClass:
189 EmitObjCAtThrowStmt(S: cast<ObjCAtThrowStmt>(Val: *S));
190 break;
191 case Stmt::ObjCAtSynchronizedStmtClass:
192 EmitObjCAtSynchronizedStmt(S: cast<ObjCAtSynchronizedStmt>(Val: *S));
193 break;
194 case Stmt::ObjCForCollectionStmtClass:
195 EmitObjCForCollectionStmt(S: cast<ObjCForCollectionStmt>(Val: *S));
196 break;
197 case Stmt::ObjCAutoreleasePoolStmtClass:
198 EmitObjCAutoreleasePoolStmt(S: cast<ObjCAutoreleasePoolStmt>(Val: *S));
199 break;
200
201 case Stmt::CXXTryStmtClass:
202 EmitCXXTryStmt(S: cast<CXXTryStmt>(Val: *S));
203 break;
204 case Stmt::CXXForRangeStmtClass:
205 EmitCXXForRangeStmt(S: cast<CXXForRangeStmt>(Val: *S), Attrs);
206 break;
207 case Stmt::CXXExpansionStmtPatternClass:
208 llvm_unreachable("unexpanded expansion statements should not be emitted");
209 case Stmt::CXXExpansionStmtInstantiationClass:
210 EmitCXXExpansionStmtInstantiation(S: cast<CXXExpansionStmtInstantiation>(Val: *S));
211 break;
212 case Stmt::SEHTryStmtClass:
213 EmitSEHTryStmt(S: cast<SEHTryStmt>(Val: *S));
214 break;
215 case Stmt::OMPMetaDirectiveClass:
216 EmitOMPMetaDirective(S: cast<OMPMetaDirective>(Val: *S));
217 break;
218 case Stmt::OMPCanonicalLoopClass:
219 EmitOMPCanonicalLoop(S: cast<OMPCanonicalLoop>(Val: S));
220 break;
221 case Stmt::OMPParallelDirectiveClass:
222 EmitOMPParallelDirective(S: cast<OMPParallelDirective>(Val: *S));
223 break;
224 case Stmt::OMPSimdDirectiveClass:
225 EmitOMPSimdDirective(S: cast<OMPSimdDirective>(Val: *S));
226 break;
227 case Stmt::OMPTileDirectiveClass:
228 EmitOMPTileDirective(S: cast<OMPTileDirective>(Val: *S));
229 break;
230 case Stmt::OMPStripeDirectiveClass:
231 EmitOMPStripeDirective(S: cast<OMPStripeDirective>(Val: *S));
232 break;
233 case Stmt::OMPUnrollDirectiveClass:
234 EmitOMPUnrollDirective(S: cast<OMPUnrollDirective>(Val: *S));
235 break;
236 case Stmt::OMPReverseDirectiveClass:
237 EmitOMPReverseDirective(S: cast<OMPReverseDirective>(Val: *S));
238 break;
239 case Stmt::OMPSplitDirectiveClass:
240 EmitOMPSplitDirective(S: cast<OMPSplitDirective>(Val: *S));
241 break;
242 case Stmt::OMPInterchangeDirectiveClass:
243 EmitOMPInterchangeDirective(S: cast<OMPInterchangeDirective>(Val: *S));
244 break;
245 case Stmt::OMPFlattenDirectiveClass:
246 EmitOMPFlattenDirective(S: cast<OMPFlattenDirective>(Val: *S));
247 break;
248 case Stmt::OMPFuseDirectiveClass:
249 EmitOMPFuseDirective(S: cast<OMPFuseDirective>(Val: *S));
250 break;
251 case Stmt::OMPForDirectiveClass:
252 EmitOMPForDirective(S: cast<OMPForDirective>(Val: *S));
253 break;
254 case Stmt::OMPForSimdDirectiveClass:
255 EmitOMPForSimdDirective(S: cast<OMPForSimdDirective>(Val: *S));
256 break;
257 case Stmt::OMPSectionsDirectiveClass:
258 EmitOMPSectionsDirective(S: cast<OMPSectionsDirective>(Val: *S));
259 break;
260 case Stmt::OMPSectionDirectiveClass:
261 EmitOMPSectionDirective(S: cast<OMPSectionDirective>(Val: *S));
262 break;
263 case Stmt::OMPSingleDirectiveClass:
264 EmitOMPSingleDirective(S: cast<OMPSingleDirective>(Val: *S));
265 break;
266 case Stmt::OMPMasterDirectiveClass:
267 EmitOMPMasterDirective(S: cast<OMPMasterDirective>(Val: *S));
268 break;
269 case Stmt::OMPCriticalDirectiveClass:
270 EmitOMPCriticalDirective(S: cast<OMPCriticalDirective>(Val: *S));
271 break;
272 case Stmt::OMPParallelForDirectiveClass:
273 EmitOMPParallelForDirective(S: cast<OMPParallelForDirective>(Val: *S));
274 break;
275 case Stmt::OMPParallelForSimdDirectiveClass:
276 EmitOMPParallelForSimdDirective(S: cast<OMPParallelForSimdDirective>(Val: *S));
277 break;
278 case Stmt::OMPParallelMasterDirectiveClass:
279 EmitOMPParallelMasterDirective(S: cast<OMPParallelMasterDirective>(Val: *S));
280 break;
281 case Stmt::OMPParallelSectionsDirectiveClass:
282 EmitOMPParallelSectionsDirective(S: cast<OMPParallelSectionsDirective>(Val: *S));
283 break;
284 case Stmt::OMPTaskDirectiveClass:
285 EmitOMPTaskDirective(S: cast<OMPTaskDirective>(Val: *S));
286 break;
287 case Stmt::OMPTaskyieldDirectiveClass:
288 EmitOMPTaskyieldDirective(S: cast<OMPTaskyieldDirective>(Val: *S));
289 break;
290 case Stmt::OMPErrorDirectiveClass:
291 EmitOMPErrorDirective(S: cast<OMPErrorDirective>(Val: *S));
292 break;
293 case Stmt::OMPBarrierDirectiveClass:
294 EmitOMPBarrierDirective(S: cast<OMPBarrierDirective>(Val: *S));
295 break;
296 case Stmt::OMPTaskwaitDirectiveClass:
297 EmitOMPTaskwaitDirective(S: cast<OMPTaskwaitDirective>(Val: *S));
298 break;
299 case Stmt::OMPTaskgroupDirectiveClass:
300 EmitOMPTaskgroupDirective(S: cast<OMPTaskgroupDirective>(Val: *S));
301 break;
302 case Stmt::OMPFlushDirectiveClass:
303 EmitOMPFlushDirective(S: cast<OMPFlushDirective>(Val: *S));
304 break;
305 case Stmt::OMPDepobjDirectiveClass:
306 EmitOMPDepobjDirective(S: cast<OMPDepobjDirective>(Val: *S));
307 break;
308 case Stmt::OMPScanDirectiveClass:
309 EmitOMPScanDirective(S: cast<OMPScanDirective>(Val: *S));
310 break;
311 case Stmt::OMPOrderedStandaloneDirectiveClass:
312 EmitOMPOrderedStandaloneDirective(S: cast<OMPOrderedStandaloneDirective>(Val: *S));
313 break;
314 case Stmt::OMPOrderedBlockAssocDirectiveClass:
315 EmitOMPOrderedBlockAssocDirective(S: cast<OMPOrderedBlockAssocDirective>(Val: *S));
316 break;
317 case Stmt::OMPAtomicDirectiveClass:
318 EmitOMPAtomicDirective(S: cast<OMPAtomicDirective>(Val: *S));
319 break;
320 case Stmt::OMPTargetDirectiveClass:
321 EmitOMPTargetDirective(S: cast<OMPTargetDirective>(Val: *S));
322 break;
323 case Stmt::OMPTeamsDirectiveClass:
324 EmitOMPTeamsDirective(S: cast<OMPTeamsDirective>(Val: *S));
325 break;
326 case Stmt::OMPCancellationPointDirectiveClass:
327 EmitOMPCancellationPointDirective(S: cast<OMPCancellationPointDirective>(Val: *S));
328 break;
329 case Stmt::OMPCancelDirectiveClass:
330 EmitOMPCancelDirective(S: cast<OMPCancelDirective>(Val: *S));
331 break;
332 case Stmt::OMPTargetDataDirectiveClass:
333 EmitOMPTargetDataDirective(S: cast<OMPTargetDataDirective>(Val: *S));
334 break;
335 case Stmt::OMPTargetEnterDataDirectiveClass:
336 EmitOMPTargetEnterDataDirective(S: cast<OMPTargetEnterDataDirective>(Val: *S));
337 break;
338 case Stmt::OMPTargetExitDataDirectiveClass:
339 EmitOMPTargetExitDataDirective(S: cast<OMPTargetExitDataDirective>(Val: *S));
340 break;
341 case Stmt::OMPTargetParallelDirectiveClass:
342 EmitOMPTargetParallelDirective(S: cast<OMPTargetParallelDirective>(Val: *S));
343 break;
344 case Stmt::OMPTargetParallelForDirectiveClass:
345 EmitOMPTargetParallelForDirective(S: cast<OMPTargetParallelForDirective>(Val: *S));
346 break;
347 case Stmt::OMPTaskLoopDirectiveClass:
348 EmitOMPTaskLoopDirective(S: cast<OMPTaskLoopDirective>(Val: *S));
349 break;
350 case Stmt::OMPTaskLoopSimdDirectiveClass:
351 EmitOMPTaskLoopSimdDirective(S: cast<OMPTaskLoopSimdDirective>(Val: *S));
352 break;
353 case Stmt::OMPMasterTaskLoopDirectiveClass:
354 EmitOMPMasterTaskLoopDirective(S: cast<OMPMasterTaskLoopDirective>(Val: *S));
355 break;
356 case Stmt::OMPMaskedTaskLoopDirectiveClass:
357 EmitOMPMaskedTaskLoopDirective(S: cast<OMPMaskedTaskLoopDirective>(Val: *S));
358 break;
359 case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
360 EmitOMPMasterTaskLoopSimdDirective(
361 S: cast<OMPMasterTaskLoopSimdDirective>(Val: *S));
362 break;
363 case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
364 EmitOMPMaskedTaskLoopSimdDirective(
365 S: cast<OMPMaskedTaskLoopSimdDirective>(Val: *S));
366 break;
367 case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
368 EmitOMPParallelMasterTaskLoopDirective(
369 S: cast<OMPParallelMasterTaskLoopDirective>(Val: *S));
370 break;
371 case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
372 EmitOMPParallelMaskedTaskLoopDirective(
373 S: cast<OMPParallelMaskedTaskLoopDirective>(Val: *S));
374 break;
375 case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
376 EmitOMPParallelMasterTaskLoopSimdDirective(
377 S: cast<OMPParallelMasterTaskLoopSimdDirective>(Val: *S));
378 break;
379 case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
380 EmitOMPParallelMaskedTaskLoopSimdDirective(
381 S: cast<OMPParallelMaskedTaskLoopSimdDirective>(Val: *S));
382 break;
383 case Stmt::OMPDistributeDirectiveClass:
384 EmitOMPDistributeDirective(S: cast<OMPDistributeDirective>(Val: *S));
385 break;
386 case Stmt::OMPTargetUpdateDirectiveClass:
387 EmitOMPTargetUpdateDirective(S: cast<OMPTargetUpdateDirective>(Val: *S));
388 break;
389 case Stmt::OMPDistributeParallelForDirectiveClass:
390 EmitOMPDistributeParallelForDirective(
391 S: cast<OMPDistributeParallelForDirective>(Val: *S));
392 break;
393 case Stmt::OMPDistributeParallelForSimdDirectiveClass:
394 EmitOMPDistributeParallelForSimdDirective(
395 S: cast<OMPDistributeParallelForSimdDirective>(Val: *S));
396 break;
397 case Stmt::OMPDistributeSimdDirectiveClass:
398 EmitOMPDistributeSimdDirective(S: cast<OMPDistributeSimdDirective>(Val: *S));
399 break;
400 case Stmt::OMPTargetParallelForSimdDirectiveClass:
401 EmitOMPTargetParallelForSimdDirective(
402 S: cast<OMPTargetParallelForSimdDirective>(Val: *S));
403 break;
404 case Stmt::OMPTargetSimdDirectiveClass:
405 EmitOMPTargetSimdDirective(S: cast<OMPTargetSimdDirective>(Val: *S));
406 break;
407 case Stmt::OMPTeamsDistributeDirectiveClass:
408 EmitOMPTeamsDistributeDirective(S: cast<OMPTeamsDistributeDirective>(Val: *S));
409 break;
410 case Stmt::OMPTeamsDistributeSimdDirectiveClass:
411 EmitOMPTeamsDistributeSimdDirective(
412 S: cast<OMPTeamsDistributeSimdDirective>(Val: *S));
413 break;
414 case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
415 EmitOMPTeamsDistributeParallelForSimdDirective(
416 S: cast<OMPTeamsDistributeParallelForSimdDirective>(Val: *S));
417 break;
418 case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
419 EmitOMPTeamsDistributeParallelForDirective(
420 S: cast<OMPTeamsDistributeParallelForDirective>(Val: *S));
421 break;
422 case Stmt::OMPTargetTeamsDirectiveClass:
423 EmitOMPTargetTeamsDirective(S: cast<OMPTargetTeamsDirective>(Val: *S));
424 break;
425 case Stmt::OMPTargetTeamsDistributeDirectiveClass:
426 EmitOMPTargetTeamsDistributeDirective(
427 S: cast<OMPTargetTeamsDistributeDirective>(Val: *S));
428 break;
429 case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
430 EmitOMPTargetTeamsDistributeParallelForDirective(
431 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: *S));
432 break;
433 case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
434 EmitOMPTargetTeamsDistributeParallelForSimdDirective(
435 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: *S));
436 break;
437 case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
438 EmitOMPTargetTeamsDistributeSimdDirective(
439 S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: *S));
440 break;
441 case Stmt::OMPInteropDirectiveClass:
442 EmitOMPInteropDirective(S: cast<OMPInteropDirective>(Val: *S));
443 break;
444 case Stmt::OMPDispatchDirectiveClass:
445 CGM.ErrorUnsupported(S, Type: "OpenMP dispatch directive");
446 break;
447 case Stmt::OMPScopeDirectiveClass:
448 EmitOMPScopeDirective(S: cast<OMPScopeDirective>(Val: *S));
449 break;
450 case Stmt::OMPMaskedDirectiveClass:
451 EmitOMPMaskedDirective(S: cast<OMPMaskedDirective>(Val: *S));
452 break;
453 case Stmt::OMPGenericLoopDirectiveClass:
454 EmitOMPGenericLoopDirective(S: cast<OMPGenericLoopDirective>(Val: *S));
455 break;
456 case Stmt::OMPTeamsGenericLoopDirectiveClass:
457 EmitOMPTeamsGenericLoopDirective(S: cast<OMPTeamsGenericLoopDirective>(Val: *S));
458 break;
459 case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
460 EmitOMPTargetTeamsGenericLoopDirective(
461 S: cast<OMPTargetTeamsGenericLoopDirective>(Val: *S));
462 break;
463 case Stmt::OMPParallelGenericLoopDirectiveClass:
464 EmitOMPParallelGenericLoopDirective(
465 S: cast<OMPParallelGenericLoopDirective>(Val: *S));
466 break;
467 case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
468 EmitOMPTargetParallelGenericLoopDirective(
469 S: cast<OMPTargetParallelGenericLoopDirective>(Val: *S));
470 break;
471 case Stmt::OMPTargetGenericLoopDirectiveClass:
472 EmitOMPTargetSimdDirective(S: cast<OMPTargetGenericLoopDirective>(Val: *S));
473 break;
474 case Stmt::OMPParallelMaskedDirectiveClass:
475 EmitOMPParallelMaskedDirective(S: cast<OMPParallelMaskedDirective>(Val: *S));
476 break;
477 case Stmt::OMPAssumeDirectiveClass:
478 EmitOMPAssumeDirective(S: cast<OMPAssumeDirective>(Val: *S));
479 break;
480 case Stmt::OpenACCComputeConstructClass:
481 EmitOpenACCComputeConstruct(S: cast<OpenACCComputeConstruct>(Val: *S));
482 break;
483 case Stmt::OpenACCLoopConstructClass:
484 EmitOpenACCLoopConstruct(S: cast<OpenACCLoopConstruct>(Val: *S));
485 break;
486 case Stmt::OpenACCCombinedConstructClass:
487 EmitOpenACCCombinedConstruct(S: cast<OpenACCCombinedConstruct>(Val: *S));
488 break;
489 case Stmt::OpenACCDataConstructClass:
490 EmitOpenACCDataConstruct(S: cast<OpenACCDataConstruct>(Val: *S));
491 break;
492 case Stmt::OpenACCEnterDataConstructClass:
493 EmitOpenACCEnterDataConstruct(S: cast<OpenACCEnterDataConstruct>(Val: *S));
494 break;
495 case Stmt::OpenACCExitDataConstructClass:
496 EmitOpenACCExitDataConstruct(S: cast<OpenACCExitDataConstruct>(Val: *S));
497 break;
498 case Stmt::OpenACCHostDataConstructClass:
499 EmitOpenACCHostDataConstruct(S: cast<OpenACCHostDataConstruct>(Val: *S));
500 break;
501 case Stmt::OpenACCWaitConstructClass:
502 EmitOpenACCWaitConstruct(S: cast<OpenACCWaitConstruct>(Val: *S));
503 break;
504 case Stmt::OpenACCInitConstructClass:
505 EmitOpenACCInitConstruct(S: cast<OpenACCInitConstruct>(Val: *S));
506 break;
507 case Stmt::OpenACCShutdownConstructClass:
508 EmitOpenACCShutdownConstruct(S: cast<OpenACCShutdownConstruct>(Val: *S));
509 break;
510 case Stmt::OpenACCSetConstructClass:
511 EmitOpenACCSetConstruct(S: cast<OpenACCSetConstruct>(Val: *S));
512 break;
513 case Stmt::OpenACCUpdateConstructClass:
514 EmitOpenACCUpdateConstruct(S: cast<OpenACCUpdateConstruct>(Val: *S));
515 break;
516 case Stmt::OpenACCAtomicConstructClass:
517 EmitOpenACCAtomicConstruct(S: cast<OpenACCAtomicConstruct>(Val: *S));
518 break;
519 case Stmt::OpenACCCacheConstructClass:
520 EmitOpenACCCacheConstruct(S: cast<OpenACCCacheConstruct>(Val: *S));
521 break;
522 }
523}
524
525bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
526 ArrayRef<const Attr *> Attrs) {
527 switch (S->getStmtClass()) {
528 default:
529 return false;
530 case Stmt::NullStmtClass:
531 break;
532 case Stmt::CompoundStmtClass:
533 EmitCompoundStmt(S: cast<CompoundStmt>(Val: *S));
534 break;
535 case Stmt::DeclStmtClass:
536 EmitDeclStmt(S: cast<DeclStmt>(Val: *S));
537 break;
538 case Stmt::LabelStmtClass:
539 EmitLabelStmt(S: cast<LabelStmt>(Val: *S));
540 break;
541 case Stmt::AttributedStmtClass:
542 EmitAttributedStmt(S: cast<AttributedStmt>(Val: *S));
543 break;
544 case Stmt::GotoStmtClass:
545 EmitGotoStmt(S: cast<GotoStmt>(Val: *S));
546 break;
547 case Stmt::BreakStmtClass:
548 EmitBreakStmt(S: cast<BreakStmt>(Val: *S));
549 break;
550 case Stmt::ContinueStmtClass:
551 EmitContinueStmt(S: cast<ContinueStmt>(Val: *S));
552 break;
553 case Stmt::DefaultStmtClass:
554 EmitDefaultStmt(S: cast<DefaultStmt>(Val: *S), Attrs);
555 break;
556 case Stmt::CaseStmtClass:
557 EmitCaseStmt(S: cast<CaseStmt>(Val: *S), Attrs);
558 break;
559 case Stmt::DeferStmtClass:
560 EmitDeferStmt(S: cast<DeferStmt>(Val: *S));
561 break;
562 case Stmt::SEHLeaveStmtClass:
563 EmitSEHLeaveStmt(S: cast<SEHLeaveStmt>(Val: *S));
564 break;
565 case Stmt::SYCLKernelCallStmtClass:
566 EmitSYCLKernelCallStmt(S: cast<SYCLKernelCallStmt>(Val: *S));
567 break;
568 }
569 return true;
570}
571
572/// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true,
573/// this captures the expression result of the last sub-statement and returns it
574/// (for use by the statement expression extension).
575Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
576 AggValueSlot AggSlot) {
577 PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
578 "LLVM IR generation of compound statement ('{}')");
579
580 // Keep track of the current cleanup stack depth, including debug scopes.
581 LexicalScope Scope(*this, S.getSourceRange());
582
583 return EmitCompoundStmtWithoutScope(S, GetLast, AVS: AggSlot);
584}
585
586Address
587CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
588 bool GetLast,
589 AggValueSlot AggSlot) {
590
591 for (CompoundStmt::const_body_iterator I = S.body_begin(),
592 E = S.body_end() - GetLast;
593 I != E; ++I)
594 EmitStmt(S: *I);
595
596 Address RetAlloca = Address::invalid();
597 if (GetLast) {
598 // We have to special case labels here. They are statements, but when put
599 // at the end of a statement expression, they yield the value of their
600 // subexpression. Handle this by walking through all labels we encounter,
601 // emitting them before we evaluate the subexpr.
602 // Similar issues arise for attributed statements.
603 const Stmt *LastStmt = S.body_back();
604 while (!isa<Expr>(Val: LastStmt)) {
605 if (const auto *LS = dyn_cast<LabelStmt>(Val: LastStmt)) {
606 EmitLabel(D: LS->getDecl());
607 LastStmt = LS->getSubStmt();
608 } else if (const auto *AS = dyn_cast<AttributedStmt>(Val: LastStmt)) {
609 // FIXME: Update this if we ever have attributes that affect the
610 // semantics of an expression.
611 LastStmt = AS->getSubStmt();
612 } else {
613 llvm_unreachable("unknown value statement");
614 }
615 }
616
617 EnsureInsertPoint();
618
619 const Expr *E = cast<Expr>(Val: LastStmt);
620 QualType ExprTy = E->getType();
621 if (hasAggregateEvaluationKind(T: ExprTy)) {
622 EmitAggExpr(E, AS: AggSlot);
623 } else {
624 // We can't return an RValue here because there might be cleanups at
625 // the end of the StmtExpr. Because of that, we have to emit the result
626 // here into a temporary alloca.
627 RetAlloca = CreateMemTempWithoutCast(T: ExprTy);
628 EmitAnyExprToMem(E, Location: RetAlloca, Quals: Qualifiers(),
629 /*IsInit*/ IsInitializer: false);
630 }
631 }
632
633 return RetAlloca;
634}
635
636void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
637 llvm::UncondBrInst *BI = dyn_cast<llvm::UncondBrInst>(Val: BB->getTerminator());
638
639 // If there is a cleanup stack, then we it isn't worth trying to
640 // simplify this block (we would need to remove it from the scope map
641 // and cleanup entry).
642 if (!EHStack.empty())
643 return;
644
645 // Can only simplify direct branches.
646 if (!BI)
647 return;
648
649 // Can only simplify empty blocks.
650 if (BI->getIterator() != BB->begin())
651 return;
652
653 BB->replaceAllUsesWith(V: BI->getSuccessor());
654 BI->eraseFromParent();
655 BB->eraseFromParent();
656}
657
658void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
659 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
660
661 // Fall out of the current block (if necessary).
662 EmitBranch(Block: BB);
663
664 if (IsFinished && BB->use_empty()) {
665 delete BB;
666 return;
667 }
668
669 // Place the block after the current block, if possible, or else at
670 // the end of the function.
671 if (CurBB && CurBB->getParent())
672 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
673 else
674 CurFn->insert(Position: CurFn->end(), BB);
675 Builder.SetInsertPoint(BB);
676}
677
678void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
679 // Emit a branch from the current block to the target one if this
680 // was a real block. If this was just a fall-through block after a
681 // terminator, don't emit it.
682 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
683
684 if (!CurBB || CurBB->hasTerminator()) {
685 // If there is no insert point or the previous block is already
686 // terminated, don't touch it.
687 } else {
688 // Otherwise, create a fall-through branch.
689 Builder.CreateBr(Dest: Target);
690 }
691
692 Builder.ClearInsertionPoint();
693}
694
695void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
696 bool inserted = false;
697 for (llvm::User *u : block->users()) {
698 if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(Val: u)) {
699 CurFn->insert(Position: std::next(x: insn->getParent()->getIterator()), BB: block);
700 inserted = true;
701 break;
702 }
703 }
704
705 if (!inserted)
706 CurFn->insert(Position: CurFn->end(), BB: block);
707
708 Builder.SetInsertPoint(block);
709}
710
711CodeGenFunction::JumpDest
712CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
713 JumpDest &Dest = LabelMap[D];
714 if (Dest.isValid()) return Dest;
715
716 // Create, but don't insert, the new block.
717 Dest = JumpDest(createBasicBlock(name: D->getName()),
718 EHScopeStack::stable_iterator::invalid(),
719 NextCleanupDestIndex++);
720 return Dest;
721}
722
723void CodeGenFunction::EmitLabel(const LabelDecl *D) {
724 // Add this label to the current lexical scope if we're within any
725 // normal cleanups. Jumps "in" to this label --- when permitted by
726 // the language --- may need to be routed around such cleanups.
727 if (EHStack.hasNormalCleanups() && CurLexicalScope)
728 CurLexicalScope->addLabel(label: D);
729
730 JumpDest &Dest = LabelMap[D];
731
732 // If we didn't need a forward reference to this label, just go
733 // ahead and create a destination at the current scope.
734 if (!Dest.isValid()) {
735 Dest = getJumpDestInCurrentScope(Name: D->getName());
736
737 // Otherwise, we need to give this label a target depth and remove
738 // it from the branch-fixups list.
739 } else {
740 assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
741 Dest.setScopeDepth(EHStack.stable_begin());
742 ResolveBranchFixups(Target: Dest.getBlock());
743 }
744
745 EmitBlock(BB: Dest.getBlock());
746
747 // Emit debug info for labels.
748 if (CGDebugInfo *DI = getDebugInfo()) {
749 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
750 DI->setLocation(D->getLocation());
751 DI->EmitLabel(D, Builder);
752 }
753 }
754
755 incrementProfileCounter(S: D->getStmt());
756}
757
758/// Change the cleanup scope of the labels in this lexical scope to
759/// match the scope of the enclosing context.
760void CodeGenFunction::LexicalScope::rescopeLabels() {
761 assert(!Labels.empty());
762 EHScopeStack::stable_iterator innermostScope
763 = CGF.EHStack.getInnermostNormalCleanup();
764
765 // Change the scope depth of all the labels.
766 for (const LabelDecl *Label : Labels) {
767 assert(CGF.LabelMap.count(Label));
768 JumpDest &dest = CGF.LabelMap.find(Val: Label)->second;
769 assert(dest.getScopeDepth().isValid());
770 assert(innermostScope.encloses(dest.getScopeDepth()));
771 dest.setScopeDepth(innermostScope);
772 }
773
774 // Reparent the labels if the new scope also has cleanups.
775 if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
776 ParentScope->Labels.append(in_start: Labels.begin(), in_end: Labels.end());
777 }
778}
779
780
781void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
782 EmitLabel(D: S.getDecl());
783
784 // IsEHa - emit eha.scope.begin if it's a side entry of a scope
785 if (getLangOpts().EHAsynch && S.isSideEntry())
786 EmitSehCppScopeBegin();
787
788 EmitStmt(S: S.getSubStmt());
789}
790
791void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
792 bool nomerge = InNoMergeAttributedStmt;
793 bool noinline = InNoInlineAttributedStmt;
794 bool alwaysinline = InAlwaysInlineAttributedStmt;
795 bool noconvergent = InNoConvergentAttributedStmt;
796 StringRef amdgpuAVMode = AMDGPUAvailableVisibleMode;
797 HLSLControlFlowHintAttr::Spelling flattenOrBranch = HLSLControlFlowAttr;
798 const CallExpr *musttail = MustTailCall;
799 const AtomicAttr *AA = nullptr;
800
801 for (const auto *A : S.getAttrs()) {
802 switch (A->getKind()) {
803 default:
804 break;
805 case attr::NoMerge:
806 nomerge = true;
807 break;
808 case attr::NoInline:
809 noinline = true;
810 alwaysinline = false;
811 break;
812 case attr::AlwaysInline:
813 alwaysinline = true;
814 noinline = false;
815 break;
816 case attr::NoConvergent:
817 noconvergent = true;
818 break;
819 case attr::MustTail: {
820 const Stmt *Sub = S.getSubStmt();
821 const ReturnStmt *R = cast<ReturnStmt>(Val: Sub);
822 musttail = cast<CallExpr>(Val: R->getRetValue()->IgnoreParens());
823 } break;
824 case attr::CXXAssume: {
825 const Expr *Assumption = cast<CXXAssumeAttr>(Val: A)->getAssumption();
826 if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() &&
827 !Assumption->HasSideEffects(Ctx: getContext())) {
828 llvm::Value *AssumptionVal = EmitCheckedArgForAssume(E: Assumption);
829 Builder.CreateAssumption(Cond: AssumptionVal);
830 }
831 } break;
832 case attr::Atomic:
833 AA = cast<AtomicAttr>(Val: A);
834 break;
835 case attr::AMDGPUAvailableVisible:
836 amdgpuAVMode = cast<AMDGPUAvailableVisibleAttr>(Val: A)->getMode();
837 break;
838 case attr::HLSLControlFlowHint: {
839 flattenOrBranch = cast<HLSLControlFlowHintAttr>(Val: A)->getSemanticSpelling();
840 } break;
841 }
842 }
843
844 assert(!(alwaysinline && noinline) &&
845 "alwaysinline and noinline are mutually exclusive");
846
847 SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);
848 SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);
849 SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);
850 SaveAndRestore save_noconvergent(InNoConvergentAttributedStmt, noconvergent);
851 SaveAndRestore save_amdgpuav(AMDGPUAvailableVisibleMode, amdgpuAVMode);
852 SaveAndRestore save_musttail(MustTailCall, musttail);
853 SaveAndRestore save_flattenOrBranch(HLSLControlFlowAttr, flattenOrBranch);
854 CGAtomicOptionsRAII AORAII(CGM, AA);
855 EmitStmt(S: S.getSubStmt(), Attrs: S.getAttrs());
856}
857
858void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
859 // If this code is reachable then emit a stop point (if generating
860 // debug info). We have to do this ourselves because we are on the
861 // "simple" statement path.
862 if (HaveInsertPoint())
863 EmitStopPoint(S: &S);
864
865 // Reinitialize the variables this goto bypasses, whose scope it re-enters.
866 // Backward gotos reinit here while forward gotos are recorded for
867 // EmitAutoVarAlloca to patch once the alloca exists. Skip when jump sources
868 // are unknown (computed goto); EmitAutoVarAlloca then uses function-scope
869 // init.
870 if (HaveInsertPoint() && !Bypasses.isAlwaysBypassed()) {
871 emitBypassedVarInitsForSource(Source: &S);
872 BypassingForwardJumps.push_back(Elt: {.Block: Builder.GetInsertBlock(), .Source: &S});
873 }
874
875 ApplyAtomGroup Grp(getDebugInfo());
876 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: S.getLabel()));
877}
878
879
880void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
881 ApplyAtomGroup Grp(getDebugInfo());
882 if (const LabelDecl *Target = S.getConstantTarget()) {
883 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: Target));
884 return;
885 }
886
887 // Ensure that we have an i8* for our PHI node.
888 llvm::Value *V = Builder.CreateBitCast(V: EmitScalarExpr(E: S.getTarget()),
889 DestTy: Int8PtrTy, Name: "addr");
890 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
891
892 // Get the basic block for the indirect goto.
893 llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
894
895 // The first instruction in the block has to be the PHI for the switch dest,
896 // add an entry for this branch.
897 cast<llvm::PHINode>(Val: IndGotoBB->begin())->addIncoming(V, BB: CurBB);
898
899 EmitBranch(Target: IndGotoBB);
900 if (CurBB && CurBB->hasTerminator())
901 addInstToCurrentSourceAtom(KeyInstruction: CurBB->getTerminator(), Backup: nullptr);
902}
903
904void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
905 const Stmt *Else = S.getElse();
906
907 // The else branch of a consteval if statement is always the only branch that
908 // can be runtime evaluated.
909 if (S.isConsteval()) {
910 const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;
911 if (Executed) {
912 RunCleanupsScope ExecutedScope(*this);
913 EmitStmt(S: Executed);
914 }
915 return;
916 }
917
918 // C99 6.8.4.1: The first substatement is executed if the expression compares
919 // unequal to 0. The condition must be a scalar type.
920 LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
921 ApplyDebugLocation DL(*this, S.getCond());
922
923 if (S.getInit()) {
924 EmitStmt(S: S.getInit());
925
926 // The init statement may have cleared the insertion point (e.g. it ended in
927 // a 'noreturn' call); the condition emitted below needs a valid one.
928 EnsureInsertPoint();
929 }
930
931 if (S.getConditionVariable())
932 EmitDecl(D: *S.getConditionVariable());
933
934 // If the condition constant folds and can be elided, try to avoid emitting
935 // the condition and the dead arm of the if/else.
936 bool CondConstant;
937 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: CondConstant,
938 AllowLabels: S.isConstexpr())) {
939 // Figure out which block (then or else) is executed.
940 const Stmt *Executed = S.getThen();
941 const Stmt *Skipped = Else;
942 if (!CondConstant) // Condition false?
943 std::swap(a&: Executed, b&: Skipped);
944
945 // If the skipped block has no labels in it, just emit the executed block.
946 // This avoids emitting dead code and simplifies the CFG substantially.
947 if (S.isConstexpr() || !ContainsLabel(S: Skipped)) {
948 incrementProfileCounter(ExecSkip: CondConstant ? UseExecPath : UseSkipPath, S: &S,
949 /*UseBoth=*/true);
950 if (Executed) {
951 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
952 RunCleanupsScope ExecutedScope(*this);
953 EmitStmt(S: Executed);
954 }
955 PGO->markStmtMaybeUsed(S: Skipped);
956 return;
957 }
958 }
959
960 auto HasSkip = hasSkipCounter(S: &S);
961
962 // Otherwise, the condition did not fold, or we couldn't elide it. Just emit
963 // the conditional branch.
964 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.then");
965 llvm::BasicBlock *ContBlock = createBasicBlock(name: "if.end");
966 llvm::BasicBlock *ElseBlock =
967 (Else || HasSkip ? createBasicBlock(name: "if.else") : ContBlock);
968 // Prefer the PGO based weights over the likelihood attribute.
969 // When the build isn't optimized the metadata isn't used, so don't generate
970 // it.
971 // Also, differentiate between disabled PGO and a never executed branch with
972 // PGO. Assuming PGO is in use:
973 // - we want to ignore the [[likely]] attribute if the branch is never
974 // executed,
975 // - assuming the profile is poor, preserving the attribute may still be
976 // beneficial.
977 // As an approximation, preserve the attribute only if both the branch and the
978 // parent context were not executed.
979 Stmt::Likelihood LH = Stmt::LH_None;
980 uint64_t ThenCount = getProfileCount(S: S.getThen());
981 if (!ThenCount && !getCurrentProfileCount() &&
982 CGM.getCodeGenOpts().OptimizationLevel)
983 LH = Stmt::getLikelihood(Then: S.getThen(), Else);
984
985 // When measuring MC/DC, always fully evaluate the condition up front using
986 // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
987 // executing the body of the if.then or if.else. This is useful for when
988 // there is a 'return' within the body, but this is particularly beneficial
989 // when one if-stmt is nested within another if-stmt so that all of the MC/DC
990 // updates are kept linear and consistent.
991 if (!CGM.getCodeGenOpts().MCDCCoverage) {
992 EmitBranchOnBoolExpr(Cond: S.getCond(), TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount: ThenCount, LH,
993 /*ConditionalOp=*/nullptr,
994 /*ConditionalDecl=*/S.getConditionVariable());
995 } else {
996 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
997 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
998 Builder.CreateCondBr(Cond: BoolCondVal, True: ThenBlock, False: ElseBlock);
999 }
1000
1001 // Emit the 'then' code.
1002 EmitBlock(BB: ThenBlock);
1003 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1004 {
1005 RunCleanupsScope ThenScope(*this);
1006 EmitStmt(S: S.getThen());
1007 }
1008 EmitBranch(Target: ContBlock);
1009
1010 // Emit the 'else' code if present.
1011 if (Else) {
1012 {
1013 // There is no need to emit line number for an unconditional branch.
1014 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1015 EmitBlock(BB: ElseBlock);
1016 }
1017 // Add a counter to else block unless it has CounterExpr.
1018 if (HasSkip)
1019 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1020 {
1021 RunCleanupsScope ElseScope(*this);
1022 EmitStmt(S: Else);
1023 }
1024 {
1025 // There is no need to emit line number for an unconditional branch.
1026 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1027 EmitBranch(Target: ContBlock);
1028 }
1029 } else if (HasSkip) {
1030 EmitBlock(BB: ElseBlock);
1031 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1032 EmitBranch(Target: ContBlock);
1033 }
1034
1035 // Emit the continuation block for code after the if.
1036 EmitBlock(BB: ContBlock, IsFinished: true);
1037}
1038
1039bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression,
1040 bool HasEmptyBody) {
1041 return CodeGenUtils::checkIfLoopMustProgress(
1042 LangOpts: getLangOpts(), CGOpts: CGM.getCodeGenOpts(), Ctx&: getContext(), ControllingExpression,
1043 HasEmptyBody,
1044 RemoveMustProgress: [this] { CurFn->removeFnAttr(Kind: llvm::Attribute::MustProgress); });
1045}
1046
1047void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
1048 ArrayRef<const Attr *> WhileAttrs) {
1049 // Emit the header for the loop, which will also become
1050 // the continue target.
1051 JumpDest LoopHeader = getJumpDestInCurrentScope(Name: "while.cond");
1052 EmitBlock(BB: LoopHeader.getBlock());
1053
1054 if (CGM.shouldEmitConvergenceTokens())
1055 ConvergenceTokenStack.push_back(
1056 Elt: emitConvergenceLoopToken(BB: LoopHeader.getBlock()));
1057
1058 // Create an exit block for when the condition fails, which will
1059 // also become the break target.
1060 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "while.end");
1061
1062 // Store the blocks to use for break and continue.
1063 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopHeader));
1064
1065 // C++ [stmt.while]p2:
1066 // When the condition of a while statement is a declaration, the
1067 // scope of the variable that is declared extends from its point
1068 // of declaration (3.3.2) to the end of the while statement.
1069 // [...]
1070 // The object created in a condition is destroyed and created
1071 // with each iteration of the loop.
1072 RunCleanupsScope ConditionScope(*this);
1073
1074 if (S.getConditionVariable())
1075 EmitDecl(D: *S.getConditionVariable());
1076
1077 // Evaluate the conditional in the while header. C99 6.8.5.1: The
1078 // evaluation of the controlling expression takes place before each
1079 // execution of the loop body.
1080 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1081
1082 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1083
1084 // while(1) is common, avoid extra exit blocks. Be sure
1085 // to correctly handle break/continue though.
1086 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1087 bool EmitBoolCondBranch = !C || !C->isOne();
1088 const SourceRange &R = S.getSourceRange();
1089 LoopStack.push(
1090 Header: LoopHeader.getBlock(), Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: WhileAttrs,
1091 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()), EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1092 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: CodeGenUtils::hasEmptyLoopBody(S)));
1093
1094 // As long as the condition is true, go to the loop body.
1095 llvm::BasicBlock *LoopBody = createBasicBlock(name: "while.body");
1096 if (EmitBoolCondBranch) {
1097 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1098 if (hasSkipCounter(S: &S) || ConditionScope.requiresCleanups())
1099 ExitBlock = createBasicBlock(name: "while.exit");
1100 llvm::MDNode *Weights =
1101 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1102 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1103 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1104 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1105 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock, BranchWeights: Weights);
1106 // Key Instructions: Emit the condition and branch as separate source
1107 // location atoms otherwise we may omit a step onto the loop condition in
1108 // favour of the `while` keyword.
1109 // FIXME: We could have the branch as the backup location for the condition,
1110 // which would probably be a better experience. Explore this later.
1111 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1112 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1113 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1114
1115 if (ExitBlock != LoopExit.getBlock()) {
1116 EmitBlock(BB: ExitBlock);
1117 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1118 EmitBranchThroughCleanup(Dest: LoopExit);
1119 }
1120 } else if (const Attr *A = Stmt::getLikelihoodAttr(S: S.getBody())) {
1121 CGM.getDiags().Report(Loc: A->getLocation(),
1122 DiagID: diag::warn_attribute_has_no_effect_on_infinite_loop)
1123 << A << A->getRange();
1124 CGM.getDiags().Report(
1125 Loc: S.getWhileLoc(),
1126 DiagID: diag::note_attribute_has_no_effect_on_infinite_loop_here)
1127 << SourceRange(S.getWhileLoc(), S.getRParenLoc());
1128 }
1129
1130 // Emit the loop body. We have to emit this in a cleanup scope
1131 // because it might be a singleton DeclStmt.
1132 {
1133 RunCleanupsScope BodyScope(*this);
1134 EmitBlock(BB: LoopBody);
1135 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1136 EmitStmt(S: S.getBody());
1137 }
1138
1139 BreakContinueStack.pop_back();
1140
1141 // Immediately force cleanup.
1142 ConditionScope.ForceCleanup();
1143
1144 EmitStopPoint(S: &S);
1145 // Branch to the loop header again.
1146 EmitBranch(Target: LoopHeader.getBlock());
1147
1148 LoopStack.pop();
1149
1150 // Emit the exit block.
1151 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1152
1153 // The LoopHeader typically is just a branch if we skipped emitting
1154 // a branch, try to erase it.
1155 if (!EmitBoolCondBranch) {
1156 SimplifyForwardingBlocks(BB: LoopHeader.getBlock());
1157 PGO->markStmtAsUsed(Skipped: true, S: &S);
1158 }
1159
1160 if (CGM.shouldEmitConvergenceTokens())
1161 ConvergenceTokenStack.pop_back();
1162}
1163
1164void CodeGenFunction::EmitDoStmt(const DoStmt &S,
1165 ArrayRef<const Attr *> DoAttrs) {
1166 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "do.end");
1167 JumpDest LoopCond = getJumpDestInCurrentScope(Name: "do.cond");
1168
1169 uint64_t ParentCount = getCurrentProfileCount();
1170
1171 // Store the blocks to use for break and continue.
1172 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopCond));
1173
1174 // Emit the body of the loop.
1175 llvm::BasicBlock *LoopBody = createBasicBlock(name: "do.body");
1176
1177 EmitBlockWithFallThrough(BB: LoopBody, S: &S);
1178
1179 if (CGM.shouldEmitConvergenceTokens())
1180 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: LoopBody));
1181
1182 {
1183 RunCleanupsScope BodyScope(*this);
1184 EmitStmt(S: S.getBody());
1185 }
1186
1187 EmitBlock(BB: LoopCond.getBlock());
1188
1189 // C99 6.8.5.2: "The evaluation of the controlling expression takes place
1190 // after each execution of the loop body."
1191
1192 // Evaluate the conditional in the while header.
1193 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1194 // compares unequal to 0. The condition must be a scalar type.
1195 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1196
1197 BreakContinueStack.pop_back();
1198
1199 // "do {} while (0)" is common in macros, avoid extra blocks. Be sure
1200 // to correctly handle break/continue though.
1201 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1202 bool EmitBoolCondBranch = !C || !C->isZero();
1203
1204 const SourceRange &R = S.getSourceRange();
1205 LoopStack.push(
1206 Header: LoopBody, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: DoAttrs,
1207 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()), EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1208 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: CodeGenUtils::hasEmptyLoopBody(S)));
1209
1210 auto *LoopFalse = (hasSkipCounter(S: &S) ? createBasicBlock(name: "do.loopfalse")
1211 : LoopExit.getBlock());
1212
1213 // As long as the condition is true, iterate the loop.
1214 if (EmitBoolCondBranch) {
1215 uint64_t BackedgeCount = getProfileCount(S: S.getBody()) - ParentCount;
1216 auto *I = Builder.CreateCondBr(
1217 Cond: BoolCondVal, True: LoopBody, False: LoopFalse,
1218 BranchWeights: createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: BackedgeCount));
1219
1220 // Key Instructions: Emit the condition and branch as separate source
1221 // location atoms otherwise we may omit a step onto the loop condition in
1222 // favour of the closing brace.
1223 // FIXME: We could have the branch as the backup location for the condition,
1224 // which would probably be a better experience (no jumping to the brace).
1225 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1226 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1227 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1228 }
1229
1230 LoopStack.pop();
1231
1232 if (LoopFalse != LoopExit.getBlock()) {
1233 EmitBlock(BB: LoopFalse);
1234 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S, /*UseBoth=*/true);
1235 }
1236
1237 // Emit the exit block.
1238 EmitBlock(BB: LoopExit.getBlock());
1239
1240 // The DoCond block typically is just a branch if we skipped
1241 // emitting a branch, try to erase it.
1242 if (!EmitBoolCondBranch)
1243 SimplifyForwardingBlocks(BB: LoopCond.getBlock());
1244
1245 if (CGM.shouldEmitConvergenceTokens())
1246 ConvergenceTokenStack.pop_back();
1247}
1248
1249void CodeGenFunction::EmitForStmt(const ForStmt &S,
1250 ArrayRef<const Attr *> ForAttrs) {
1251 std::optional<LexicalScope> ForScope;
1252 JumpDest LoopExit;
1253 if (getLangOpts().C99 || getLangOpts().CPlusPlus) {
1254 LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1255 ForScope.emplace(args&: *this, args: S.getSourceRange());
1256
1257 // Evaluate the first part before the loop.
1258 if (S.getInit())
1259 EmitStmt(S: S.getInit());
1260 } else {
1261 // Before C99 the init variable belongs to the enclosing scope, so create
1262 // the loop exit after its cleanups: leaving the loop must not run them.
1263 if (S.getInit())
1264 EmitStmt(S: S.getInit());
1265 LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1266 }
1267
1268 // Start the loop with a block that tests the condition.
1269 // If there's an increment, the continue scope will be overwritten
1270 // later.
1271 JumpDest CondDest = getJumpDestInCurrentScope(Name: "for.cond");
1272 llvm::BasicBlock *CondBlock = CondDest.getBlock();
1273 EmitBlock(BB: CondBlock);
1274
1275 if (CGM.shouldEmitConvergenceTokens())
1276 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1277
1278 const SourceRange &R = S.getSourceRange();
1279 LoopStack.push(
1280 Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1281 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()), EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1282 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: CodeGenUtils::hasEmptyLoopBody(S)));
1283
1284 // Create a cleanup scope for the condition variable cleanups.
1285 LexicalScope ConditionScope(*this, S.getSourceRange());
1286
1287 // If the for loop doesn't have an increment we can just use the condition as
1288 // the continue block. Otherwise, if there is no condition variable, we can
1289 // form the continue block now. If there is a condition variable, we can't
1290 // form the continue block until after we've emitted the condition, because
1291 // the condition is in scope in the increment, but Sema's jump diagnostics
1292 // ensure that there are no continues from the condition variable that jump
1293 // to the loop increment.
1294 JumpDest Continue;
1295 if (!S.getInc())
1296 Continue = CondDest;
1297 else if (!S.getConditionVariable())
1298 Continue = getJumpDestInCurrentScope(Name: "for.inc");
1299 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1300
1301 if (S.getCond()) {
1302 // If the for statement has a condition scope, emit the local variable
1303 // declaration.
1304 if (S.getConditionVariable()) {
1305 EmitDecl(D: *S.getConditionVariable());
1306
1307 // We have entered the condition variable's scope, so we're now able to
1308 // jump to the continue block.
1309 Continue = S.getInc() ? getJumpDestInCurrentScope(Name: "for.inc") : CondDest;
1310 BreakContinueStack.back().ContinueBlock = Continue;
1311 }
1312
1313 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1314 // If there are any cleanups between here and the loop-exit scope,
1315 // create a block to stage a loop exit along.
1316 if (hasSkipCounter(S: &S) || (ForScope && ForScope->requiresCleanups()))
1317 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1318
1319 // As long as the condition is true, iterate the loop.
1320 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1321
1322 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1323 // compares unequal to 0. The condition must be a scalar type.
1324 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1325
1326 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1327
1328 llvm::MDNode *Weights =
1329 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1330 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1331 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1332 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1333
1334 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1335 // Key Instructions: Emit the condition and branch as separate atoms to
1336 // match existing loop stepping behaviour. FIXME: We could have the branch
1337 // as the backup location for the condition, which would probably be a
1338 // better experience (no jumping to the brace).
1339 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1340 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1341 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1342
1343 if (ExitBlock != LoopExit.getBlock()) {
1344 EmitBlock(BB: ExitBlock);
1345 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1346 EmitBranchThroughCleanup(Dest: LoopExit);
1347 }
1348
1349 EmitBlock(BB: ForBody);
1350 } else {
1351 // Treat it as a non-zero constant. Don't even create a new block for the
1352 // body, just fall into it.
1353 PGO->markStmtAsUsed(Skipped: true, S: &S);
1354 }
1355
1356 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1357
1358 {
1359 // Create a separate cleanup scope for the body, in case it is not
1360 // a compound statement.
1361 RunCleanupsScope BodyScope(*this);
1362 EmitStmt(S: S.getBody());
1363 }
1364
1365 // The last block in the loop's body (which unconditionally branches to the
1366 // `inc` block if there is one).
1367 auto *FinalBodyBB = Builder.GetInsertBlock();
1368
1369 // If there is an increment, emit it next.
1370 if (S.getInc()) {
1371 EmitBlock(BB: Continue.getBlock());
1372 EmitStmt(S: S.getInc());
1373 }
1374
1375 BreakContinueStack.pop_back();
1376
1377 ConditionScope.ForceCleanup();
1378
1379 EmitStopPoint(S: &S);
1380 EmitBranch(Target: CondBlock);
1381
1382 if (ForScope)
1383 ForScope->ForceCleanup();
1384
1385 LoopStack.pop();
1386
1387 // Emit the fall-through block.
1388 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1389
1390 if (CGM.shouldEmitConvergenceTokens())
1391 ConvergenceTokenStack.pop_back();
1392
1393 if (FinalBodyBB) {
1394 // Key Instructions: We want the for closing brace to be step-able on to
1395 // match existing behaviour.
1396 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1397 }
1398}
1399
1400void
1401CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1402 ArrayRef<const Attr *> ForAttrs) {
1403 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1404
1405 LexicalScope ForScope(*this, S.getSourceRange());
1406
1407 // Evaluate the first pieces before the loop.
1408 if (S.getInit())
1409 EmitStmt(S: S.getInit());
1410 EmitStmt(S: S.getRangeStmt());
1411 EmitStmt(S: S.getBeginStmt());
1412 EmitStmt(S: S.getEndStmt());
1413
1414 // Start the loop with a block that tests the condition.
1415 // If there's an increment, the continue scope will be overwritten
1416 // later.
1417 llvm::BasicBlock *CondBlock = createBasicBlock(name: "for.cond");
1418 EmitBlock(BB: CondBlock);
1419
1420 if (CGM.shouldEmitConvergenceTokens())
1421 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1422
1423 const SourceRange &R = S.getSourceRange();
1424 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1425 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1426 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
1427
1428 // If there are any cleanups between here and the loop-exit scope,
1429 // create a block to stage a loop exit along.
1430 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1431 if (hasSkipCounter(S: &S) || ForScope.requiresCleanups())
1432 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1433
1434 // The loop body, consisting of the specified body and the loop variable.
1435 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1436
1437 // The body is executed if the expression, contextually converted
1438 // to bool, is true.
1439 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1440 llvm::MDNode *Weights =
1441 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1442 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1443 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1444 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1445 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1446 // Key Instructions: Emit the condition and branch as separate atoms to
1447 // match existing loop stepping behaviour. FIXME: We could have the branch as
1448 // the backup location for the condition, which would probably be a better
1449 // experience.
1450 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1451 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1452 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1453
1454 if (ExitBlock != LoopExit.getBlock()) {
1455 EmitBlock(BB: ExitBlock);
1456 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1457 EmitBranchThroughCleanup(Dest: LoopExit);
1458 }
1459
1460 EmitBlock(BB: ForBody);
1461 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1462
1463 // Create a block for the increment. In case of a 'continue', we jump there.
1464 JumpDest Continue = getJumpDestInCurrentScope(Name: "for.inc");
1465
1466 // Store the blocks to use for break and continue.
1467 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1468
1469 {
1470 // Create a separate cleanup scope for the loop variable and body.
1471 LexicalScope BodyScope(*this, S.getSourceRange());
1472 EmitStmt(S: S.getLoopVarStmt());
1473 EmitStmt(S: S.getBody());
1474 }
1475 // The last block in the loop's body (which unconditionally branches to the
1476 // `inc` block if there is one).
1477 auto *FinalBodyBB = Builder.GetInsertBlock();
1478
1479 EmitStopPoint(S: &S);
1480 // If there is an increment, emit it next.
1481 EmitBlock(BB: Continue.getBlock());
1482 EmitStmt(S: S.getInc());
1483
1484 BreakContinueStack.pop_back();
1485
1486 EmitBranch(Target: CondBlock);
1487
1488 ForScope.ForceCleanup();
1489
1490 LoopStack.pop();
1491
1492 // Emit the fall-through block.
1493 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1494
1495 if (CGM.shouldEmitConvergenceTokens())
1496 ConvergenceTokenStack.pop_back();
1497
1498 if (FinalBodyBB) {
1499 // We want the for closing brace to be step-able on to match existing
1500 // behaviour.
1501 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1502 }
1503}
1504
1505void CodeGenFunction::EmitCXXExpansionStmtInstantiation(
1506 const CXXExpansionStmtInstantiation &S) {
1507 LexicalScope Scope(*this, S.getSourceRange());
1508
1509 for (const Stmt *DS : S.getPreambleStmts())
1510 EmitStmt(S: DS);
1511
1512 if (S.getInstantiations().empty())
1513 return;
1514
1515 JumpDest ExpandExit = getJumpDestInCurrentScope(Name: "expand.end");
1516 JumpDest ContinueDest;
1517 for (auto [N, Inst] : enumerate(First: S.getInstantiations())) {
1518 if (N == S.getInstantiations().size() - 1)
1519 ContinueDest = ExpandExit;
1520 else
1521 ContinueDest = getJumpDestInCurrentScope(Name: "expand.next");
1522
1523 LexicalScope ExpansionScope(*this, Inst->getSourceRange());
1524 BreakContinueStack.push_back(Elt: BreakContinue(S, ExpandExit, ContinueDest));
1525 EmitStmt(S: Inst);
1526 BreakContinueStack.pop_back();
1527 EmitBlock(BB: ContinueDest.getBlock(), IsFinished: true);
1528 }
1529}
1530
1531void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1532 if (RV.isScalar()) {
1533 Builder.CreateStore(Val: RV.getScalarVal(), Addr: ReturnValue);
1534 } else if (RV.isAggregate()) {
1535 LValue Dest = MakeAddrLValue(Addr: ReturnValue, T: Ty);
1536 LValue Src = MakeAddrLValue(Addr: RV.getAggregateAddress(), T: Ty);
1537 EmitAggregateCopy(Dest, Src, EltTy: Ty, MayOverlap: getOverlapForReturnValue());
1538 } else {
1539 EmitStoreOfComplex(V: RV.getComplexVal(), dest: MakeAddrLValue(Addr: ReturnValue, T: Ty),
1540 /*init*/ isInit: true);
1541 }
1542 EmitBranchThroughCleanup(Dest: ReturnBlock);
1543}
1544
1545namespace {
1546// RAII struct used to save and restore a return statment's result expression.
1547struct SaveRetExprRAII {
1548 SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1549 : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1550 CGF.RetExpr = RetExpr;
1551 }
1552 ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1553 const Expr *OldRetExpr;
1554 CodeGenFunction &CGF;
1555};
1556} // namespace
1557
1558/// Determine if the given call uses the swiftasync calling convention.
1559static bool isSwiftAsyncCallee(const CallExpr *CE) {
1560 auto calleeQualType = CE->getCallee()->getType();
1561 const FunctionType *calleeType = nullptr;
1562 if (calleeQualType->isFunctionPointerType() ||
1563 calleeQualType->isFunctionReferenceType() ||
1564 calleeQualType->isBlockPointerType() ||
1565 calleeQualType->isMemberFunctionPointerType()) {
1566 calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1567 } else if (auto *ty = dyn_cast<FunctionType>(Val&: calleeQualType)) {
1568 calleeType = ty;
1569 } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(Val: CE)) {
1570 if (auto methodDecl = CMCE->getMethodDecl()) {
1571 // getMethodDecl() doesn't handle member pointers at the moment.
1572 calleeType = methodDecl->getType()->castAs<FunctionType>();
1573 } else {
1574 return false;
1575 }
1576 } else {
1577 return false;
1578 }
1579 return calleeType->getCallConv() == CallingConv::CC_SwiftAsync;
1580}
1581
1582/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1583/// if the function returns void, or may be missing one if the function returns
1584/// non-void. Fun stuff :).
1585void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1586 ApplyAtomGroup Grp(getDebugInfo());
1587 if (requiresReturnValueCheck()) {
1588 llvm::Constant *SLoc = EmitCheckSourceLocation(Loc: S.getBeginLoc());
1589 auto *SLocPtr =
1590 new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1591 llvm::GlobalVariable::PrivateLinkage, SLoc);
1592 SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1593 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: SLocPtr);
1594 assert(ReturnLocation.isValid() && "No valid return location");
1595 Builder.CreateStore(Val: SLocPtr, Addr: ReturnLocation);
1596 }
1597
1598 // Returning from an outlined SEH helper is UB, and we already warn on it.
1599 if (IsOutlinedSEHHelper) {
1600 Builder.CreateUnreachable();
1601 Builder.ClearInsertionPoint();
1602 }
1603
1604 // Emit the result value, even if unused, to evaluate the side effects.
1605 const Expr *RV = S.getRetValue();
1606
1607 // Record the result expression of the return statement. The recorded
1608 // expression is used to determine whether a block capture's lifetime should
1609 // end at the end of the full expression as opposed to the end of the scope
1610 // enclosing the block expression.
1611 //
1612 // This permits a small, easily-implemented exception to our over-conservative
1613 // rules about not jumping to statements following block literals with
1614 // non-trivial cleanups.
1615 SaveRetExprRAII SaveRetExpr(RV, *this);
1616
1617 RunCleanupsScope cleanupScope(*this);
1618 if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(Val: RV))
1619 RV = EWC->getSubExpr();
1620
1621 // If we're in a swiftasynccall function, and the return expression is a
1622 // call to a swiftasynccall function, mark the call as the musttail call.
1623 std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail;
1624 if (RV && CurFnInfo &&
1625 CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) {
1626 if (auto CE = dyn_cast<CallExpr>(Val: RV)) {
1627 if (isSwiftAsyncCallee(CE)) {
1628 SaveMustTail.emplace(args&: MustTailCall, args&: CE);
1629 }
1630 }
1631 }
1632
1633 // FIXME: Clean this up by using an LValue for ReturnTemp,
1634 // EmitStoreThroughLValue, and EmitAnyExpr.
1635 // Check if the NRVO candidate was not globalized in OpenMP mode.
1636 if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1637 S.getNRVOCandidate()->isNRVOVariable() &&
1638 (!getLangOpts().OpenMP ||
1639 !CGM.getOpenMPRuntime()
1640 .getAddressOfLocalVariable(CGF&: *this, VD: S.getNRVOCandidate())
1641 .isValid())) {
1642 // Apply the named return value optimization for this return statement,
1643 // which means doing nothing: the appropriate result has already been
1644 // constructed into the NRVO variable.
1645
1646 // If there is an NRVO flag for this variable, set it to 1 into indicate
1647 // that the cleanup code should not destroy the variable.
1648 if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1649 Builder.CreateFlagStore(Value: Builder.getTrue(), Addr: NRVOFlag);
1650 } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1651 // Make sure not to return anything, but evaluate the expression
1652 // for side effects.
1653 if (RV) {
1654 EmitAnyExpr(E: RV);
1655 }
1656 } else if (!RV) {
1657 // Do nothing (return value is left uninitialized)
1658 } else if (FnRetTy->isReferenceType()) {
1659 // If this function returns a reference, take the address of the expression
1660 // rather than the value.
1661 RValue Result = EmitReferenceBindingToExpr(E: RV);
1662 auto *I = Builder.CreateStore(Val: Result.getScalarVal(), Addr: ReturnValue);
1663 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1664 } else {
1665 switch (getEvaluationKind(T: RV->getType())) {
1666 case TEK_Scalar: {
1667 llvm::Value *Ret = EmitScalarExpr(E: RV);
1668 if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1669 EmitStoreOfScalar(value: Ret, lvalue: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1670 /*isInit*/ true);
1671 } else {
1672 auto *I = Builder.CreateStore(Val: Ret, Addr: ReturnValue);
1673 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1674 }
1675 break;
1676 }
1677 case TEK_Complex:
1678 EmitComplexExprIntoLValue(E: RV, dest: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1679 /*isInit*/ true);
1680 break;
1681 case TEK_Aggregate:
1682 EmitAggExpr(E: RV, AS: AggValueSlot::forAddr(
1683 addr: ReturnValue, quals: Qualifiers(),
1684 isDestructed: AggValueSlot::IsDestructed,
1685 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1686 isAliased: AggValueSlot::IsNotAliased,
1687 mayOverlap: getOverlapForReturnValue()));
1688 break;
1689 }
1690 }
1691
1692 ++NumReturnExprs;
1693 if (!RV || RV->isEvaluatable(Ctx: getContext()))
1694 ++NumSimpleReturnExprs;
1695
1696 cleanupScope.ForceCleanup();
1697 EmitBranchThroughCleanup(Dest: ReturnBlock);
1698}
1699
1700void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1701 // As long as debug info is modeled with instructions, we have to ensure we
1702 // have a place to insert here and write the stop point here.
1703 if (HaveInsertPoint())
1704 EmitStopPoint(S: &S);
1705
1706 for (const auto *I : S.decls())
1707 EmitDecl(D: *I, /*EvaluateConditionDecl=*/true);
1708}
1709
1710auto CodeGenFunction::GetDestForLoopControlStmt(const LoopControlStmt &S)
1711 -> const BreakContinue * {
1712 if (!S.isNamed())
1713 return &BreakContinueStack.back();
1714
1715 const Stmt *LoopOrSwitch = S.getNamedLoopOrSwitch();
1716 assert(LoopOrSwitch && "break/continue target label not available?");
1717 for (const BreakContinue &BC : llvm::reverse(C&: BreakContinueStack))
1718 if (BC.LoopOrSwitch == LoopOrSwitch)
1719 return &BC;
1720
1721 llvm_unreachable("break/continue target not found");
1722}
1723
1724void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1725 assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1726
1727 // If this code is reachable then emit a stop point (if generating
1728 // debug info). We have to do this ourselves because we are on the
1729 // "simple" statement path.
1730 if (HaveInsertPoint())
1731 EmitStopPoint(S: &S);
1732
1733 ApplyAtomGroup Grp(getDebugInfo());
1734 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->BreakBlock);
1735}
1736
1737void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1738 assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1739
1740 // If this code is reachable then emit a stop point (if generating
1741 // debug info). We have to do this ourselves because we are on the
1742 // "simple" statement path.
1743 if (HaveInsertPoint())
1744 EmitStopPoint(S: &S);
1745
1746 ApplyAtomGroup Grp(getDebugInfo());
1747 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->ContinueBlock);
1748}
1749
1750/// EmitCaseStmtRange - If case statement range is not too big then
1751/// add multiple cases to switch instruction, one for each value within
1752/// the range. If range is too big then emit "if" condition check.
1753void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1754 ArrayRef<const Attr *> Attrs) {
1755 assert(S.getRHS() && "Expected RHS value in CaseStmt");
1756
1757 llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(Ctx: getContext());
1758 llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(Ctx: getContext());
1759
1760 // Emit the code for this case. We do this first to make sure it is
1761 // properly chained from our predecessor before generating the
1762 // switch machinery to enter this block.
1763 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1764 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1765 EmitStmt(S: S.getSubStmt());
1766
1767 // If range is empty, do nothing.
1768 if (LHS.isSigned() ? RHS.slt(RHS: LHS) : RHS.ult(RHS: LHS))
1769 return;
1770
1771 Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1772 llvm::APInt Range = RHS - LHS;
1773 // FIXME: parameters such as this should not be hardcoded.
1774 if (Range.getBitWidth() < 7 ||
1775 Range.ult(RHS: llvm::APInt(Range.getBitWidth(), 64))) {
1776 // Range is small enough to add multiple switch instruction cases.
1777 uint64_t Total = getProfileCount(S: &S);
1778 unsigned NCases = Range.getZExtValue() + 1;
1779 // We only have one region counter for the entire set of cases here, so we
1780 // need to divide the weights evenly between the generated cases, ensuring
1781 // that the total weight is preserved. E.g., a weight of 5 over three cases
1782 // will be distributed as weights of 2, 2, and 1.
1783 uint64_t Weight = Total / NCases, Rem = Total % NCases;
1784 for (unsigned I = 0; I != NCases; ++I) {
1785 if (SwitchWeights)
1786 SwitchWeights->push_back(Elt: Weight + (Rem ? 1 : 0));
1787 else if (SwitchLikelihood)
1788 SwitchLikelihood->push_back(Elt: LH);
1789
1790 if (Rem)
1791 Rem--;
1792 SwitchInsn->addCase(OnVal: Builder.getInt(AI: LHS), Dest: CaseDest);
1793 ++LHS;
1794 }
1795 return;
1796 }
1797
1798 // The range is too big. Emit "if" condition into a new block,
1799 // making sure to save and restore the current insertion point.
1800 llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1801
1802 // Push this test onto the chain of range checks (which terminates
1803 // in the default basic block). The switch's default will be changed
1804 // to the top of this chain after switch emission is complete.
1805 llvm::BasicBlock *FalseDest = CaseRangeBlock;
1806 CaseRangeBlock = createBasicBlock(name: "sw.caserange");
1807
1808 CurFn->insert(Position: CurFn->end(), BB: CaseRangeBlock);
1809 Builder.SetInsertPoint(CaseRangeBlock);
1810
1811 // Emit range check.
1812 llvm::Value *Diff =
1813 Builder.CreateSub(LHS: SwitchInsn->getCondition(), RHS: Builder.getInt(AI: LHS));
1814 llvm::Value *Cond =
1815 Builder.CreateICmpULE(LHS: Diff, RHS: Builder.getInt(AI: Range), Name: "inbounds");
1816
1817 llvm::MDNode *Weights = nullptr;
1818 if (SwitchWeights) {
1819 uint64_t ThisCount = getProfileCount(S: &S);
1820 uint64_t DefaultCount = (*SwitchWeights)[0];
1821 Weights = createProfileWeights(TrueCount: ThisCount, FalseCount: DefaultCount);
1822
1823 // Since we're chaining the switch default through each large case range, we
1824 // need to update the weight for the default, ie, the first case, to include
1825 // this case.
1826 (*SwitchWeights)[0] += ThisCount;
1827 } else if (SwitchLikelihood)
1828 Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1829
1830 Builder.CreateCondBr(Cond, True: CaseDest, False: FalseDest, BranchWeights: Weights);
1831
1832 // Restore the appropriate insertion point.
1833 if (RestoreBB)
1834 Builder.SetInsertPoint(RestoreBB);
1835 else
1836 Builder.ClearInsertionPoint();
1837}
1838
1839void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1840 ArrayRef<const Attr *> Attrs) {
1841 // If there is no enclosing switch instance that we're aware of, then this
1842 // case statement and its block can be elided. This situation only happens
1843 // when we've constant-folded the switch, are emitting the constant case,
1844 // and part of the constant case includes another case statement. For
1845 // instance: switch (4) { case 4: do { case 5: } while (1); }
1846 if (!SwitchInsn) {
1847 EmitStmt(S: S.getSubStmt());
1848 return;
1849 }
1850
1851 // Handle case ranges.
1852 if (S.getRHS()) {
1853 EmitCaseStmtRange(S, Attrs);
1854 return;
1855 }
1856
1857 llvm::ConstantInt *CaseVal =
1858 Builder.getInt(AI: S.getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1859
1860 // Emit debuginfo for the case value if it is an enum value.
1861 // Look through implicit casts and check the subexpression of ConstantExpr.
1862 const auto *CE = dyn_cast<ConstantExpr>(Val: S.getLHS()->IgnoreImpCasts());
1863 const Expr *SubExpr = CE ? CE->getSubExpr() : S.getLHS()->IgnoreImpCasts();
1864 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: SubExpr))
1865 if (CGDebugInfo *Dbg = getDebugInfo())
1866 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
1867 Dbg->EmitGlobalVariable(VD: DRE->getDecl(),
1868 Init: APValue(llvm::APSInt(CaseVal->getValue())));
1869
1870 if (SwitchLikelihood)
1871 SwitchLikelihood->push_back(Elt: Stmt::getLikelihood(Attrs));
1872
1873 // If the body of the case is just a 'break', try to not emit an empty block.
1874 // If we're profiling or we're not optimizing, leave the block in for better
1875 // debug and coverage analysis.
1876 if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1877 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1878 isa<BreakStmt>(Val: S.getSubStmt())) {
1879 JumpDest Block = BreakContinueStack.back().BreakBlock;
1880
1881 // Only do this optimization if there are no cleanups that need emitting.
1882 if (isObviouslyBranchWithoutCleanups(Dest: Block)) {
1883 if (SwitchWeights)
1884 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1885 SwitchInsn->addCase(OnVal: CaseVal, Dest: Block.getBlock());
1886
1887 // If there was a fallthrough into this case, make sure to redirect it to
1888 // the end of the switch as well.
1889 if (Builder.GetInsertBlock()) {
1890 Builder.CreateBr(Dest: Block.getBlock());
1891 Builder.ClearInsertionPoint();
1892 }
1893 return;
1894 }
1895 }
1896
1897 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1898 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1899 if (SwitchWeights)
1900 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1901 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1902
1903 // Recursively emitting the statement is acceptable, but is not wonderful for
1904 // code where we have many case statements nested together, i.e.:
1905 // case 1:
1906 // case 2:
1907 // case 3: etc.
1908 // Handling this recursively will create a new block for each case statement
1909 // that falls through to the next case which is IR intensive. It also causes
1910 // deep recursion which can run into stack depth limitations. Handle
1911 // sequential non-range case statements specially.
1912 //
1913 // TODO When the next case has a likelihood attribute the code returns to the
1914 // recursive algorithm. Maybe improve this case if it becomes common practice
1915 // to use a lot of attributes.
1916 const CaseStmt *CurCase = &S;
1917 const CaseStmt *NextCase = dyn_cast<CaseStmt>(Val: S.getSubStmt());
1918
1919 // Otherwise, iteratively add consecutive cases to this switch stmt.
1920 while (NextCase && NextCase->getRHS() == nullptr) {
1921 CurCase = NextCase;
1922 llvm::ConstantInt *CaseVal =
1923 Builder.getInt(AI: CurCase->getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1924
1925 if (SwitchWeights)
1926 SwitchWeights->push_back(Elt: getProfileCount(S: NextCase));
1927 if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1928 CaseDest = createBasicBlock(name: "sw.bb");
1929 EmitBlockWithFallThrough(BB: CaseDest, S: CurCase);
1930 }
1931 // Since this loop is only executed when the CaseStmt has no attributes
1932 // use a hard-coded value.
1933 if (SwitchLikelihood)
1934 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
1935
1936 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1937 NextCase = dyn_cast<CaseStmt>(Val: CurCase->getSubStmt());
1938 }
1939
1940 // Generate a stop point for debug info if the case statement is
1941 // followed by a default statement. A fallthrough case before a
1942 // default case gets its own branch target.
1943 if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1944 EmitStopPoint(S: CurCase);
1945
1946 // Normal default recursion for non-cases.
1947 EmitStmt(S: CurCase->getSubStmt());
1948}
1949
1950void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1951 ArrayRef<const Attr *> Attrs) {
1952 // If there is no enclosing switch instance that we're aware of, then this
1953 // default statement can be elided. This situation only happens when we've
1954 // constant-folded the switch.
1955 if (!SwitchInsn) {
1956 EmitStmt(S: S.getSubStmt());
1957 return;
1958 }
1959
1960 llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1961 assert(DefaultBlock->empty() &&
1962 "EmitDefaultStmt: Default block already defined?");
1963
1964 if (SwitchLikelihood)
1965 SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1966
1967 EmitBlockWithFallThrough(BB: DefaultBlock, S: &S);
1968
1969 EmitStmt(S: S.getSubStmt());
1970}
1971
1972namespace {
1973struct EmitDeferredStatement final : EHScopeStack::Cleanup {
1974 const DeferStmt &Stmt;
1975 EmitDeferredStatement(const DeferStmt *Stmt) : Stmt(*Stmt) {}
1976
1977 void Emit(CodeGenFunction &CGF, Flags) override {
1978 // Take care that any cleanups pushed by the body of a '_Defer' statement
1979 // don't clobber the current cleanup slot value.
1980 //
1981 // Assume we have a scope that pushes a cleanup; when that scope is exited,
1982 // we need to run that cleanup; this is accomplished by emitting the cleanup
1983 // into a separate block and then branching to that block at scope exit.
1984 //
1985 // Where this gets complicated is if we exit the scope in multiple different
1986 // ways; e.g. in a 'for' loop, we may exit the scope of its body by falling
1987 // off the end (in which case we need to run the cleanup and then branch to
1988 // the increment), or by 'break'ing out of the loop (in which case we need
1989 // to run the cleanup and then branch to the loop exit block); in both cases
1990 // we first branch to the cleanup block to run the cleanup, but the block we
1991 // need to jump to *after* running the cleanup is different.
1992 //
1993 // This is accomplished using a local integer variable called the 'cleanup
1994 // slot': before branching to the cleanup block, we store a value into that
1995 // slot. Then, in the cleanup block, after running the cleanup, we load the
1996 // value of that variable and 'switch' on it to branch to the appropriate
1997 // continuation block.
1998 //
1999 // The problem that arises once '_Defer' statements are involved is that the
2000 // body of a '_Defer' is an arbitrary statement which itself can create more
2001 // cleanups. This means we may end up overwriting the cleanup slot before we
2002 // ever have a chance to 'switch' on it, which means that once we *do* get
2003 // to the 'switch', we end up in whatever block the cleanup code happened to
2004 // pick as the default 'switch' exit label!
2005 //
2006 // That is, what is normally supposed to happen is something like:
2007 //
2008 // 1. Store 'X' to cleanup slot.
2009 // 2. Branch to cleanup block.
2010 // 3. Execute cleanup.
2011 // 4. Read value from cleanup slot.
2012 // 5. Branch to the block associated with 'X'.
2013 //
2014 // But if we encounter a _Defer' statement that contains a cleanup, then
2015 // what might instead happen is:
2016 //
2017 // 1. Store 'X' to cleanup slot.
2018 // 2. Branch to cleanup block.
2019 // 3. Execute cleanup; this ends up pushing another cleanup, so:
2020 // 3a. Store 'Y' to cleanup slot.
2021 // 3b. Run steps 2–5 recursively.
2022 // 4. Read value from cleanup slot, which is now 'Y' instead of 'X'.
2023 // 5. Branch to the block associated with 'Y'... which doesn't even
2024 // exist because the value 'Y' is only meaningful for the inner
2025 // cleanup. The result is we just branch 'somewhere random'.
2026 //
2027 // The rest of the cleanup code simply isn't prepared to handle this case
2028 // because most other cleanups can't push more cleanups, and thus, emitting
2029 // other cleanups generally cannot clobber the cleanup slot.
2030 //
2031 // To prevent this from happening, save the current cleanup slot value and
2032 // restore it after emitting the '_Defer' statement.
2033 llvm::Value *SavedCleanupDest = nullptr;
2034 if (CGF.NormalCleanupDest.isValid())
2035 SavedCleanupDest =
2036 CGF.Builder.CreateLoad(Addr: CGF.NormalCleanupDest, Name: "cleanup.dest.saved");
2037
2038 CGF.EmitStmt(S: Stmt.getBody());
2039
2040 if (SavedCleanupDest && CGF.HaveInsertPoint())
2041 CGF.Builder.CreateStore(Val: SavedCleanupDest, Addr: CGF.NormalCleanupDest);
2042
2043 // Cleanups must end with an insert point.
2044 CGF.EnsureInsertPoint();
2045 }
2046};
2047} // namespace
2048
2049void CodeGenFunction::EmitDeferStmt(const DeferStmt &S) {
2050 EHStack.pushCleanup<EmitDeferredStatement>(Kind: NormalAndEHCleanup, A: &S);
2051}
2052
2053/// CollectStatementsForCase - Given the body of a 'switch' statement and a
2054/// constant value that is being switched on, see if we can dead code eliminate
2055/// the body of the switch to a simple series of statements to emit. Basically,
2056/// on a switch (5) we want to find these statements:
2057/// case 5:
2058/// printf(...); <--
2059/// ++i; <--
2060/// break;
2061///
2062/// and add them to the ResultStmts vector. If it is unsafe to do this
2063/// transformation (for example, one of the elided statements contains a label
2064/// that might be jumped to), return CSFC_Failure. If we handled it and 'S'
2065/// should include statements after it (e.g. the printf() line is a substmt of
2066/// the case) then return CSFC_FallThrough. If we handled it and found a break
2067/// statement, then return CSFC_Success.
2068///
2069/// If Case is non-null, then we are looking for the specified case, checking
2070/// that nothing we jump over contains labels. If Case is null, then we found
2071/// the case and are looking for the break.
2072///
2073/// If the recursive walk actually finds our Case, then we set FoundCase to
2074/// true.
2075///
2076enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
2077static CSFC_Result CollectStatementsForCase(const Stmt *S,
2078 const SwitchCase *Case,
2079 bool &FoundCase,
2080 SmallVectorImpl<const Stmt*> &ResultStmts) {
2081 // If this is a null statement, just succeed.
2082 if (!S)
2083 return Case ? CSFC_Success : CSFC_FallThrough;
2084
2085 // If this is the switchcase (case 4: or default) that we're looking for, then
2086 // we're in business. Just add the substatement.
2087 if (const SwitchCase *SC = dyn_cast<SwitchCase>(Val: S)) {
2088 if (S == Case) {
2089 FoundCase = true;
2090 return CollectStatementsForCase(S: SC->getSubStmt(), Case: nullptr, FoundCase,
2091 ResultStmts);
2092 }
2093
2094 // Otherwise, this is some other case or default statement, just ignore it.
2095 return CollectStatementsForCase(S: SC->getSubStmt(), Case, FoundCase,
2096 ResultStmts);
2097 }
2098
2099 // If we are in the live part of the code and we found our break statement,
2100 // return a success!
2101 if (!Case && isa<BreakStmt>(Val: S))
2102 return CSFC_Success;
2103
2104 // If this is a switch statement, then it might contain the SwitchCase, the
2105 // break, or neither.
2106 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: S)) {
2107 // Handle this as two cases: we might be looking for the SwitchCase (if so
2108 // the skipped statements must be skippable) or we might already have it.
2109 CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
2110 bool StartedInLiveCode = FoundCase;
2111 unsigned StartSize = ResultStmts.size();
2112
2113 // If we've not found the case yet, scan through looking for it.
2114 if (Case) {
2115 // Keep track of whether we see a skipped declaration. The code could be
2116 // using the declaration even if it is skipped, so we can't optimize out
2117 // the decl if the kept statements might refer to it.
2118 bool HadSkippedDecl = false;
2119
2120 // If we're looking for the case, just see if we can skip each of the
2121 // substatements.
2122 for (; Case && I != E; ++I) {
2123 HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(S: *I);
2124
2125 switch (CollectStatementsForCase(S: *I, Case, FoundCase, ResultStmts)) {
2126 case CSFC_Failure: return CSFC_Failure;
2127 case CSFC_Success:
2128 // A successful result means that either 1) that the statement doesn't
2129 // have the case and is skippable, or 2) does contain the case value
2130 // and also contains the break to exit the switch. In the later case,
2131 // we just verify the rest of the statements are elidable.
2132 if (FoundCase) {
2133 // If we found the case and skipped declarations, we can't do the
2134 // optimization.
2135 if (HadSkippedDecl)
2136 return CSFC_Failure;
2137
2138 for (++I; I != E; ++I)
2139 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2140 return CSFC_Failure;
2141 return CSFC_Success;
2142 }
2143 break;
2144 case CSFC_FallThrough:
2145 // If we have a fallthrough condition, then we must have found the
2146 // case started to include statements. Consider the rest of the
2147 // statements in the compound statement as candidates for inclusion.
2148 assert(FoundCase && "Didn't find case but returned fallthrough?");
2149 // We recursively found Case, so we're not looking for it anymore.
2150 Case = nullptr;
2151
2152 // If we found the case and skipped declarations, we can't do the
2153 // optimization.
2154 if (HadSkippedDecl)
2155 return CSFC_Failure;
2156 break;
2157 }
2158 }
2159
2160 if (!FoundCase)
2161 return CSFC_Success;
2162
2163 assert(!HadSkippedDecl && "fallthrough after skipping decl");
2164 }
2165
2166 // If we have statements in our range, then we know that the statements are
2167 // live and need to be added to the set of statements we're tracking.
2168 bool AnyDecls = false;
2169 for (; I != E; ++I) {
2170 AnyDecls |= CodeGenFunction::mightAddDeclToScope(S: *I);
2171
2172 switch (CollectStatementsForCase(S: *I, Case: nullptr, FoundCase, ResultStmts)) {
2173 case CSFC_Failure: return CSFC_Failure;
2174 case CSFC_FallThrough:
2175 // A fallthrough result means that the statement was simple and just
2176 // included in ResultStmt, keep adding them afterwards.
2177 break;
2178 case CSFC_Success:
2179 // A successful result means that we found the break statement and
2180 // stopped statement inclusion. We just ensure that any leftover stmts
2181 // are skippable and return success ourselves.
2182 for (++I; I != E; ++I)
2183 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2184 return CSFC_Failure;
2185 return CSFC_Success;
2186 }
2187 }
2188
2189 // If we're about to fall out of a scope without hitting a 'break;', we
2190 // can't perform the optimization if there were any decls in that scope
2191 // (we'd lose their end-of-lifetime).
2192 if (AnyDecls) {
2193 // If the entire compound statement was live, there's one more thing we
2194 // can try before giving up: emit the whole thing as a single statement.
2195 // We can do that unless the statement contains a 'break;'.
2196 // FIXME: Such a break must be at the end of a construct within this one.
2197 // We could emit this by just ignoring the BreakStmts entirely.
2198 if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
2199 ResultStmts.resize(N: StartSize);
2200 ResultStmts.push_back(Elt: S);
2201 } else {
2202 return CSFC_Failure;
2203 }
2204 }
2205
2206 return CSFC_FallThrough;
2207 }
2208
2209 // Okay, this is some other statement that we don't handle explicitly, like a
2210 // for statement or increment etc. If we are skipping over this statement,
2211 // just verify it doesn't have labels, which would make it invalid to elide.
2212 if (Case) {
2213 if (CodeGenFunction::ContainsLabel(S, IgnoreCaseStmts: true))
2214 return CSFC_Failure;
2215 return CSFC_Success;
2216 }
2217
2218 // Otherwise, we want to include this statement. Everything is cool with that
2219 // so long as it doesn't contain a break out of the switch we're in.
2220 if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
2221
2222 // Otherwise, everything is great. Include the statement and tell the caller
2223 // that we fall through and include the next statement as well.
2224 ResultStmts.push_back(Elt: S);
2225 return CSFC_FallThrough;
2226}
2227
2228/// FindCaseStatementsForValue - Find the case statement being jumped to and
2229/// then invoke CollectStatementsForCase to find the list of statements to emit
2230/// for a switch on constant. See the comment above CollectStatementsForCase
2231/// for more details.
2232static bool FindCaseStatementsForValue(const SwitchStmt &S,
2233 const llvm::APSInt &ConstantCondValue,
2234 SmallVectorImpl<const Stmt*> &ResultStmts,
2235 ASTContext &C,
2236 const SwitchCase *&ResultCase) {
2237 // First step, find the switch case that is being branched to. We can do this
2238 // efficiently by scanning the SwitchCase list.
2239 const SwitchCase *Case = S.getSwitchCaseList();
2240 const DefaultStmt *DefaultCase = nullptr;
2241
2242 for (; Case; Case = Case->getNextSwitchCase()) {
2243 // It's either a default or case. Just remember the default statement in
2244 // case we're not jumping to any numbered cases.
2245 if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Val: Case)) {
2246 DefaultCase = DS;
2247 continue;
2248 }
2249
2250 // Check to see if this case is the one we're looking for.
2251 const CaseStmt *CS = cast<CaseStmt>(Val: Case);
2252 // Don't handle case ranges yet.
2253 if (CS->getRHS()) return false;
2254
2255 // If we found our case, remember it as 'case'.
2256 if (CS->getLHS()->EvaluateKnownConstInt(Ctx: C) == ConstantCondValue)
2257 break;
2258 }
2259
2260 // If we didn't find a matching case, we use a default if it exists, or we
2261 // elide the whole switch body!
2262 if (!Case) {
2263 // It is safe to elide the body of the switch if it doesn't contain labels
2264 // etc. If it is safe, return successfully with an empty ResultStmts list.
2265 if (!DefaultCase)
2266 return !CodeGenFunction::ContainsLabel(S: &S);
2267 Case = DefaultCase;
2268 }
2269
2270 // Ok, we know which case is being jumped to, try to collect all the
2271 // statements that follow it. This can fail for a variety of reasons. Also,
2272 // check to see that the recursive walk actually found our case statement.
2273 // Insane cases like this can fail to find it in the recursive walk since we
2274 // don't handle every stmt kind:
2275 // switch (4) {
2276 // while (1) {
2277 // case 4: ...
2278 bool FoundCase = false;
2279 ResultCase = Case;
2280 return CollectStatementsForCase(S: S.getBody(), Case, FoundCase,
2281 ResultStmts) != CSFC_Failure &&
2282 FoundCase;
2283}
2284
2285static std::optional<SmallVector<uint64_t, 16>>
2286getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
2287 // Are there enough branches to weight them?
2288 if (Likelihoods.size() <= 1)
2289 return std::nullopt;
2290
2291 uint64_t NumUnlikely = 0;
2292 uint64_t NumNone = 0;
2293 uint64_t NumLikely = 0;
2294 for (const auto LH : Likelihoods) {
2295 switch (LH) {
2296 case Stmt::LH_Unlikely:
2297 ++NumUnlikely;
2298 break;
2299 case Stmt::LH_None:
2300 ++NumNone;
2301 break;
2302 case Stmt::LH_Likely:
2303 ++NumLikely;
2304 break;
2305 }
2306 }
2307
2308 // Is there a likelihood attribute used?
2309 if (NumUnlikely == 0 && NumLikely == 0)
2310 return std::nullopt;
2311
2312 // When multiple cases share the same code they can be combined during
2313 // optimization. In that case the weights of the branch will be the sum of
2314 // the individual weights. Make sure the combined sum of all neutral cases
2315 // doesn't exceed the value of a single likely attribute.
2316 // The additions both avoid divisions by 0 and make sure the weights of None
2317 // don't exceed the weight of Likely.
2318 const uint64_t Likely = INT32_MAX / (NumLikely + 2);
2319 const uint64_t None = Likely / (NumNone + 1);
2320 const uint64_t Unlikely = 0;
2321
2322 SmallVector<uint64_t, 16> Result;
2323 Result.reserve(N: Likelihoods.size());
2324 for (const auto LH : Likelihoods) {
2325 switch (LH) {
2326 case Stmt::LH_Unlikely:
2327 Result.push_back(Elt: Unlikely);
2328 break;
2329 case Stmt::LH_None:
2330 Result.push_back(Elt: None);
2331 break;
2332 case Stmt::LH_Likely:
2333 Result.push_back(Elt: Likely);
2334 break;
2335 }
2336 }
2337
2338 return Result;
2339}
2340
2341void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
2342 // Handle nested switch statements.
2343 llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
2344 SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
2345 SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
2346 llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
2347
2348 // See if we can constant fold the condition of the switch and therefore only
2349 // emit the live case statement (if any) of the switch.
2350 llvm::APSInt ConstantCondValue;
2351 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: ConstantCondValue)) {
2352 SmallVector<const Stmt*, 4> CaseStmts;
2353 const SwitchCase *Case = nullptr;
2354 if (FindCaseStatementsForValue(S, ConstantCondValue, ResultStmts&: CaseStmts,
2355 C&: getContext(), ResultCase&: Case)) {
2356 if (Case)
2357 incrementProfileCounter(S: Case);
2358 RunCleanupsScope ExecutedScope(*this);
2359
2360 if (S.getInit())
2361 EmitStmt(S: S.getInit());
2362
2363 // Emit the condition variable if needed inside the entire cleanup scope
2364 // used by this special case for constant folded switches.
2365 if (S.getConditionVariable())
2366 EmitDecl(D: *S.getConditionVariable(), /*EvaluateConditionDecl=*/true);
2367
2368 // At this point, we are no longer "within" a switch instance, so
2369 // we can temporarily enforce this to ensure that any embedded case
2370 // statements are not emitted.
2371 SwitchInsn = nullptr;
2372
2373 // Okay, we can dead code eliminate everything except this case. Emit the
2374 // specified series of statements and we're good.
2375 for (const Stmt *CaseStmt : CaseStmts)
2376 EmitStmt(S: CaseStmt);
2377 incrementProfileCounter(S: &S);
2378 PGO->markStmtMaybeUsed(S: S.getBody());
2379
2380 // Now we want to restore the saved switch instance so that nested
2381 // switches continue to function properly
2382 SwitchInsn = SavedSwitchInsn;
2383
2384 return;
2385 }
2386 }
2387
2388 JumpDest SwitchExit = getJumpDestInCurrentScope(Name: "sw.epilog");
2389
2390 RunCleanupsScope ConditionScope(*this);
2391
2392 if (S.getInit()) {
2393 EmitStmt(S: S.getInit());
2394
2395 // The init statement may have cleared the insertion point (e.g. it ended in
2396 // a 'noreturn' call); the condition emitted below needs a valid one.
2397 EnsureInsertPoint();
2398 }
2399
2400 if (S.getConditionVariable())
2401 EmitDecl(D: *S.getConditionVariable());
2402 llvm::Value *CondV = EmitScalarExpr(E: S.getCond());
2403 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
2404
2405 // Create basic block to hold stuff that comes after switch
2406 // statement. We also need to create a default block now so that
2407 // explicit case ranges tests can have a place to jump to on
2408 // failure.
2409 llvm::BasicBlock *DefaultBlock = createBasicBlock(name: "sw.default");
2410
2411 // The dispatch is the jump that bypasses any declarations sitting between the
2412 // switch and its case labels, so the initialization goes here, ahead of the
2413 // switch instruction -- not at the case labels. A case label is also reached
2414 // by falling through from the case above it, and that edge bypasses nothing;
2415 // initializing there would clobber a variable the previous case had written.
2416 // The declarations are inside the body and so have no alloca yet, hence the
2417 // patch-it-in-later handling in EmitAutoVarAlloca.
2418 if (!Bypasses.isAlwaysBypassed())
2419 BypassingForwardJumps.push_back(Elt: {.Block: Builder.GetInsertBlock(), .Source: &S});
2420
2421 SwitchInsn = Builder.CreateSwitch(V: CondV, Dest: DefaultBlock);
2422 addInstToNewSourceAtom(KeyInstruction: SwitchInsn, Backup: CondV);
2423
2424 if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {
2425 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2426 llvm::ConstantInt *BranchHintConstant =
2427 HLSLControlFlowAttr ==
2428 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2429 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2430 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2431 llvm::Metadata *Vals[] = {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2432 MDHelper.createConstant(C: BranchHintConstant)};
2433 SwitchInsn->setMetadata(Kind: "hlsl.controlflow.hint",
2434 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2435 }
2436
2437 if (PGO->haveRegionCounts()) {
2438 // Walk the SwitchCase list to find how many there are.
2439 uint64_t DefaultCount = 0;
2440 unsigned NumCases = 0;
2441 for (const SwitchCase *Case = S.getSwitchCaseList();
2442 Case;
2443 Case = Case->getNextSwitchCase()) {
2444 if (isa<DefaultStmt>(Val: Case))
2445 DefaultCount = getProfileCount(S: Case);
2446 NumCases += 1;
2447 }
2448 SwitchWeights = new SmallVector<uint64_t, 16>();
2449 SwitchWeights->reserve(N: NumCases);
2450 // The default needs to be first. We store the edge count, so we already
2451 // know the right weight.
2452 SwitchWeights->push_back(Elt: DefaultCount);
2453 } else if (CGM.getCodeGenOpts().OptimizationLevel) {
2454 SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
2455 // Initialize the default case.
2456 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
2457 }
2458
2459 CaseRangeBlock = DefaultBlock;
2460
2461 // Clear the insertion point to indicate we are in unreachable code.
2462 Builder.ClearInsertionPoint();
2463
2464 // All break statements jump to NextBlock. If BreakContinueStack is non-empty
2465 // then reuse last ContinueBlock.
2466 JumpDest OuterContinue;
2467 if (!BreakContinueStack.empty())
2468 OuterContinue = BreakContinueStack.back().ContinueBlock;
2469
2470 BreakContinueStack.push_back(Elt: BreakContinue(S, SwitchExit, OuterContinue));
2471
2472 // Emit switch body.
2473 EmitStmt(S: S.getBody());
2474
2475 BreakContinueStack.pop_back();
2476
2477 // Update the default block in case explicit case range tests have
2478 // been chained on top.
2479 SwitchInsn->setDefaultDest(CaseRangeBlock);
2480
2481 // If a default was never emitted:
2482 if (!DefaultBlock->getParent()) {
2483 // If we have cleanups, emit the default block so that there's a
2484 // place to jump through the cleanups from.
2485 if (ConditionScope.requiresCleanups()) {
2486 EmitBlock(BB: DefaultBlock);
2487
2488 // Otherwise, just forward the default block to the switch end.
2489 } else {
2490 DefaultBlock->replaceAllUsesWith(V: SwitchExit.getBlock());
2491 delete DefaultBlock;
2492 }
2493 }
2494
2495 ConditionScope.ForceCleanup();
2496
2497 // Close the last case (or DefaultBlock).
2498 EmitBranch(Target: SwitchExit.getBlock());
2499
2500 // Insert a False Counter if SwitchStmt doesn't have DefaultStmt.
2501 if (hasSkipCounter(S: S.getCond())) {
2502 auto *ImplicitDefaultBlock = createBasicBlock(name: "sw.false");
2503 EmitBlock(BB: ImplicitDefaultBlock);
2504 incrementProfileCounter(ExecSkip: UseSkipPath, S: S.getCond());
2505 Builder.CreateBr(Dest: SwitchInsn->getDefaultDest());
2506 SwitchInsn->setDefaultDest(ImplicitDefaultBlock);
2507 }
2508
2509 // Emit continuation.
2510 EmitBlock(BB: SwitchExit.getBlock(), IsFinished: true);
2511 incrementProfileCounter(S: &S);
2512
2513 // If the switch has a condition wrapped by __builtin_unpredictable,
2514 // create metadata that specifies that the switch is unpredictable.
2515 // Don't bother if not optimizing because that metadata would not be used.
2516 auto *Call = dyn_cast<CallExpr>(Val: S.getCond());
2517 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2518 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2519 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2520 llvm::MDBuilder MDHelper(getLLVMContext());
2521 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_unpredictable,
2522 Node: MDHelper.createUnpredictable());
2523 }
2524 }
2525
2526 if (SwitchWeights) {
2527 assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2528 "switch weights do not match switch cases");
2529 // If there's only one jump destination there's no sense weighting it.
2530 if (SwitchWeights->size() > 1)
2531 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2532 Node: createProfileWeights(Weights: *SwitchWeights));
2533 delete SwitchWeights;
2534 } else if (SwitchLikelihood) {
2535 assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2536 "switch likelihoods do not match switch cases");
2537 std::optional<SmallVector<uint64_t, 16>> LHW =
2538 getLikelihoodWeights(Likelihoods: *SwitchLikelihood);
2539 if (LHW) {
2540 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2541 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2542 Node: createProfileWeights(Weights: *LHW));
2543 }
2544 delete SwitchLikelihood;
2545 }
2546 SwitchInsn = SavedSwitchInsn;
2547 SwitchWeights = SavedSwitchWeights;
2548 SwitchLikelihood = SavedSwitchLikelihood;
2549 CaseRangeBlock = SavedCRBlock;
2550}
2551
2552std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2553 const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2554 QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2555 if (Info.allowsRegister() || !Info.allowsMemory()) {
2556 if (CodeGenFunction::hasScalarEvaluationKind(T: InputType))
2557 return {EmitLoadOfLValue(V: InputValue, Loc).getScalarVal(), nullptr};
2558
2559 llvm::Type *Ty = ConvertType(T: InputType);
2560 uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2561 if ((Size <= 64 && llvm::isPowerOf2_64(Value: Size)) ||
2562 getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty)) {
2563 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2564
2565 return {Builder.CreateLoad(Addr: InputValue.getAddress().withElementType(ElemTy: Ty)),
2566 nullptr};
2567 }
2568 }
2569
2570 Address Addr = InputValue.getAddress();
2571 ConstraintStr += '*';
2572 return {InputValue.getPointer(CGF&: *this), Addr.getElementType()};
2573}
2574std::pair<llvm::Value *, llvm::Type *>
2575CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2576 const Expr *InputExpr,
2577 std::string &ConstraintStr) {
2578 // If this can't be a register or memory, i.e., has to be a constant
2579 // (immediate or symbolic), try to emit it as such.
2580 if (!Info.allowsRegister() && !Info.allowsMemory()) {
2581 if (Info.requiresImmediateConstant()) {
2582 Expr::EvalResult EVResult;
2583 InputExpr->EvaluateAsRValue(Result&: EVResult, Ctx: getContext(), InConstantContext: true);
2584
2585 llvm::APSInt IntResult;
2586 if (EVResult.Val.toIntegralConstant(Result&: IntResult, SrcTy: InputExpr->getType(),
2587 Ctx: getContext()))
2588 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: IntResult), nullptr};
2589 }
2590
2591 Expr::EvalResult Result;
2592 if (InputExpr->EvaluateAsInt(Result, Ctx: getContext()))
2593 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: Result.Val.getInt()),
2594 nullptr};
2595 }
2596
2597 if (Info.allowsRegister() || !Info.allowsMemory())
2598 if (CodeGenFunction::hasScalarEvaluationKind(T: InputExpr->getType()))
2599 return {EmitScalarExpr(E: InputExpr), nullptr};
2600 if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2601 return {EmitScalarExpr(E: InputExpr), nullptr};
2602 InputExpr = InputExpr->IgnoreParenNoopCasts(Ctx: getContext());
2603 LValue Dest = EmitLValue(E: InputExpr);
2604 return EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr,
2605 Loc: InputExpr->getExprLoc());
2606}
2607
2608/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2609/// asm call instruction. The !srcloc MDNode contains a list of constant
2610/// integers which are the source locations of the start of each line in the
2611/// asm.
2612static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2613 CodeGenFunction &CGF) {
2614 SmallVector<llvm::Metadata *, 8> Locs;
2615 // Add the location of the first line to the MDNode.
2616 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
2617 Ty: CGF.Int64Ty, V: Str->getBeginLoc().getRawEncoding())));
2618 StringRef StrVal = Str->getString();
2619 if (!StrVal.empty()) {
2620 const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2621 const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2622 unsigned StartToken = 0;
2623 unsigned ByteOffset = 0;
2624
2625 // Add the location of the start of each subsequent line of the asm to the
2626 // MDNode.
2627 for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2628 if (StrVal[i] != '\n') continue;
2629 SourceLocation LineLoc = Str->getLocationOfByte(
2630 ByteNo: i + 1, SM, Features: LangOpts, Target: CGF.getTarget(), StartToken: &StartToken, StartTokenByteOffset: &ByteOffset);
2631 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(
2632 C: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: LineLoc.getRawEncoding())));
2633 }
2634 }
2635
2636 return llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: Locs);
2637}
2638
2639namespace clang {
2640
2641/// This structure holds the information gathered about the constraints for an
2642/// inline assembly statement. It helps in separating the constraint processing
2643/// from the code generation.
2644class AsmConstraintsInfo {
2645 CodeGenFunction &CGF;
2646 CodeGenModule &CGM; // Per-module state.
2647 const AsmStmt &S;
2648 CGBuilderTy &Builder;
2649
2650 // The final asm string.
2651 std::string AsmString;
2652
2653 // The output and input constraints.
2654 SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2655 SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2656
2657 // Constraint strings.
2658 std::string Constraints;
2659 std::string InOutConstraints;
2660
2661 // Keep track of out constraints for tied input operand.
2662 std::vector<std::string> OutputConstraints;
2663
2664 // Keep track of argument types.
2665 std::vector<llvm::Value *> Args;
2666 std::vector<llvm::Type *> ArgTypes;
2667 std::vector<llvm::Type *> ArgElemTypes;
2668
2669 // Keep track of result register constraints.
2670 std::vector<LValue> ResultRegDests;
2671 std::vector<QualType> ResultRegQualTys;
2672 std::vector<llvm::Type *> ResultRegTypes;
2673 std::vector<llvm::Type *> ResultTruncRegTypes;
2674
2675 llvm::BitVector ResultTypeRequiresCast;
2676
2677 // Keep track of in/out constraints.
2678 std::vector<llvm::Value *> InOutArgs;
2679 std::vector<llvm::Type *> InOutArgTypes;
2680 std::vector<llvm::Type *> InOutArgElemTypes;
2681
2682 // Destination blocks for 'asm gotos'.
2683 llvm::BasicBlock *DefaultDest = nullptr;
2684 SmallVector<llvm::BasicBlock *, 3> IndirectDests;
2685
2686 std::vector<std::optional<std::pair<unsigned, unsigned>>> ResultBounds;
2687
2688 // An inline asm can be marked readonly if it meets the following
2689 // conditions:
2690 //
2691 // - it doesn't have any sideeffects
2692 // - it doesn't clobber memory
2693 // - it doesn't return a value by-reference
2694 //
2695 // It can be marked readnone if it doesn't have any input memory
2696 // constraints in addition to meeting the conditions listed above.
2697 bool ReadOnly = true;
2698 bool ReadNone = true;
2699
2700 bool GetOutputAndInputConstraints();
2701 void HandleOutputConstraints();
2702 void HandleMSStyleAsmBlob();
2703 void HandleInputConstraints();
2704 bool HandleLabels();
2705 bool HandleClobbers();
2706 void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2707 bool HasUnwindClobber, bool NoMerge, bool NoConvergent,
2708 std::vector<llvm::Value *> &RegResults);
2709 void EmitAsmStores(const llvm::ArrayRef<llvm::Value *> RegResults);
2710
2711 void EmitHipStdParUnsupportedAsm() {
2712 constexpr auto Name = "__ASM__hipstdpar_unsupported";
2713
2714 std::string Asm;
2715 if (auto GCCAsm = dyn_cast<GCCAsmStmt>(Val: &S))
2716 Asm = GCCAsm->getAsmString();
2717
2718 auto &Ctx = getLLVMContext();
2719 auto StrTy = llvm::ConstantDataArray::getString(Context&: Ctx, Initializer: Asm);
2720 auto FnTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx),
2721 Params: {StrTy->getType()}, isVarArg: false);
2722 auto UBF = CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2723
2724 Builder.CreateCall(Callee: UBF, Args: {StrTy});
2725 }
2726
2727 ASTContext &getContext() { return CGF.getContext(); }
2728 llvm::LLVMContext &getLLVMContext() { return CGF.getLLVMContext(); }
2729 const TargetInfo &getTarget() const { return CGF.getTarget(); }
2730 const LangOptions &getLangOpts() const { return CGF.getLangOpts(); }
2731 const TargetCodeGenInfo &getTargetHooks() const {
2732 return CGM.getTargetCodeGenInfo();
2733 }
2734
2735public:
2736 AsmConstraintsInfo(CodeGenFunction &CGF, const AsmStmt &S)
2737 : CGF(CGF), CGM(CGF.CGM), S(S), Builder(CGF.Builder),
2738 AsmString(S.generateAsmString(C: CGF.getContext())) {}
2739
2740 void EmitAsmStmt();
2741};
2742
2743} // namespace clang
2744
2745void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2746 // Pop all cleanup blocks at the end of the asm statement.
2747 CodeGenFunction::RunCleanupsScope Cleanups(*this);
2748
2749 // Get all the output and input constraints together.
2750 AsmConstraintsInfo AsmInfo(*this, S);
2751 AsmInfo.EmitAsmStmt();
2752}
2753
2754void AsmConstraintsInfo::EmitAsmStmt() {
2755 if (!GetOutputAndInputConstraints())
2756 return EmitHipStdParUnsupportedAsm();
2757
2758 // Handle output constraints.
2759 HandleOutputConstraints();
2760
2761 // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2762 // to the return value slot. Only do this when returning in registers.
2763 HandleMSStyleAsmBlob();
2764
2765 // Handle input constraints.
2766 HandleInputConstraints();
2767
2768 // Handle 'asm goto' labels.
2769 bool IsGCCAsmGoto = HandleLabels();
2770
2771 // Handle any clobbers.
2772 bool HasUnwindClobber = HandleClobbers();
2773 assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
2774 "unwind clobber can't be used with asm goto");
2775
2776 // Add machine specific clobbers
2777 std::string_view MachineClobbers = getTarget().getClobbers();
2778 if (!MachineClobbers.empty()) {
2779 if (!Constraints.empty())
2780 Constraints += ',';
2781 Constraints += MachineClobbers;
2782 }
2783
2784 llvm::Type *ResultType;
2785 if (ResultRegTypes.empty())
2786 ResultType = CGF.VoidTy;
2787 else if (ResultRegTypes.size() == 1)
2788 ResultType = ResultRegTypes[0];
2789 else
2790 ResultType = llvm::StructType::get(Context&: getLLVMContext(), Elements: ResultRegTypes);
2791
2792 llvm::FunctionType *FTy =
2793 llvm::FunctionType::get(Result: ResultType, Params: ArgTypes, isVarArg: false);
2794
2795 bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
2796
2797 llvm::InlineAsm::AsmDialect GnuAsmDialect =
2798 CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
2799 ? llvm::InlineAsm::AD_ATT
2800 : llvm::InlineAsm::AD_Intel;
2801 llvm::InlineAsm::AsmDialect AsmDialect =
2802 isa<MSAsmStmt>(Val: &S) ? llvm::InlineAsm::AD_Intel : GnuAsmDialect;
2803
2804 llvm::InlineAsm *IA = llvm::InlineAsm::get(
2805 Ty: FTy, AsmString, Constraints, hasSideEffects: HasSideEffect,
2806 /* IsAlignStack */ isAlignStack: false, asmDialect: AsmDialect, canThrow: HasUnwindClobber);
2807 std::vector<llvm::Value *> RegResults;
2808 llvm::CallBrInst *CBR;
2809 llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>
2810 CBRRegResults;
2811
2812 if (IsGCCAsmGoto) {
2813 if (CGF.HaveInsertPoint() && !CGF.Bypasses.isAlwaysBypassed()) {
2814 CGF.emitBypassedVarInitsForSource(Source: &S);
2815 CGF.BypassingForwardJumps.push_back(Elt: {.Block: Builder.GetInsertBlock(), .Source: &S});
2816 }
2817 CBR = Builder.CreateCallBr(Callee: IA, DefaultDest, IndirectDests, Args);
2818 CGF.EmitBlock(BB: DefaultDest);
2819 UpdateAsmCallInst(Result&: *CBR, HasSideEffect,
2820 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2821 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2822
2823 // Because we are emitting code top to bottom, we don't have enough
2824 // information at this point to know precisely whether we have a critical
2825 // edge. If we have outputs, split all indirect destinations.
2826 if (!RegResults.empty()) {
2827 unsigned I = 0;
2828 for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {
2829 llvm::Twine SynthName = Dest->getName() + ".split";
2830 llvm::BasicBlock *SynthBB = CGF.createBasicBlock(name: SynthName);
2831 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2832 Builder.SetInsertPoint(SynthBB);
2833
2834 if (ResultRegTypes.size() == 1) {
2835 CBRRegResults[SynthBB].push_back(Elt: CBR);
2836 } else {
2837 for (unsigned J = 0, E = ResultRegTypes.size(); J != E; ++J) {
2838 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: CBR, Idxs: J, Name: "asmresult");
2839 CBRRegResults[SynthBB].push_back(Elt: Tmp);
2840 }
2841 }
2842
2843 CGF.EmitBranch(Target: Dest);
2844 CGF.EmitBlock(BB: SynthBB);
2845 CBR->setIndirectDest(i: I++, B: SynthBB);
2846 }
2847 }
2848 } else if (HasUnwindClobber) {
2849 llvm::CallBase *Result = CGF.EmitCallOrInvoke(Callee: IA, Args, Name: "");
2850 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2851 /*HasUnwindClobber=*/true, NoMerge: CGF.InNoMergeAttributedStmt,
2852 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2853 } else {
2854 llvm::CallInst *Result =
2855 Builder.CreateCall(Callee: IA, Args, OpBundles: CGF.getBundlesForFunclet(Callee: IA));
2856 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2857 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2858 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2859 }
2860
2861 EmitAsmStores(RegResults);
2862
2863 // If this is an asm goto with outputs, repeat EmitAsmStores, but with a
2864 // different insertion point; one for each indirect destination and with
2865 // CBRRegResults rather than RegResults.
2866 if (IsGCCAsmGoto && !CBRRegResults.empty()) {
2867 for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {
2868 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2869 Builder.SetInsertPoint(--(Succ->end()));
2870 EmitAsmStores(RegResults: CBRRegResults[Succ]);
2871 }
2872 }
2873}
2874
2875/// Gather and validate the output and input constraints for the given inline
2876/// assembly statement. This ensures that the constraints are valid for the
2877/// target and prepares them for further processing.
2878bool AsmConstraintsInfo::GetOutputAndInputConstraints() {
2879 bool IsValidTargetAsm = true;
2880 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2881 for (unsigned I = 0, E = S.getNumOutputs(); I != E && IsValidTargetAsm; I++) {
2882 StringRef Name;
2883 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2884 Name = GAS->getOutputName(i: I);
2885
2886 TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i: I), Name);
2887
2888 bool IsValid = getTarget().validateOutputConstraint(Info);
2889 if (IsHipStdPar && !IsValid)
2890 IsValidTargetAsm = false;
2891 else
2892 assert(IsValid && "Failed to parse output constraint");
2893
2894 OutputConstraintInfos.push_back(Elt: Info);
2895 }
2896
2897 for (unsigned I = 0, E = S.getNumInputs(); I != E && IsValidTargetAsm; I++) {
2898 StringRef Name;
2899 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2900 Name = GAS->getInputName(i: I);
2901
2902 TargetInfo::ConstraintInfo Info(S.getInputConstraint(i: I), Name);
2903
2904 bool IsValid =
2905 getTarget().validateInputConstraint(OutputConstraints: OutputConstraintInfos, info&: Info);
2906 if (IsHipStdPar && !IsValid)
2907 IsValidTargetAsm = false;
2908 else
2909 assert(IsValid && "Failed to parse input constraint");
2910
2911 InputConstraintInfos.push_back(Elt: Info);
2912 }
2913
2914 return IsValidTargetAsm;
2915}
2916
2917/// Process the output constraints of an inline assembly statement. This method
2918/// handles the complexity of determining whether an output should be a
2919/// register or memory operand, manages tied operands, and prepares the
2920/// necessary arguments for the LLVM inline asm call.
2921void AsmConstraintsInfo::HandleOutputConstraints() {
2922 // Keep track of defined physregs.
2923 llvm::SmallSet<std::string, 8> PhysRegOutputs;
2924
2925 for (unsigned I = 0, E = S.getNumOutputs(); I != E; I++) {
2926 TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[I];
2927
2928 // Simplify the output constraint.
2929 std::string OutputConstraint(S.getOutputConstraint(i: I));
2930 OutputConstraint = getTarget().simplifyConstraint(
2931 Constraint: StringRef(OutputConstraint).substr(Start: 1), OutCons: &OutputConstraintInfos);
2932
2933 const Expr *OutExpr = S.getOutputExpr(i: I);
2934 OutExpr = OutExpr->IgnoreParenNoopCasts(Ctx: getContext());
2935
2936 std::string GCCReg;
2937 OutputConstraint = S.addVariableConstraints(
2938 Constraint: OutputConstraint, AsmExpr: *OutExpr, Target: getTarget(), EarlyClobber: Info.earlyClobber(),
2939 UnsupportedCB: [&](const Stmt *UnspStmt, StringRef Msg) {
2940 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
2941 },
2942 GCCReg: &GCCReg);
2943
2944 // Give an error on multiple outputs to same physreg.
2945 if (!GCCReg.empty() && !PhysRegOutputs.insert(V: GCCReg).second)
2946 CGM.Error(loc: S.getAsmLoc(), error: "multiple outputs to hard register: " + GCCReg);
2947
2948 OutputConstraints.push_back(x: OutputConstraint);
2949 LValue Dest = CGF.EmitLValue(E: OutExpr);
2950 if (!Constraints.empty())
2951 Constraints += ',';
2952
2953 // If this is a register output, then make the inline asm return it
2954 // by-value. If this is a memory result, return the value by-reference.
2955 QualType QTy = OutExpr->getType();
2956 const bool IsScalarOrAggregate =
2957 CodeGenFunction::hasScalarEvaluationKind(T: QTy) ||
2958 CodeGenFunction::hasAggregateEvaluationKind(T: QTy);
2959
2960 if (!Info.allowsMemory() && IsScalarOrAggregate) {
2961 Constraints += "=" + OutputConstraint;
2962 ResultRegQualTys.push_back(x: QTy);
2963 ResultRegDests.push_back(x: Dest);
2964
2965 ResultBounds.emplace_back(args: Info.getOutputOperandBounds());
2966
2967 llvm::Type *Ty = CGF.ConvertTypeForMem(T: QTy);
2968 const bool RequiresCast =
2969 Info.allowsRegister() &&
2970 (getTargetHooks().isScalarizableAsmOperand(CGF, Ty) ||
2971 Ty->isAggregateType());
2972
2973 ResultTruncRegTypes.push_back(x: Ty);
2974 ResultTypeRequiresCast.push_back(Val: RequiresCast);
2975
2976 if (RequiresCast) {
2977 if (unsigned Size = getContext().getTypeSize(T: QTy))
2978 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2979 else
2980 CGM.Error(loc: OutExpr->getExprLoc(), error: "output size should not be zero");
2981 }
2982
2983 ResultRegTypes.push_back(x: Ty);
2984
2985 // If this output is tied to an input, and if the input is larger, then
2986 // we need to set the actual result type of the inline asm node to be the
2987 // same as the input type.
2988 if (Info.hasMatchingInput()) {
2989 unsigned InputNo;
2990 for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2991 TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2992 if (Input.hasTiedOperand() && Input.getTiedOperand() == I)
2993 break;
2994 }
2995 assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2996
2997 QualType InputTy = S.getInputExpr(i: InputNo)->getType();
2998 QualType OutputType = OutExpr->getType();
2999
3000 uint64_t InputSize = getContext().getTypeSize(T: InputTy);
3001 if (getContext().getTypeSize(T: OutputType) < InputSize)
3002 // Form the asm to return the value as a larger integer or fp type.
3003 ResultRegTypes.back() = CGF.ConvertType(T: InputTy);
3004 }
3005
3006 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3007 CGF, Constraint: OutputConstraint, Ty: ResultRegTypes.back()))
3008 ResultRegTypes.back() = AdjTy;
3009 else
3010 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3011 DiagID: diag::err_asm_invalid_type_in_input)
3012 << OutExpr->getType() << OutputConstraint;
3013
3014 // Update largest vector width for any vector types.
3015 if (auto *VT = dyn_cast<llvm::VectorType>(Val: ResultRegTypes.back()))
3016 CGF.LargestVectorWidth =
3017 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3018 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3019 } else {
3020 Address DestAddr = Dest.getAddress();
3021
3022 // Matrix types in memory are represented by arrays, but accessed through
3023 // vector pointers, with the alignment specified on the access operation.
3024 // For inline assembly, update pointer arguments to use vector pointers.
3025 // Otherwise there will be a mis-match if the matrix is also an
3026 // input-argument which is represented as vector.
3027 if (isa<MatrixType>(Val: OutExpr->getType().getCanonicalType()))
3028 DestAddr =
3029 DestAddr.withElementType(ElemTy: CGF.ConvertType(T: OutExpr->getType()));
3030
3031 ArgTypes.push_back(x: DestAddr.getType());
3032 ArgElemTypes.push_back(x: DestAddr.getElementType());
3033 Args.push_back(x: DestAddr.emitRawPointer(CGF));
3034
3035 Constraints += "=*" + OutputConstraint;
3036 ReadOnly = false;
3037 ReadNone = false;
3038 }
3039
3040 if (!Info.isReadWrite())
3041 continue;
3042
3043 InOutConstraints += ',';
3044
3045 const Expr *InputExpr = S.getOutputExpr(i: I);
3046 llvm::Value *Arg;
3047 llvm::Type *ArgElemType;
3048 std::tie(args&: Arg, args&: ArgElemType) =
3049 CGF.EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(),
3050 ConstraintStr&: InOutConstraints, Loc: InputExpr->getExprLoc());
3051
3052 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3053 CGF, Constraint: OutputConstraint, Ty: Arg->getType()))
3054 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3055
3056 // Update largest vector width for any vector types.
3057 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3058 CGF.LargestVectorWidth =
3059 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3060 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3061
3062 // Only tie earlyclobber physregs.
3063 if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
3064 InOutConstraints += llvm::utostr(X: I);
3065 else
3066 InOutConstraints += OutputConstraint;
3067
3068 InOutArgTypes.push_back(x: Arg->getType());
3069 InOutArgElemTypes.push_back(x: ArgElemType);
3070 InOutArgs.push_back(x: Arg);
3071 }
3072}
3073
3074/// Special handling for Microsoft-style inline assembly blocks. This ensures
3075/// that return registers (like EAX:EDX) are correctly mapped to the function's
3076/// return value slot when necessary.
3077void AsmConstraintsInfo::HandleMSStyleAsmBlob() {
3078 if (!isa<MSAsmStmt>(Val: &S))
3079 return;
3080
3081 const ABIArgInfo &RetAI = CGF.CurFnInfo->getReturnInfo();
3082 if (!RetAI.isDirect() && !RetAI.isExtend())
3083 return;
3084
3085 // Make a fake lvalue for the return value slot.
3086 LValue ReturnSlot =
3087 CGF.MakeAddrLValueWithoutTBAA(Addr: CGF.ReturnValue, T: CGF.FnRetTy);
3088 CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
3089 CGF, ReturnValue: ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
3090 ResultRegDests, AsmString, NumOutputs: S.getNumOutputs());
3091 CGF.SawAsmBlock = true;
3092}
3093
3094/// Process the input constraints of an inline assembly statement. It handles
3095/// type conversions, extensions for tied operands, and collects the necessary
3096/// LLVM values to be passed to the inline assembly call.
3097void AsmConstraintsInfo::HandleInputConstraints() {
3098 ASTContext &Ctx = getContext();
3099
3100 for (unsigned I = 0, E = S.getNumInputs(); I != E; I++) {
3101 TargetInfo::ConstraintInfo &Info = InputConstraintInfos[I];
3102 const Expr *InputExpr = S.getInputExpr(i: I);
3103
3104 if (Info.allowsMemory())
3105 ReadNone = false;
3106
3107 if (!Constraints.empty())
3108 Constraints += ',';
3109
3110 // Simplify the input constraint.
3111 std::string InputConstraint(S.getInputConstraint(i: I));
3112 InputConstraint =
3113 getTarget().simplifyConstraint(Constraint: InputConstraint, OutCons: &OutputConstraintInfos);
3114
3115 InputConstraint = S.addVariableConstraints(
3116 Constraint: InputConstraint, AsmExpr: *InputExpr->IgnoreParenNoopCasts(Ctx), Target: getTarget(),
3117 EarlyClobber: false /* No EarlyClobber */,
3118 UnsupportedCB: [&](const Stmt *UnspStmt, std::string_view Msg) {
3119 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
3120 });
3121
3122 std::string ReplaceConstraint(InputConstraint);
3123 llvm::Value *Arg;
3124 llvm::Type *ArgElemType;
3125 std::tie(args&: Arg, args&: ArgElemType) = CGF.EmitAsmInput(Info, InputExpr, ConstraintStr&: Constraints);
3126
3127 // If this input argument is tied to a larger output result, extend the
3128 // input to be the same size as the output. The LLVM backend wants to see
3129 // the input and output of a matching constraint be the same size. Note
3130 // that GCC does not define what the top bits are here. We use zext because
3131 // that is usually cheaper, but LLVM IR should really get an anyext someday.
3132 if (Info.hasTiedOperand()) {
3133 unsigned Output = Info.getTiedOperand();
3134 QualType OutputType = S.getOutputExpr(i: Output)->getType();
3135 QualType InputTy = InputExpr->getType();
3136
3137 if (Ctx.getTypeSize(T: OutputType) > Ctx.getTypeSize(T: InputTy)) {
3138 // Use ptrtoint as appropriate so that we can do our extension.
3139 if (isa<llvm::PointerType>(Val: Arg->getType()))
3140 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: CGF.IntPtrTy);
3141
3142 llvm::Type *OutputTy = CGF.ConvertType(T: OutputType);
3143 if (isa<llvm::IntegerType>(Val: OutputTy))
3144 Arg = Builder.CreateZExt(V: Arg, DestTy: OutputTy);
3145 else if (isa<llvm::PointerType>(Val: OutputTy))
3146 Arg = Builder.CreateZExt(V: Arg, DestTy: CGF.IntPtrTy);
3147 else if (OutputTy->isFloatingPointTy())
3148 Arg = Builder.CreateFPExt(V: Arg, DestTy: OutputTy);
3149 }
3150
3151 // Deal with the tied operands' constraint code in adjustInlineAsmType.
3152 ReplaceConstraint = OutputConstraints[Output];
3153 }
3154
3155 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3156 CGF, Constraint: ReplaceConstraint, Ty: Arg->getType()))
3157 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3158 else
3159 CGM.getDiags().Report(Loc: S.getAsmLoc(), DiagID: diag::err_asm_invalid_type_in_input)
3160 << InputExpr->getType() << InputConstraint;
3161
3162 // Update largest vector width for any vector types.
3163 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3164 CGF.LargestVectorWidth =
3165 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3166 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3167
3168 ArgTypes.push_back(x: Arg->getType());
3169 ArgElemTypes.push_back(x: ArgElemType);
3170 Args.push_back(x: Arg);
3171
3172 Constraints += InputConstraint;
3173 }
3174
3175 // Append the "input" part of in/out constraints.
3176 for (unsigned I = 0, E = InOutArgs.size(); I != E; I++) {
3177 ArgTypes.push_back(x: InOutArgTypes[I]);
3178 ArgElemTypes.push_back(x: InOutArgElemTypes[I]);
3179 Args.push_back(x: InOutArgs[I]);
3180 }
3181
3182 Constraints += InOutConstraints;
3183}
3184
3185/// Handle labels in an 'asm goto' statement. This method resolves the symbolic
3186/// labels to LLVM basic blocks and updates the constraint string to reflect
3187/// the indirect jump targets.
3188bool AsmConstraintsInfo::HandleLabels() {
3189 if (const auto *GS = dyn_cast<GCCAsmStmt>(Val: &S); GS && GS->isAsmGoto()) {
3190 for (const auto *E : GS->labels()) {
3191 CodeGenFunction::JumpDest Dest = CGF.getJumpDestForLabel(D: E->getLabel());
3192 IndirectDests.push_back(Elt: Dest.getBlock());
3193
3194 if (!Constraints.empty())
3195 Constraints += ',';
3196
3197 Constraints += "!i";
3198 }
3199
3200 DefaultDest = CGF.createBasicBlock(name: "asm.fallthrough");
3201 return true;
3202 }
3203
3204 return false;
3205}
3206
3207/// Process clobber constraints for an inline assembly statement. This
3208/// identifies which registers or system state (like "memory" or "cc") are
3209/// modified by the assembly block, which is crucial for correct optimization
3210/// and side-effect modeling.
3211bool AsmConstraintsInfo::HandleClobbers() {
3212 bool HasUnwindClobber = false;
3213 for (unsigned I = 0, E = S.getNumClobbers(); I != E; I++) {
3214 std::string Clobber = S.getClobber(i: I);
3215
3216 if (Clobber == "unwind") {
3217 HasUnwindClobber = true;
3218 continue;
3219 }
3220
3221 if (Clobber == "memory") {
3222 ReadOnly = false;
3223 ReadNone = false;
3224 } else if (Clobber != "cc") {
3225 Clobber = getTarget().getNormalizedGCCRegisterName(Name: Clobber);
3226 if (CGM.getCodeGenOpts().StackClashProtector &&
3227 getTarget().isSPRegName(Clobber)) {
3228 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3229 DiagID: diag::warn_stack_clash_protection_inline_asm);
3230 }
3231 }
3232
3233 if (isa<MSAsmStmt>(Val: &S)) {
3234 if (Clobber == "eax" || Clobber == "edx") {
3235 if (Constraints.find(s: "=&A") != std::string::npos)
3236 continue;
3237
3238 std::string::size_type position1 =
3239 Constraints.find(str: "={" + Clobber + "}");
3240 if (position1 != std::string::npos) {
3241 Constraints.insert(pos: position1 + 1, s: "&");
3242 continue;
3243 }
3244
3245 std::string::size_type position2 = Constraints.find(s: "=A");
3246 if (position2 != std::string::npos) {
3247 Constraints.insert(pos: position2 + 1, s: "&");
3248 continue;
3249 }
3250 }
3251 }
3252
3253 if (!Constraints.empty())
3254 Constraints += ',';
3255
3256 Constraints += "~{" + Clobber + '}';
3257 }
3258
3259 return HasUnwindClobber;
3260}
3261
3262void AsmConstraintsInfo::UpdateAsmCallInst(
3263 llvm::CallBase &Result, bool HasSideEffect, bool HasUnwindClobber,
3264 bool NoMerge, bool NoConvergent, std::vector<llvm::Value *> &RegResults) {
3265 if (!HasUnwindClobber)
3266 Result.addFnAttr(Kind: llvm::Attribute::NoUnwind);
3267
3268 if (NoMerge)
3269 Result.addFnAttr(Kind: llvm::Attribute::NoMerge);
3270
3271 // Attach readnone and readonly attributes.
3272 if (!HasSideEffect) {
3273 if (ReadNone)
3274 Result.setDoesNotAccessMemory();
3275 else if (ReadOnly)
3276 Result.setOnlyReadsMemory();
3277 }
3278
3279 // Add elementtype attribute for indirect constraints.
3280 for (auto Pair : llvm::enumerate(First&: ArgElemTypes)) {
3281 if (Pair.value()) {
3282 auto Attr = llvm::Attribute::get(
3283 Context&: getLLVMContext(), Kind: llvm::Attribute::ElementType, Ty: Pair.value());
3284 Result.addParamAttr(ArgNo: Pair.index(), Attr);
3285 }
3286 }
3287
3288 // Slap the source location of the inline asm into a !srcloc metadata on the
3289 // call.
3290 const StringLiteral *SL;
3291 if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(Val: &S);
3292 gccAsmStmt &&
3293 (SL = dyn_cast<StringLiteral>(Val: gccAsmStmt->getAsmStringExpr()))) {
3294 Result.setMetadata(Kind: "srcloc", Node: getAsmSrcLocInfo(Str: SL, CGF));
3295 } else {
3296 // At least put the line number on MS inline asm blobs and GCC asm constexpr
3297 // strings.
3298 llvm::Constant *Loc =
3299 llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: S.getAsmLoc().getRawEncoding());
3300 Result.setMetadata(Kind: "srcloc",
3301 Node: llvm::MDNode::get(Context&: getLLVMContext(),
3302 MDs: llvm::ConstantAsMetadata::get(C: Loc)));
3303 }
3304
3305 // Make inline-asm calls Key for the debug info feature Key Instructions.
3306 CGF.addInstToNewSourceAtom(KeyInstruction: &Result, Backup: nullptr);
3307
3308 if (!NoConvergent && getLangOpts().assumeFunctionsAreConvergent())
3309 // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
3310 // convergent (meaning, they may call an intrinsically convergent op, such
3311 // as bar.sync, and so can't have certain optimizations applied around
3312 // them) unless it's explicitly marked 'noconvergent'.
3313 Result.addFnAttr(Kind: llvm::Attribute::Convergent);
3314
3315 // Extract all of the register value results from the asm.
3316 if (ResultRegTypes.size() == 1) {
3317 RegResults.push_back(x: &Result);
3318 } else {
3319 for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
3320 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: &Result, Idxs: i, Name: "asmresult");
3321 RegResults.push_back(x: Tmp);
3322 }
3323 }
3324}
3325
3326void AsmConstraintsInfo::EmitAsmStores(
3327 const llvm::ArrayRef<llvm::Value *> RegResults) {
3328 llvm::LLVMContext &CTX = getLLVMContext();
3329
3330 assert(RegResults.size() == ResultRegTypes.size());
3331 assert(RegResults.size() == ResultTruncRegTypes.size());
3332 assert(RegResults.size() == ResultRegDests.size());
3333
3334 // ResultRegDests can also be populated by addReturnRegisterOutputs() above,
3335 // in which case its size may grow.
3336 assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
3337 assert(ResultBounds.size() <= ResultRegDests.size());
3338
3339 for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
3340 llvm::Value *Tmp = RegResults[i];
3341 llvm::Type *TruncTy = ResultTruncRegTypes[i];
3342
3343 if (i < ResultBounds.size() && ResultBounds[i].has_value()) {
3344 const auto [LowerBound, UpperBound] = ResultBounds[i].value();
3345
3346 // FIXME: Support for nonzero lower bounds not yet implemented.
3347 assert(LowerBound == 0 && "Output operand lower bound is not zero.");
3348
3349 llvm::Constant *UpperBoundConst =
3350 llvm::ConstantInt::get(Ty: Tmp->getType(), V: UpperBound);
3351 llvm::Value *IsBooleanValue =
3352 Builder.CreateCmp(Pred: llvm::CmpInst::ICMP_ULT, LHS: Tmp, RHS: UpperBoundConst);
3353 llvm::Function *FnAssume = CGM.getIntrinsic(IID: llvm::Intrinsic::assume);
3354
3355 Builder.CreateCall(Callee: FnAssume, Args: IsBooleanValue);
3356 }
3357
3358 // ResultTypeRequiresCast elements correspond to the first
3359 // ResultTypeRequiresCast.size() elements of RegResults.
3360 const bool RequiresCast =
3361 i < ResultTypeRequiresCast.size() && ResultTypeRequiresCast[i];
3362
3363 // If the result type of the LLVM IR asm doesn't match the result type of
3364 // the expression, do the conversion. An output returned as an integer is
3365 // converted where it's stored below.
3366 if (ResultRegTypes[i] != TruncTy && !RequiresCast) {
3367 // Truncate the integer result to the right size, note that TruncTy can be
3368 // a pointer.
3369 if (TruncTy->isFloatingPointTy())
3370 Tmp = Builder.CreateFPTrunc(V: Tmp, DestTy: TruncTy);
3371 else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
3372 uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(Ty: TruncTy);
3373 Tmp = Builder.CreateTrunc(
3374 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)ResSize));
3375 Tmp = Builder.CreateIntToPtr(V: Tmp, DestTy: TruncTy);
3376 } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
3377 uint64_t TmpSize =
3378 CGM.getDataLayout().getTypeSizeInBits(Ty: Tmp->getType());
3379 Tmp = Builder.CreatePtrToInt(
3380 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)TmpSize));
3381 Tmp = Builder.CreateTrunc(V: Tmp, DestTy: TruncTy);
3382 } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
3383 Tmp = Builder.CreateZExtOrTrunc(V: Tmp, DestTy: TruncTy);
3384 } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
3385 Tmp = Builder.CreateBitCast(V: Tmp, DestTy: TruncTy);
3386 }
3387 }
3388
3389 ApplyAtomGroup Grp(CGF.getDebugInfo());
3390 LValue Dest = ResultRegDests[i];
3391
3392 if (RequiresCast) {
3393 unsigned Size = getContext().getTypeSize(T: ResultRegQualTys[i]);
3394 Address A = Dest.getAddress().withElementType(ElemTy: ResultRegTypes[i]);
3395
3396 if (getTargetHooks().isScalarizableAsmOperand(CGF, Ty: TruncTy)) {
3397 llvm::StoreInst *S =
3398 Builder.CreateStore(Val: Tmp, Addr: A, IsVolatile: Dest.isVolatileQualified());
3399 CGF.addInstToCurrentSourceAtom(KeyInstruction: S, Backup: S->getValueOperand());
3400 continue;
3401 }
3402
3403 QualType Ty = getContext().getIntTypeForBitwidth(DestWidth: Size, /*Signed=*/false);
3404 if (Ty.isNull()) {
3405 const Expr *OutExpr = S.getOutputExpr(i);
3406 CGM.getDiags().Report(Loc: OutExpr->getExprLoc(),
3407 DiagID: diag::err_store_value_to_reg);
3408 return;
3409 }
3410
3411 // The output's type may not be an integer: an output tied to a larger
3412 // input has the input's type (e.g., a pointer), and a mask register
3413 // output ("k" on X86) is a vector of i1. Convert it to an integer of the
3414 // same size, then store only the destination's bits.
3415 llvm::Type *IntTy = llvm::IntegerType::get(C&: CTX, NumBits: Size);
3416 uint64_t TmpSize = CGM.getDataLayout().getTypeSizeInBits(Ty: Tmp->getType());
3417 Tmp = Builder.CreateBitOrPointerCast(
3418 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)TmpSize));
3419 Tmp = Builder.CreateTrunc(V: Tmp, DestTy: IntTy);
3420
3421 // The store writes all of the destination's members at once, so it must
3422 // be allowed to alias an access to any of them. A TBAA access type must
3423 // be a scalar, so use may-alias (char), which aliases any access. Keep
3424 // the destination's qualifiers, such as volatile.
3425 Dest = CGF.MakeAddrLValue(
3426 Addr: Dest.getAddress().withElementType(ElemTy: IntTy),
3427 T: getContext().getQualifiedType(T: Ty, Qs: Dest.getQuals()),
3428 BaseInfo: Dest.getBaseInfo(), TBAAInfo: TBAAAccessInfo::getMayAliasInfo());
3429 }
3430
3431 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Tmp), Dst: Dest);
3432 }
3433}
3434
3435LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
3436 const RecordDecl *RD = S.getCapturedRecordDecl();
3437 CanQualType RecordTy = getContext().getCanonicalTagType(TD: RD);
3438
3439 // Initialize the captured struct.
3440 LValue SlotLV =
3441 MakeAddrLValue(Addr: CreateMemTemp(T: RecordTy, Name: "agg.captured"), T: RecordTy);
3442
3443 RecordDecl::field_iterator CurField = RD->field_begin();
3444 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
3445 E = S.capture_init_end();
3446 I != E; ++I, ++CurField) {
3447 LValue LV = EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
3448 if (CurField->hasCapturedVLAType()) {
3449 EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
3450 } else {
3451 EmitInitializerForField(Field: *CurField, LHS: LV, Init: *I);
3452 }
3453 }
3454
3455 return SlotLV;
3456}
3457
3458/// Generate an outlined function for the body of a CapturedStmt, store any
3459/// captured variables into the captured struct, and call the outlined function.
3460llvm::Function *
3461CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
3462 LValue CapStruct = InitCapturedStruct(S);
3463
3464 // Emit the CapturedDecl
3465 CodeGenFunction CGF(CGM, true);
3466 CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
3467 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
3468 delete CGF.CapturedStmtInfo;
3469
3470 // Emit call to the helper function.
3471 EmitCallOrInvoke(Callee: F, Args: CapStruct.getPointer(CGF&: *this));
3472
3473 return F;
3474}
3475
3476Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
3477 LValue CapStruct = InitCapturedStruct(S);
3478 return CapStruct.getAddress();
3479}
3480
3481/// Creates the outlined function for a CapturedStmt.
3482llvm::Function *
3483CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
3484 assert(CapturedStmtInfo &&
3485 "CapturedStmtInfo should be set when generating the captured function");
3486 const CapturedDecl *CD = S.getCapturedDecl();
3487 const RecordDecl *RD = S.getCapturedRecordDecl();
3488 SourceLocation Loc = S.getBeginLoc();
3489 assert(CD->hasBody() && "missing CapturedDecl body");
3490
3491 // Build the argument list.
3492 ASTContext &Ctx = CGM.getContext();
3493 FunctionArgList Args;
3494 Args.append(in_start: CD->param_begin(), in_end: CD->param_end());
3495
3496 // Create the function declaration.
3497 const CGFunctionInfo &FuncInfo =
3498 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
3499 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
3500
3501 llvm::Function *F =
3502 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
3503 N: CapturedStmtInfo->getHelperName(), M: &CGM.getModule());
3504 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
3505 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3506 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3507 if (CD->isNothrow())
3508 F->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3509
3510 // Generate the function.
3511 StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc: CD->getLocation(),
3512 StartLoc: CD->getBody()->getBeginLoc());
3513 // Set the context parameter in CapturedStmtInfo.
3514 Address DeclPtr = GetAddrOfLocalVar(VD: CD->getContextParam());
3515 CapturedStmtInfo->setContextValue(Builder.CreateLoad(Addr: DeclPtr));
3516
3517 // Initialize variable-length arrays.
3518 LValue Base = MakeNaturalAlignRawAddrLValue(
3519 V: CapturedStmtInfo->getContextValue(), T: Ctx.getCanonicalTagType(TD: RD));
3520 for (auto *FD : RD->fields()) {
3521 if (FD->hasCapturedVLAType()) {
3522 auto *ExprArg =
3523 EmitLoadOfLValue(V: EmitLValueForField(Base, Field: FD), Loc: S.getBeginLoc())
3524 .getScalarVal();
3525 auto VAT = FD->getCapturedVLAType();
3526 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
3527 }
3528 }
3529
3530 // If 'this' is captured, load it into CXXThisValue.
3531 if (CapturedStmtInfo->isCXXThisExprCaptured()) {
3532 FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
3533 LValue ThisLValue = EmitLValueForField(Base, Field: FD);
3534 CXXThisValue = EmitLoadOfLValue(V: ThisLValue, Loc).getScalarVal();
3535 }
3536
3537 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
3538 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
3539 FinishFunction(EndLoc: CD->getBodyRBrace());
3540
3541 return F;
3542}
3543
3544// Returns the first convergence entry/loop/anchor instruction found in |BB|.
3545// std::nullptr otherwise.
3546static llvm::ConvergenceControlInst *getConvergenceToken(llvm::BasicBlock *BB) {
3547 for (auto &I : *BB) {
3548 if (auto *CI = dyn_cast<llvm::ConvergenceControlInst>(Val: &I))
3549 return CI;
3550 }
3551 return nullptr;
3552}
3553
3554void CodeGenFunction::addConvergenceControlBundle(
3555 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
3556 if (!CGM.shouldEmitConvergenceTokens())
3557 return;
3558
3559 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3560 assert(ParentToken);
3561 Bundles.emplace_back(Args: "convergencectrl", Args&: ParentToken);
3562}
3563
3564void CodeGenFunction::setConvergentCallSite(llvm::CallBase *Call,
3565 bool IsConvergent) {
3566 if (IsConvergent && !Call->isConvergent())
3567 Call->setConvergent();
3568
3569 assert((IsConvergent || !Call->isConvergent() ||
3570 !CGM.shouldEmitConvergenceTokens()) &&
3571 "convergent call is missing a convergence control token");
3572}
3573
3574llvm::ConvergenceControlInst *
3575CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {
3576 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3577 assert(ParentToken);
3578 return llvm::ConvergenceControlInst::CreateLoop(BB&: *BB, Parent: ParentToken);
3579}
3580
3581llvm::ConvergenceControlInst *
3582CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
3583 llvm::BasicBlock *BB = &F->getEntryBlock();
3584 llvm::ConvergenceControlInst *Token = getConvergenceToken(BB);
3585 if (Token)
3586 return Token;
3587
3588 // Adding a convergence token requires the function to be marked as
3589 // convergent.
3590 F->setConvergent();
3591 return llvm::ConvergenceControlInst::CreateEntry(BB&: *BB);
3592}
3593