1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Stmt nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGDebugInfo.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "CodeGenPGO.h"
18#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Expr.h"
21#include "clang/AST/Stmt.h"
22#include "clang/AST/StmtSYCL.h"
23#include "clang/AST/StmtVisitor.h"
24#include "clang/Basic/Builtins.h"
25#include "clang/Basic/DiagnosticSema.h"
26#include "clang/Basic/PrettyStackTrace.h"
27#include "clang/Basic/SourceManager.h"
28#include "clang/Basic/TargetInfo.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/IR/Assumptions.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/InlineAsm.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/MDBuilder.h"
38#include "llvm/Support/SaveAndRestore.h"
39#include <optional>
40
41using namespace clang;
42using namespace CodeGen;
43
44//===----------------------------------------------------------------------===//
45// Statement Emission
46//===----------------------------------------------------------------------===//
47
48void CodeGenFunction::EmitStopPoint(const Stmt *S) {
49 if (CGDebugInfo *DI = getDebugInfo()) {
50 SourceLocation Loc;
51 Loc = S->getBeginLoc();
52 DI->EmitLocation(Builder, Loc);
53
54 LastStopPoint = Loc;
55 }
56}
57
58void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
59 assert(S && "Null statement?");
60 PGO->setCurrentStmt(S);
61
62 // These statements have their own debug info handling.
63 if (EmitSimpleStmt(S, Attrs))
64 return;
65
66 // Check if we are generating unreachable code.
67 if (!HaveInsertPoint()) {
68 // If so, and the statement doesn't contain a label, then we do not need to
69 // generate actual code. This is safe because (1) the current point is
70 // unreachable, so we don't need to execute the code, and (2) we've already
71 // handled the statements which update internal data structures (like the
72 // local variable map) which could be used by subsequent statements.
73 if (!ContainsLabel(S)) {
74 // Verify that any decl statements were handled as simple, they may be in
75 // scope of subsequent reachable statements.
76 assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
77 PGO->markStmtMaybeUsed(S);
78 return;
79 }
80
81 // Otherwise, make a new block to hold the code.
82 EnsureInsertPoint();
83 }
84
85 // Generate a stoppoint if we are emitting debug info.
86 EmitStopPoint(S);
87
88 // Ignore all OpenMP directives except for simd if OpenMP with Simd is
89 // enabled.
90 if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
91 if (const auto *D = dyn_cast<OMPExecutableDirective>(Val: S)) {
92 EmitSimpleOMPExecutableDirective(D: *D);
93 return;
94 }
95 }
96
97 switch (S->getStmtClass()) {
98 case Stmt::NoStmtClass:
99 case Stmt::CXXCatchStmtClass:
100 case Stmt::SEHExceptStmtClass:
101 case Stmt::SEHFinallyStmtClass:
102 case Stmt::MSDependentExistsStmtClass:
103 case Stmt::UnresolvedSYCLKernelCallStmtClass:
104 llvm_unreachable("invalid statement class to emit generically");
105 case Stmt::NullStmtClass:
106 case Stmt::CompoundStmtClass:
107 case Stmt::DeclStmtClass:
108 case Stmt::LabelStmtClass:
109 case Stmt::AttributedStmtClass:
110 case Stmt::GotoStmtClass:
111 case Stmt::BreakStmtClass:
112 case Stmt::ContinueStmtClass:
113 case Stmt::DefaultStmtClass:
114 case Stmt::CaseStmtClass:
115 case Stmt::DeferStmtClass:
116 case Stmt::SEHLeaveStmtClass:
117 case Stmt::SYCLKernelCallStmtClass:
118 llvm_unreachable("should have emitted these statements as simple");
119
120#define STMT(Type, Base)
121#define ABSTRACT_STMT(Op)
122#define EXPR(Type, Base) \
123 case Stmt::Type##Class:
124#include "clang/AST/StmtNodes.inc"
125 {
126 // Remember the block we came in on.
127 llvm::BasicBlock *incoming = Builder.GetInsertBlock();
128 assert(incoming && "expression emission must have an insertion point");
129
130 EmitIgnoredExpr(E: cast<Expr>(Val: S));
131
132 llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
133 assert(outgoing && "expression emission cleared block!");
134
135 // The expression emitters assume (reasonably!) that the insertion
136 // point is always set. To maintain that, the call-emission code
137 // for noreturn functions has to enter a new block with no
138 // predecessors. We want to kill that block and mark the current
139 // insertion point unreachable in the common case of a call like
140 // "exit();". Since expression emission doesn't otherwise create
141 // blocks with no predecessors, we can just test for that.
142 // However, we must be careful not to do this to our incoming
143 // block, because *statement* emission does sometimes create
144 // reachable blocks which will have no predecessors until later in
145 // the function. This occurs with, e.g., labels that are not
146 // reachable by fallthrough.
147 if (incoming != outgoing && outgoing->use_empty()) {
148 outgoing->eraseFromParent();
149 Builder.ClearInsertionPoint();
150 }
151 break;
152 }
153
154 case Stmt::IndirectGotoStmtClass:
155 EmitIndirectGotoStmt(S: cast<IndirectGotoStmt>(Val: *S)); break;
156
157 case Stmt::IfStmtClass: EmitIfStmt(S: cast<IfStmt>(Val: *S)); break;
158 case Stmt::WhileStmtClass: EmitWhileStmt(S: cast<WhileStmt>(Val: *S), Attrs); break;
159 case Stmt::DoStmtClass: EmitDoStmt(S: cast<DoStmt>(Val: *S), Attrs); break;
160 case Stmt::ForStmtClass: EmitForStmt(S: cast<ForStmt>(Val: *S), Attrs); break;
161
162 case Stmt::ReturnStmtClass: EmitReturnStmt(S: cast<ReturnStmt>(Val: *S)); break;
163
164 case Stmt::SwitchStmtClass: EmitSwitchStmt(S: cast<SwitchStmt>(Val: *S)); break;
165 case Stmt::GCCAsmStmtClass: // Intentional fall-through.
166 case Stmt::MSAsmStmtClass: EmitAsmStmt(S: cast<AsmStmt>(Val: *S)); break;
167 case Stmt::CoroutineBodyStmtClass:
168 EmitCoroutineBody(S: cast<CoroutineBodyStmt>(Val: *S));
169 break;
170 case Stmt::CoreturnStmtClass:
171 EmitCoreturnStmt(S: cast<CoreturnStmt>(Val: *S));
172 break;
173 case Stmt::CapturedStmtClass: {
174 const CapturedStmt *CS = cast<CapturedStmt>(Val: S);
175 EmitCapturedStmt(S: *CS, K: CS->getCapturedRegionKind());
176 }
177 break;
178 case Stmt::ObjCAtTryStmtClass:
179 EmitObjCAtTryStmt(S: cast<ObjCAtTryStmt>(Val: *S));
180 break;
181 case Stmt::ObjCAtCatchStmtClass:
182 llvm_unreachable(
183 "@catch statements should be handled by EmitObjCAtTryStmt");
184 case Stmt::ObjCAtFinallyStmtClass:
185 llvm_unreachable(
186 "@finally statements should be handled by EmitObjCAtTryStmt");
187 case Stmt::ObjCAtThrowStmtClass:
188 EmitObjCAtThrowStmt(S: cast<ObjCAtThrowStmt>(Val: *S));
189 break;
190 case Stmt::ObjCAtSynchronizedStmtClass:
191 EmitObjCAtSynchronizedStmt(S: cast<ObjCAtSynchronizedStmt>(Val: *S));
192 break;
193 case Stmt::ObjCForCollectionStmtClass:
194 EmitObjCForCollectionStmt(S: cast<ObjCForCollectionStmt>(Val: *S));
195 break;
196 case Stmt::ObjCAutoreleasePoolStmtClass:
197 EmitObjCAutoreleasePoolStmt(S: cast<ObjCAutoreleasePoolStmt>(Val: *S));
198 break;
199
200 case Stmt::CXXTryStmtClass:
201 EmitCXXTryStmt(S: cast<CXXTryStmt>(Val: *S));
202 break;
203 case Stmt::CXXForRangeStmtClass:
204 EmitCXXForRangeStmt(S: cast<CXXForRangeStmt>(Val: *S), Attrs);
205 break;
206 case Stmt::CXXExpansionStmtPatternClass:
207 llvm_unreachable("unexpanded expansion statements should not be emitted");
208 case Stmt::CXXExpansionStmtInstantiationClass:
209 EmitCXXExpansionStmtInstantiation(S: cast<CXXExpansionStmtInstantiation>(Val: *S));
210 break;
211 case Stmt::SEHTryStmtClass:
212 EmitSEHTryStmt(S: cast<SEHTryStmt>(Val: *S));
213 break;
214 case Stmt::OMPMetaDirectiveClass:
215 EmitOMPMetaDirective(S: cast<OMPMetaDirective>(Val: *S));
216 break;
217 case Stmt::OMPCanonicalLoopClass:
218 EmitOMPCanonicalLoop(S: cast<OMPCanonicalLoop>(Val: S));
219 break;
220 case Stmt::OMPParallelDirectiveClass:
221 EmitOMPParallelDirective(S: cast<OMPParallelDirective>(Val: *S));
222 break;
223 case Stmt::OMPSimdDirectiveClass:
224 EmitOMPSimdDirective(S: cast<OMPSimdDirective>(Val: *S));
225 break;
226 case Stmt::OMPTileDirectiveClass:
227 EmitOMPTileDirective(S: cast<OMPTileDirective>(Val: *S));
228 break;
229 case Stmt::OMPStripeDirectiveClass:
230 EmitOMPStripeDirective(S: cast<OMPStripeDirective>(Val: *S));
231 break;
232 case Stmt::OMPUnrollDirectiveClass:
233 EmitOMPUnrollDirective(S: cast<OMPUnrollDirective>(Val: *S));
234 break;
235 case Stmt::OMPReverseDirectiveClass:
236 EmitOMPReverseDirective(S: cast<OMPReverseDirective>(Val: *S));
237 break;
238 case Stmt::OMPSplitDirectiveClass:
239 EmitOMPSplitDirective(S: cast<OMPSplitDirective>(Val: *S));
240 break;
241 case Stmt::OMPInterchangeDirectiveClass:
242 EmitOMPInterchangeDirective(S: cast<OMPInterchangeDirective>(Val: *S));
243 break;
244 case Stmt::OMPFuseDirectiveClass:
245 EmitOMPFuseDirective(S: cast<OMPFuseDirective>(Val: *S));
246 break;
247 case Stmt::OMPForDirectiveClass:
248 EmitOMPForDirective(S: cast<OMPForDirective>(Val: *S));
249 break;
250 case Stmt::OMPForSimdDirectiveClass:
251 EmitOMPForSimdDirective(S: cast<OMPForSimdDirective>(Val: *S));
252 break;
253 case Stmt::OMPSectionsDirectiveClass:
254 EmitOMPSectionsDirective(S: cast<OMPSectionsDirective>(Val: *S));
255 break;
256 case Stmt::OMPSectionDirectiveClass:
257 EmitOMPSectionDirective(S: cast<OMPSectionDirective>(Val: *S));
258 break;
259 case Stmt::OMPSingleDirectiveClass:
260 EmitOMPSingleDirective(S: cast<OMPSingleDirective>(Val: *S));
261 break;
262 case Stmt::OMPMasterDirectiveClass:
263 EmitOMPMasterDirective(S: cast<OMPMasterDirective>(Val: *S));
264 break;
265 case Stmt::OMPCriticalDirectiveClass:
266 EmitOMPCriticalDirective(S: cast<OMPCriticalDirective>(Val: *S));
267 break;
268 case Stmt::OMPParallelForDirectiveClass:
269 EmitOMPParallelForDirective(S: cast<OMPParallelForDirective>(Val: *S));
270 break;
271 case Stmt::OMPParallelForSimdDirectiveClass:
272 EmitOMPParallelForSimdDirective(S: cast<OMPParallelForSimdDirective>(Val: *S));
273 break;
274 case Stmt::OMPParallelMasterDirectiveClass:
275 EmitOMPParallelMasterDirective(S: cast<OMPParallelMasterDirective>(Val: *S));
276 break;
277 case Stmt::OMPParallelSectionsDirectiveClass:
278 EmitOMPParallelSectionsDirective(S: cast<OMPParallelSectionsDirective>(Val: *S));
279 break;
280 case Stmt::OMPTaskDirectiveClass:
281 EmitOMPTaskDirective(S: cast<OMPTaskDirective>(Val: *S));
282 break;
283 case Stmt::OMPTaskyieldDirectiveClass:
284 EmitOMPTaskyieldDirective(S: cast<OMPTaskyieldDirective>(Val: *S));
285 break;
286 case Stmt::OMPErrorDirectiveClass:
287 EmitOMPErrorDirective(S: cast<OMPErrorDirective>(Val: *S));
288 break;
289 case Stmt::OMPBarrierDirectiveClass:
290 EmitOMPBarrierDirective(S: cast<OMPBarrierDirective>(Val: *S));
291 break;
292 case Stmt::OMPTaskwaitDirectiveClass:
293 EmitOMPTaskwaitDirective(S: cast<OMPTaskwaitDirective>(Val: *S));
294 break;
295 case Stmt::OMPTaskgroupDirectiveClass:
296 EmitOMPTaskgroupDirective(S: cast<OMPTaskgroupDirective>(Val: *S));
297 break;
298 case Stmt::OMPFlushDirectiveClass:
299 EmitOMPFlushDirective(S: cast<OMPFlushDirective>(Val: *S));
300 break;
301 case Stmt::OMPDepobjDirectiveClass:
302 EmitOMPDepobjDirective(S: cast<OMPDepobjDirective>(Val: *S));
303 break;
304 case Stmt::OMPScanDirectiveClass:
305 EmitOMPScanDirective(S: cast<OMPScanDirective>(Val: *S));
306 break;
307 case Stmt::OMPOrderedStandaloneDirectiveClass:
308 EmitOMPOrderedStandaloneDirective(S: cast<OMPOrderedStandaloneDirective>(Val: *S));
309 break;
310 case Stmt::OMPOrderedBlockAssocDirectiveClass:
311 EmitOMPOrderedBlockAssocDirective(S: cast<OMPOrderedBlockAssocDirective>(Val: *S));
312 break;
313 case Stmt::OMPAtomicDirectiveClass:
314 EmitOMPAtomicDirective(S: cast<OMPAtomicDirective>(Val: *S));
315 break;
316 case Stmt::OMPTargetDirectiveClass:
317 EmitOMPTargetDirective(S: cast<OMPTargetDirective>(Val: *S));
318 break;
319 case Stmt::OMPTeamsDirectiveClass:
320 EmitOMPTeamsDirective(S: cast<OMPTeamsDirective>(Val: *S));
321 break;
322 case Stmt::OMPCancellationPointDirectiveClass:
323 EmitOMPCancellationPointDirective(S: cast<OMPCancellationPointDirective>(Val: *S));
324 break;
325 case Stmt::OMPCancelDirectiveClass:
326 EmitOMPCancelDirective(S: cast<OMPCancelDirective>(Val: *S));
327 break;
328 case Stmt::OMPTargetDataDirectiveClass:
329 EmitOMPTargetDataDirective(S: cast<OMPTargetDataDirective>(Val: *S));
330 break;
331 case Stmt::OMPTargetEnterDataDirectiveClass:
332 EmitOMPTargetEnterDataDirective(S: cast<OMPTargetEnterDataDirective>(Val: *S));
333 break;
334 case Stmt::OMPTargetExitDataDirectiveClass:
335 EmitOMPTargetExitDataDirective(S: cast<OMPTargetExitDataDirective>(Val: *S));
336 break;
337 case Stmt::OMPTargetParallelDirectiveClass:
338 EmitOMPTargetParallelDirective(S: cast<OMPTargetParallelDirective>(Val: *S));
339 break;
340 case Stmt::OMPTargetParallelForDirectiveClass:
341 EmitOMPTargetParallelForDirective(S: cast<OMPTargetParallelForDirective>(Val: *S));
342 break;
343 case Stmt::OMPTaskLoopDirectiveClass:
344 EmitOMPTaskLoopDirective(S: cast<OMPTaskLoopDirective>(Val: *S));
345 break;
346 case Stmt::OMPTaskLoopSimdDirectiveClass:
347 EmitOMPTaskLoopSimdDirective(S: cast<OMPTaskLoopSimdDirective>(Val: *S));
348 break;
349 case Stmt::OMPMasterTaskLoopDirectiveClass:
350 EmitOMPMasterTaskLoopDirective(S: cast<OMPMasterTaskLoopDirective>(Val: *S));
351 break;
352 case Stmt::OMPMaskedTaskLoopDirectiveClass:
353 EmitOMPMaskedTaskLoopDirective(S: cast<OMPMaskedTaskLoopDirective>(Val: *S));
354 break;
355 case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
356 EmitOMPMasterTaskLoopSimdDirective(
357 S: cast<OMPMasterTaskLoopSimdDirective>(Val: *S));
358 break;
359 case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
360 EmitOMPMaskedTaskLoopSimdDirective(
361 S: cast<OMPMaskedTaskLoopSimdDirective>(Val: *S));
362 break;
363 case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
364 EmitOMPParallelMasterTaskLoopDirective(
365 S: cast<OMPParallelMasterTaskLoopDirective>(Val: *S));
366 break;
367 case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
368 EmitOMPParallelMaskedTaskLoopDirective(
369 S: cast<OMPParallelMaskedTaskLoopDirective>(Val: *S));
370 break;
371 case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
372 EmitOMPParallelMasterTaskLoopSimdDirective(
373 S: cast<OMPParallelMasterTaskLoopSimdDirective>(Val: *S));
374 break;
375 case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
376 EmitOMPParallelMaskedTaskLoopSimdDirective(
377 S: cast<OMPParallelMaskedTaskLoopSimdDirective>(Val: *S));
378 break;
379 case Stmt::OMPDistributeDirectiveClass:
380 EmitOMPDistributeDirective(S: cast<OMPDistributeDirective>(Val: *S));
381 break;
382 case Stmt::OMPTargetUpdateDirectiveClass:
383 EmitOMPTargetUpdateDirective(S: cast<OMPTargetUpdateDirective>(Val: *S));
384 break;
385 case Stmt::OMPDistributeParallelForDirectiveClass:
386 EmitOMPDistributeParallelForDirective(
387 S: cast<OMPDistributeParallelForDirective>(Val: *S));
388 break;
389 case Stmt::OMPDistributeParallelForSimdDirectiveClass:
390 EmitOMPDistributeParallelForSimdDirective(
391 S: cast<OMPDistributeParallelForSimdDirective>(Val: *S));
392 break;
393 case Stmt::OMPDistributeSimdDirectiveClass:
394 EmitOMPDistributeSimdDirective(S: cast<OMPDistributeSimdDirective>(Val: *S));
395 break;
396 case Stmt::OMPTargetParallelForSimdDirectiveClass:
397 EmitOMPTargetParallelForSimdDirective(
398 S: cast<OMPTargetParallelForSimdDirective>(Val: *S));
399 break;
400 case Stmt::OMPTargetSimdDirectiveClass:
401 EmitOMPTargetSimdDirective(S: cast<OMPTargetSimdDirective>(Val: *S));
402 break;
403 case Stmt::OMPTeamsDistributeDirectiveClass:
404 EmitOMPTeamsDistributeDirective(S: cast<OMPTeamsDistributeDirective>(Val: *S));
405 break;
406 case Stmt::OMPTeamsDistributeSimdDirectiveClass:
407 EmitOMPTeamsDistributeSimdDirective(
408 S: cast<OMPTeamsDistributeSimdDirective>(Val: *S));
409 break;
410 case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
411 EmitOMPTeamsDistributeParallelForSimdDirective(
412 S: cast<OMPTeamsDistributeParallelForSimdDirective>(Val: *S));
413 break;
414 case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
415 EmitOMPTeamsDistributeParallelForDirective(
416 S: cast<OMPTeamsDistributeParallelForDirective>(Val: *S));
417 break;
418 case Stmt::OMPTargetTeamsDirectiveClass:
419 EmitOMPTargetTeamsDirective(S: cast<OMPTargetTeamsDirective>(Val: *S));
420 break;
421 case Stmt::OMPTargetTeamsDistributeDirectiveClass:
422 EmitOMPTargetTeamsDistributeDirective(
423 S: cast<OMPTargetTeamsDistributeDirective>(Val: *S));
424 break;
425 case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
426 EmitOMPTargetTeamsDistributeParallelForDirective(
427 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: *S));
428 break;
429 case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
430 EmitOMPTargetTeamsDistributeParallelForSimdDirective(
431 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: *S));
432 break;
433 case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
434 EmitOMPTargetTeamsDistributeSimdDirective(
435 S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: *S));
436 break;
437 case Stmt::OMPInteropDirectiveClass:
438 EmitOMPInteropDirective(S: cast<OMPInteropDirective>(Val: *S));
439 break;
440 case Stmt::OMPDispatchDirectiveClass:
441 CGM.ErrorUnsupported(S, Type: "OpenMP dispatch directive");
442 break;
443 case Stmt::OMPScopeDirectiveClass:
444 EmitOMPScopeDirective(S: cast<OMPScopeDirective>(Val: *S));
445 break;
446 case Stmt::OMPMaskedDirectiveClass:
447 EmitOMPMaskedDirective(S: cast<OMPMaskedDirective>(Val: *S));
448 break;
449 case Stmt::OMPGenericLoopDirectiveClass:
450 EmitOMPGenericLoopDirective(S: cast<OMPGenericLoopDirective>(Val: *S));
451 break;
452 case Stmt::OMPTeamsGenericLoopDirectiveClass:
453 EmitOMPTeamsGenericLoopDirective(S: cast<OMPTeamsGenericLoopDirective>(Val: *S));
454 break;
455 case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
456 EmitOMPTargetTeamsGenericLoopDirective(
457 S: cast<OMPTargetTeamsGenericLoopDirective>(Val: *S));
458 break;
459 case Stmt::OMPParallelGenericLoopDirectiveClass:
460 EmitOMPParallelGenericLoopDirective(
461 S: cast<OMPParallelGenericLoopDirective>(Val: *S));
462 break;
463 case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
464 EmitOMPTargetParallelGenericLoopDirective(
465 S: cast<OMPTargetParallelGenericLoopDirective>(Val: *S));
466 break;
467 case Stmt::OMPParallelMaskedDirectiveClass:
468 EmitOMPParallelMaskedDirective(S: cast<OMPParallelMaskedDirective>(Val: *S));
469 break;
470 case Stmt::OMPAssumeDirectiveClass:
471 EmitOMPAssumeDirective(S: cast<OMPAssumeDirective>(Val: *S));
472 break;
473 case Stmt::OpenACCComputeConstructClass:
474 EmitOpenACCComputeConstruct(S: cast<OpenACCComputeConstruct>(Val: *S));
475 break;
476 case Stmt::OpenACCLoopConstructClass:
477 EmitOpenACCLoopConstruct(S: cast<OpenACCLoopConstruct>(Val: *S));
478 break;
479 case Stmt::OpenACCCombinedConstructClass:
480 EmitOpenACCCombinedConstruct(S: cast<OpenACCCombinedConstruct>(Val: *S));
481 break;
482 case Stmt::OpenACCDataConstructClass:
483 EmitOpenACCDataConstruct(S: cast<OpenACCDataConstruct>(Val: *S));
484 break;
485 case Stmt::OpenACCEnterDataConstructClass:
486 EmitOpenACCEnterDataConstruct(S: cast<OpenACCEnterDataConstruct>(Val: *S));
487 break;
488 case Stmt::OpenACCExitDataConstructClass:
489 EmitOpenACCExitDataConstruct(S: cast<OpenACCExitDataConstruct>(Val: *S));
490 break;
491 case Stmt::OpenACCHostDataConstructClass:
492 EmitOpenACCHostDataConstruct(S: cast<OpenACCHostDataConstruct>(Val: *S));
493 break;
494 case Stmt::OpenACCWaitConstructClass:
495 EmitOpenACCWaitConstruct(S: cast<OpenACCWaitConstruct>(Val: *S));
496 break;
497 case Stmt::OpenACCInitConstructClass:
498 EmitOpenACCInitConstruct(S: cast<OpenACCInitConstruct>(Val: *S));
499 break;
500 case Stmt::OpenACCShutdownConstructClass:
501 EmitOpenACCShutdownConstruct(S: cast<OpenACCShutdownConstruct>(Val: *S));
502 break;
503 case Stmt::OpenACCSetConstructClass:
504 EmitOpenACCSetConstruct(S: cast<OpenACCSetConstruct>(Val: *S));
505 break;
506 case Stmt::OpenACCUpdateConstructClass:
507 EmitOpenACCUpdateConstruct(S: cast<OpenACCUpdateConstruct>(Val: *S));
508 break;
509 case Stmt::OpenACCAtomicConstructClass:
510 EmitOpenACCAtomicConstruct(S: cast<OpenACCAtomicConstruct>(Val: *S));
511 break;
512 case Stmt::OpenACCCacheConstructClass:
513 EmitOpenACCCacheConstruct(S: cast<OpenACCCacheConstruct>(Val: *S));
514 break;
515 }
516}
517
518bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
519 ArrayRef<const Attr *> Attrs) {
520 switch (S->getStmtClass()) {
521 default:
522 return false;
523 case Stmt::NullStmtClass:
524 break;
525 case Stmt::CompoundStmtClass:
526 EmitCompoundStmt(S: cast<CompoundStmt>(Val: *S));
527 break;
528 case Stmt::DeclStmtClass:
529 EmitDeclStmt(S: cast<DeclStmt>(Val: *S));
530 break;
531 case Stmt::LabelStmtClass:
532 EmitLabelStmt(S: cast<LabelStmt>(Val: *S));
533 break;
534 case Stmt::AttributedStmtClass:
535 EmitAttributedStmt(S: cast<AttributedStmt>(Val: *S));
536 break;
537 case Stmt::GotoStmtClass:
538 EmitGotoStmt(S: cast<GotoStmt>(Val: *S));
539 break;
540 case Stmt::BreakStmtClass:
541 EmitBreakStmt(S: cast<BreakStmt>(Val: *S));
542 break;
543 case Stmt::ContinueStmtClass:
544 EmitContinueStmt(S: cast<ContinueStmt>(Val: *S));
545 break;
546 case Stmt::DefaultStmtClass:
547 EmitDefaultStmt(S: cast<DefaultStmt>(Val: *S), Attrs);
548 break;
549 case Stmt::CaseStmtClass:
550 EmitCaseStmt(S: cast<CaseStmt>(Val: *S), Attrs);
551 break;
552 case Stmt::DeferStmtClass:
553 EmitDeferStmt(S: cast<DeferStmt>(Val: *S));
554 break;
555 case Stmt::SEHLeaveStmtClass:
556 EmitSEHLeaveStmt(S: cast<SEHLeaveStmt>(Val: *S));
557 break;
558 case Stmt::SYCLKernelCallStmtClass:
559 EmitSYCLKernelCallStmt(S: cast<SYCLKernelCallStmt>(Val: *S));
560 break;
561 }
562 return true;
563}
564
565/// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true,
566/// this captures the expression result of the last sub-statement and returns it
567/// (for use by the statement expression extension).
568Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
569 AggValueSlot AggSlot) {
570 PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
571 "LLVM IR generation of compound statement ('{}')");
572
573 // Keep track of the current cleanup stack depth, including debug scopes.
574 LexicalScope Scope(*this, S.getSourceRange());
575
576 return EmitCompoundStmtWithoutScope(S, GetLast, AVS: AggSlot);
577}
578
579Address
580CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
581 bool GetLast,
582 AggValueSlot AggSlot) {
583
584 for (CompoundStmt::const_body_iterator I = S.body_begin(),
585 E = S.body_end() - GetLast;
586 I != E; ++I)
587 EmitStmt(S: *I);
588
589 Address RetAlloca = Address::invalid();
590 if (GetLast) {
591 // We have to special case labels here. They are statements, but when put
592 // at the end of a statement expression, they yield the value of their
593 // subexpression. Handle this by walking through all labels we encounter,
594 // emitting them before we evaluate the subexpr.
595 // Similar issues arise for attributed statements.
596 const Stmt *LastStmt = S.body_back();
597 while (!isa<Expr>(Val: LastStmt)) {
598 if (const auto *LS = dyn_cast<LabelStmt>(Val: LastStmt)) {
599 EmitLabel(D: LS->getDecl());
600 LastStmt = LS->getSubStmt();
601 } else if (const auto *AS = dyn_cast<AttributedStmt>(Val: LastStmt)) {
602 // FIXME: Update this if we ever have attributes that affect the
603 // semantics of an expression.
604 LastStmt = AS->getSubStmt();
605 } else {
606 llvm_unreachable("unknown value statement");
607 }
608 }
609
610 EnsureInsertPoint();
611
612 const Expr *E = cast<Expr>(Val: LastStmt);
613 QualType ExprTy = E->getType();
614 if (hasAggregateEvaluationKind(T: ExprTy)) {
615 EmitAggExpr(E, AS: AggSlot);
616 } else {
617 // We can't return an RValue here because there might be cleanups at
618 // the end of the StmtExpr. Because of that, we have to emit the result
619 // here into a temporary alloca.
620 RetAlloca = CreateMemTempWithoutCast(T: ExprTy);
621 EmitAnyExprToMem(E, Location: RetAlloca, Quals: Qualifiers(),
622 /*IsInit*/ IsInitializer: false);
623 }
624 }
625
626 return RetAlloca;
627}
628
629void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
630 llvm::UncondBrInst *BI = dyn_cast<llvm::UncondBrInst>(Val: BB->getTerminator());
631
632 // If there is a cleanup stack, then we it isn't worth trying to
633 // simplify this block (we would need to remove it from the scope map
634 // and cleanup entry).
635 if (!EHStack.empty())
636 return;
637
638 // Can only simplify direct branches.
639 if (!BI)
640 return;
641
642 // Can only simplify empty blocks.
643 if (BI->getIterator() != BB->begin())
644 return;
645
646 BB->replaceAllUsesWith(V: BI->getSuccessor());
647 BI->eraseFromParent();
648 BB->eraseFromParent();
649}
650
651void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
652 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
653
654 // Fall out of the current block (if necessary).
655 EmitBranch(Block: BB);
656
657 if (IsFinished && BB->use_empty()) {
658 delete BB;
659 return;
660 }
661
662 // Place the block after the current block, if possible, or else at
663 // the end of the function.
664 if (CurBB && CurBB->getParent())
665 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
666 else
667 CurFn->insert(Position: CurFn->end(), BB);
668 Builder.SetInsertPoint(BB);
669}
670
671void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
672 // Emit a branch from the current block to the target one if this
673 // was a real block. If this was just a fall-through block after a
674 // terminator, don't emit it.
675 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
676
677 if (!CurBB || CurBB->hasTerminator()) {
678 // If there is no insert point or the previous block is already
679 // terminated, don't touch it.
680 } else {
681 // Otherwise, create a fall-through branch.
682 Builder.CreateBr(Dest: Target);
683 }
684
685 Builder.ClearInsertionPoint();
686}
687
688void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
689 bool inserted = false;
690 for (llvm::User *u : block->users()) {
691 if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(Val: u)) {
692 CurFn->insert(Position: std::next(x: insn->getParent()->getIterator()), BB: block);
693 inserted = true;
694 break;
695 }
696 }
697
698 if (!inserted)
699 CurFn->insert(Position: CurFn->end(), BB: block);
700
701 Builder.SetInsertPoint(block);
702}
703
704CodeGenFunction::JumpDest
705CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
706 JumpDest &Dest = LabelMap[D];
707 if (Dest.isValid()) return Dest;
708
709 // Create, but don't insert, the new block.
710 Dest = JumpDest(createBasicBlock(name: D->getName()),
711 EHScopeStack::stable_iterator::invalid(),
712 NextCleanupDestIndex++);
713 return Dest;
714}
715
716void CodeGenFunction::EmitLabel(const LabelDecl *D) {
717 // Add this label to the current lexical scope if we're within any
718 // normal cleanups. Jumps "in" to this label --- when permitted by
719 // the language --- may need to be routed around such cleanups.
720 if (EHStack.hasNormalCleanups() && CurLexicalScope)
721 CurLexicalScope->addLabel(label: D);
722
723 JumpDest &Dest = LabelMap[D];
724
725 // If we didn't need a forward reference to this label, just go
726 // ahead and create a destination at the current scope.
727 if (!Dest.isValid()) {
728 Dest = getJumpDestInCurrentScope(Name: D->getName());
729
730 // Otherwise, we need to give this label a target depth and remove
731 // it from the branch-fixups list.
732 } else {
733 assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
734 Dest.setScopeDepth(EHStack.stable_begin());
735 ResolveBranchFixups(Target: Dest.getBlock());
736 }
737
738 EmitBlock(BB: Dest.getBlock());
739
740 // Emit debug info for labels.
741 if (CGDebugInfo *DI = getDebugInfo()) {
742 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
743 DI->setLocation(D->getLocation());
744 DI->EmitLabel(D, Builder);
745 }
746 }
747
748 incrementProfileCounter(S: D->getStmt());
749}
750
751/// Change the cleanup scope of the labels in this lexical scope to
752/// match the scope of the enclosing context.
753void CodeGenFunction::LexicalScope::rescopeLabels() {
754 assert(!Labels.empty());
755 EHScopeStack::stable_iterator innermostScope
756 = CGF.EHStack.getInnermostNormalCleanup();
757
758 // Change the scope depth of all the labels.
759 for (const LabelDecl *Label : Labels) {
760 assert(CGF.LabelMap.count(Label));
761 JumpDest &dest = CGF.LabelMap.find(Val: Label)->second;
762 assert(dest.getScopeDepth().isValid());
763 assert(innermostScope.encloses(dest.getScopeDepth()));
764 dest.setScopeDepth(innermostScope);
765 }
766
767 // Reparent the labels if the new scope also has cleanups.
768 if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
769 ParentScope->Labels.append(in_start: Labels.begin(), in_end: Labels.end());
770 }
771}
772
773
774void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
775 EmitLabel(D: S.getDecl());
776
777 // IsEHa - emit eha.scope.begin if it's a side entry of a scope
778 if (getLangOpts().EHAsynch && S.isSideEntry())
779 EmitSehCppScopeBegin();
780
781 EmitStmt(S: S.getSubStmt());
782}
783
784void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
785 bool nomerge = InNoMergeAttributedStmt;
786 bool noinline = InNoInlineAttributedStmt;
787 bool alwaysinline = InAlwaysInlineAttributedStmt;
788 bool noconvergent = InNoConvergentAttributedStmt;
789 StringRef amdgpuAVMode = AMDGPUAvailableVisibleMode;
790 HLSLControlFlowHintAttr::Spelling flattenOrBranch = HLSLControlFlowAttr;
791 const CallExpr *musttail = MustTailCall;
792 const AtomicAttr *AA = nullptr;
793
794 for (const auto *A : S.getAttrs()) {
795 switch (A->getKind()) {
796 default:
797 break;
798 case attr::NoMerge:
799 nomerge = true;
800 break;
801 case attr::NoInline:
802 noinline = true;
803 alwaysinline = false;
804 break;
805 case attr::AlwaysInline:
806 alwaysinline = true;
807 noinline = false;
808 break;
809 case attr::NoConvergent:
810 noconvergent = true;
811 break;
812 case attr::MustTail: {
813 const Stmt *Sub = S.getSubStmt();
814 const ReturnStmt *R = cast<ReturnStmt>(Val: Sub);
815 musttail = cast<CallExpr>(Val: R->getRetValue()->IgnoreParens());
816 } break;
817 case attr::CXXAssume: {
818 const Expr *Assumption = cast<CXXAssumeAttr>(Val: A)->getAssumption();
819 if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() &&
820 !Assumption->HasSideEffects(Ctx: getContext())) {
821 llvm::Value *AssumptionVal = EmitCheckedArgForAssume(E: Assumption);
822 Builder.CreateAssumption(Cond: AssumptionVal);
823 }
824 } break;
825 case attr::Atomic:
826 AA = cast<AtomicAttr>(Val: A);
827 break;
828 case attr::AMDGPUAvailableVisible:
829 amdgpuAVMode = cast<AMDGPUAvailableVisibleAttr>(Val: A)->getMode();
830 break;
831 case attr::HLSLControlFlowHint: {
832 flattenOrBranch = cast<HLSLControlFlowHintAttr>(Val: A)->getSemanticSpelling();
833 } break;
834 }
835 }
836
837 assert(!(alwaysinline && noinline) &&
838 "alwaysinline and noinline are mutually exclusive");
839
840 SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);
841 SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);
842 SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);
843 SaveAndRestore save_noconvergent(InNoConvergentAttributedStmt, noconvergent);
844 SaveAndRestore save_amdgpuav(AMDGPUAvailableVisibleMode, amdgpuAVMode);
845 SaveAndRestore save_musttail(MustTailCall, musttail);
846 SaveAndRestore save_flattenOrBranch(HLSLControlFlowAttr, flattenOrBranch);
847 CGAtomicOptionsRAII AORAII(CGM, AA);
848 EmitStmt(S: S.getSubStmt(), Attrs: S.getAttrs());
849}
850
851void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
852 // If this code is reachable then emit a stop point (if generating
853 // debug info). We have to do this ourselves because we are on the
854 // "simple" statement path.
855 if (HaveInsertPoint())
856 EmitStopPoint(S: &S);
857
858 ApplyAtomGroup Grp(getDebugInfo());
859 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: S.getLabel()));
860}
861
862
863void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
864 ApplyAtomGroup Grp(getDebugInfo());
865 if (const LabelDecl *Target = S.getConstantTarget()) {
866 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: Target));
867 return;
868 }
869
870 // Ensure that we have an i8* for our PHI node.
871 llvm::Value *V = Builder.CreateBitCast(V: EmitScalarExpr(E: S.getTarget()),
872 DestTy: Int8PtrTy, Name: "addr");
873 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
874
875 // Get the basic block for the indirect goto.
876 llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
877
878 // The first instruction in the block has to be the PHI for the switch dest,
879 // add an entry for this branch.
880 cast<llvm::PHINode>(Val: IndGotoBB->begin())->addIncoming(V, BB: CurBB);
881
882 EmitBranch(Target: IndGotoBB);
883 if (CurBB && CurBB->hasTerminator())
884 addInstToCurrentSourceAtom(KeyInstruction: CurBB->getTerminator(), Backup: nullptr);
885}
886
887void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
888 const Stmt *Else = S.getElse();
889
890 // The else branch of a consteval if statement is always the only branch that
891 // can be runtime evaluated.
892 if (S.isConsteval()) {
893 const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;
894 if (Executed) {
895 RunCleanupsScope ExecutedScope(*this);
896 EmitStmt(S: Executed);
897 }
898 return;
899 }
900
901 // C99 6.8.4.1: The first substatement is executed if the expression compares
902 // unequal to 0. The condition must be a scalar type.
903 LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
904 ApplyDebugLocation DL(*this, S.getCond());
905
906 if (S.getInit()) {
907 EmitStmt(S: S.getInit());
908
909 // The init statement may have cleared the insertion point (e.g. it ended in
910 // a 'noreturn' call); the condition emitted below needs a valid one.
911 EnsureInsertPoint();
912 }
913
914 if (S.getConditionVariable())
915 EmitDecl(D: *S.getConditionVariable());
916
917 // If the condition constant folds and can be elided, try to avoid emitting
918 // the condition and the dead arm of the if/else.
919 bool CondConstant;
920 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: CondConstant,
921 AllowLabels: S.isConstexpr())) {
922 // Figure out which block (then or else) is executed.
923 const Stmt *Executed = S.getThen();
924 const Stmt *Skipped = Else;
925 if (!CondConstant) // Condition false?
926 std::swap(a&: Executed, b&: Skipped);
927
928 // If the skipped block has no labels in it, just emit the executed block.
929 // This avoids emitting dead code and simplifies the CFG substantially.
930 if (S.isConstexpr() || !ContainsLabel(S: Skipped)) {
931 incrementProfileCounter(ExecSkip: CondConstant ? UseExecPath : UseSkipPath, S: &S,
932 /*UseBoth=*/true);
933 if (Executed) {
934 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
935 RunCleanupsScope ExecutedScope(*this);
936 EmitStmt(S: Executed);
937 }
938 PGO->markStmtMaybeUsed(S: Skipped);
939 return;
940 }
941 }
942
943 auto HasSkip = hasSkipCounter(S: &S);
944
945 // Otherwise, the condition did not fold, or we couldn't elide it. Just emit
946 // the conditional branch.
947 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.then");
948 llvm::BasicBlock *ContBlock = createBasicBlock(name: "if.end");
949 llvm::BasicBlock *ElseBlock =
950 (Else || HasSkip ? createBasicBlock(name: "if.else") : ContBlock);
951 // Prefer the PGO based weights over the likelihood attribute.
952 // When the build isn't optimized the metadata isn't used, so don't generate
953 // it.
954 // Also, differentiate between disabled PGO and a never executed branch with
955 // PGO. Assuming PGO is in use:
956 // - we want to ignore the [[likely]] attribute if the branch is never
957 // executed,
958 // - assuming the profile is poor, preserving the attribute may still be
959 // beneficial.
960 // As an approximation, preserve the attribute only if both the branch and the
961 // parent context were not executed.
962 Stmt::Likelihood LH = Stmt::LH_None;
963 uint64_t ThenCount = getProfileCount(S: S.getThen());
964 if (!ThenCount && !getCurrentProfileCount() &&
965 CGM.getCodeGenOpts().OptimizationLevel)
966 LH = Stmt::getLikelihood(Then: S.getThen(), Else);
967
968 // When measuring MC/DC, always fully evaluate the condition up front using
969 // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
970 // executing the body of the if.then or if.else. This is useful for when
971 // there is a 'return' within the body, but this is particularly beneficial
972 // when one if-stmt is nested within another if-stmt so that all of the MC/DC
973 // updates are kept linear and consistent.
974 if (!CGM.getCodeGenOpts().MCDCCoverage) {
975 EmitBranchOnBoolExpr(Cond: S.getCond(), TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount: ThenCount, LH,
976 /*ConditionalOp=*/nullptr,
977 /*ConditionalDecl=*/S.getConditionVariable());
978 } else {
979 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
980 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
981 Builder.CreateCondBr(Cond: BoolCondVal, True: ThenBlock, False: ElseBlock);
982 }
983
984 // Emit the 'then' code.
985 EmitBlock(BB: ThenBlock);
986 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
987 {
988 RunCleanupsScope ThenScope(*this);
989 EmitStmt(S: S.getThen());
990 }
991 EmitBranch(Target: ContBlock);
992
993 // Emit the 'else' code if present.
994 if (Else) {
995 {
996 // There is no need to emit line number for an unconditional branch.
997 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
998 EmitBlock(BB: ElseBlock);
999 }
1000 // Add a counter to else block unless it has CounterExpr.
1001 if (HasSkip)
1002 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1003 {
1004 RunCleanupsScope ElseScope(*this);
1005 EmitStmt(S: Else);
1006 }
1007 {
1008 // There is no need to emit line number for an unconditional branch.
1009 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1010 EmitBranch(Target: ContBlock);
1011 }
1012 } else if (HasSkip) {
1013 EmitBlock(BB: ElseBlock);
1014 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1015 EmitBranch(Target: ContBlock);
1016 }
1017
1018 // Emit the continuation block for code after the if.
1019 EmitBlock(BB: ContBlock, IsFinished: true);
1020}
1021
1022bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression,
1023 bool HasEmptyBody) {
1024 if (CGM.getCodeGenOpts().getFiniteLoops() ==
1025 CodeGenOptions::FiniteLoopsKind::Never)
1026 return false;
1027
1028 // Now apply rules for plain C (see 6.8.5.6 in C11).
1029 // Loops with constant conditions do not have to make progress in any C
1030 // version.
1031 // As an extension, we consisider loops whose constant expression
1032 // can be constant-folded.
1033 Expr::EvalResult Result;
1034 bool CondIsConstInt =
1035 !ControllingExpression ||
1036 (ControllingExpression->EvaluateAsInt(Result, Ctx: getContext()) &&
1037 Result.Val.isInt());
1038
1039 bool CondIsTrue = CondIsConstInt && (!ControllingExpression ||
1040 Result.Val.getInt().getBoolValue());
1041
1042 // Loops with non-constant conditions must make progress in C11 and later.
1043 if (getLangOpts().C11 && !CondIsConstInt)
1044 return true;
1045
1046 // [C++26][intro.progress] (DR)
1047 // The implementation may assume that any thread will eventually do one of the
1048 // following:
1049 // [...]
1050 // - continue execution of a trivial infinite loop ([stmt.iter.general]).
1051 if (CGM.getCodeGenOpts().getFiniteLoops() ==
1052 CodeGenOptions::FiniteLoopsKind::Always ||
1053 getLangOpts().CPlusPlus11) {
1054 if (HasEmptyBody && CondIsTrue) {
1055 CurFn->removeFnAttr(Kind: llvm::Attribute::MustProgress);
1056 return false;
1057 }
1058 return true;
1059 }
1060 return false;
1061}
1062
1063// [C++26][stmt.iter.general] (DR)
1064// A trivially empty iteration statement is an iteration statement matching one
1065// of the following forms:
1066// - while ( expression ) ;
1067// - while ( expression ) { }
1068// - do ; while ( expression ) ;
1069// - do { } while ( expression ) ;
1070// - for ( init-statement expression(opt); ) ;
1071// - for ( init-statement expression(opt); ) { }
1072template <typename LoopStmt> static bool hasEmptyLoopBody(const LoopStmt &S) {
1073 if constexpr (std::is_same_v<LoopStmt, ForStmt>) {
1074 if (S.getInc())
1075 return false;
1076 }
1077 const Stmt *Body = S.getBody();
1078 if (!Body || isa<NullStmt>(Val: Body))
1079 return true;
1080 if (const CompoundStmt *Compound = dyn_cast<CompoundStmt>(Val: Body))
1081 return Compound->body_empty();
1082 return false;
1083}
1084
1085void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
1086 ArrayRef<const Attr *> WhileAttrs) {
1087 // Emit the header for the loop, which will also become
1088 // the continue target.
1089 JumpDest LoopHeader = getJumpDestInCurrentScope(Name: "while.cond");
1090 EmitBlock(BB: LoopHeader.getBlock());
1091
1092 if (CGM.shouldEmitConvergenceTokens())
1093 ConvergenceTokenStack.push_back(
1094 Elt: emitConvergenceLoopToken(BB: LoopHeader.getBlock()));
1095
1096 // Create an exit block for when the condition fails, which will
1097 // also become the break target.
1098 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "while.end");
1099
1100 // Store the blocks to use for break and continue.
1101 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopHeader));
1102
1103 // C++ [stmt.while]p2:
1104 // When the condition of a while statement is a declaration, the
1105 // scope of the variable that is declared extends from its point
1106 // of declaration (3.3.2) to the end of the while statement.
1107 // [...]
1108 // The object created in a condition is destroyed and created
1109 // with each iteration of the loop.
1110 RunCleanupsScope ConditionScope(*this);
1111
1112 if (S.getConditionVariable())
1113 EmitDecl(D: *S.getConditionVariable());
1114
1115 // Evaluate the conditional in the while header. C99 6.8.5.1: The
1116 // evaluation of the controlling expression takes place before each
1117 // execution of the loop body.
1118 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1119
1120 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1121
1122 // while(1) is common, avoid extra exit blocks. Be sure
1123 // to correctly handle break/continue though.
1124 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1125 bool EmitBoolCondBranch = !C || !C->isOne();
1126 const SourceRange &R = S.getSourceRange();
1127 LoopStack.push(Header: LoopHeader.getBlock(), Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
1128 Attrs: WhileAttrs, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1129 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1130 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1131
1132 // As long as the condition is true, go to the loop body.
1133 llvm::BasicBlock *LoopBody = createBasicBlock(name: "while.body");
1134 if (EmitBoolCondBranch) {
1135 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1136 if (hasSkipCounter(S: &S) || ConditionScope.requiresCleanups())
1137 ExitBlock = createBasicBlock(name: "while.exit");
1138 llvm::MDNode *Weights =
1139 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1140 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1141 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1142 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1143 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock, BranchWeights: Weights);
1144 // Key Instructions: Emit the condition and branch as separate source
1145 // location atoms otherwise we may omit a step onto the loop condition in
1146 // favour of the `while` keyword.
1147 // FIXME: We could have the branch as the backup location for the condition,
1148 // which would probably be a better experience. Explore this later.
1149 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1150 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1151 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1152
1153 if (ExitBlock != LoopExit.getBlock()) {
1154 EmitBlock(BB: ExitBlock);
1155 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1156 EmitBranchThroughCleanup(Dest: LoopExit);
1157 }
1158 } else if (const Attr *A = Stmt::getLikelihoodAttr(S: S.getBody())) {
1159 CGM.getDiags().Report(Loc: A->getLocation(),
1160 DiagID: diag::warn_attribute_has_no_effect_on_infinite_loop)
1161 << A << A->getRange();
1162 CGM.getDiags().Report(
1163 Loc: S.getWhileLoc(),
1164 DiagID: diag::note_attribute_has_no_effect_on_infinite_loop_here)
1165 << SourceRange(S.getWhileLoc(), S.getRParenLoc());
1166 }
1167
1168 // Emit the loop body. We have to emit this in a cleanup scope
1169 // because it might be a singleton DeclStmt.
1170 {
1171 RunCleanupsScope BodyScope(*this);
1172 EmitBlock(BB: LoopBody);
1173 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1174 EmitStmt(S: S.getBody());
1175 }
1176
1177 BreakContinueStack.pop_back();
1178
1179 // Immediately force cleanup.
1180 ConditionScope.ForceCleanup();
1181
1182 EmitStopPoint(S: &S);
1183 // Branch to the loop header again.
1184 EmitBranch(Target: LoopHeader.getBlock());
1185
1186 LoopStack.pop();
1187
1188 // Emit the exit block.
1189 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1190
1191 // The LoopHeader typically is just a branch if we skipped emitting
1192 // a branch, try to erase it.
1193 if (!EmitBoolCondBranch) {
1194 SimplifyForwardingBlocks(BB: LoopHeader.getBlock());
1195 PGO->markStmtAsUsed(Skipped: true, S: &S);
1196 }
1197
1198 if (CGM.shouldEmitConvergenceTokens())
1199 ConvergenceTokenStack.pop_back();
1200}
1201
1202void CodeGenFunction::EmitDoStmt(const DoStmt &S,
1203 ArrayRef<const Attr *> DoAttrs) {
1204 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "do.end");
1205 JumpDest LoopCond = getJumpDestInCurrentScope(Name: "do.cond");
1206
1207 uint64_t ParentCount = getCurrentProfileCount();
1208
1209 // Store the blocks to use for break and continue.
1210 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopCond));
1211
1212 // Emit the body of the loop.
1213 llvm::BasicBlock *LoopBody = createBasicBlock(name: "do.body");
1214
1215 EmitBlockWithFallThrough(BB: LoopBody, S: &S);
1216
1217 if (CGM.shouldEmitConvergenceTokens())
1218 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: LoopBody));
1219
1220 {
1221 RunCleanupsScope BodyScope(*this);
1222 EmitStmt(S: S.getBody());
1223 }
1224
1225 EmitBlock(BB: LoopCond.getBlock());
1226
1227 // C99 6.8.5.2: "The evaluation of the controlling expression takes place
1228 // after each execution of the loop body."
1229
1230 // Evaluate the conditional in the while header.
1231 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1232 // compares unequal to 0. The condition must be a scalar type.
1233 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1234
1235 BreakContinueStack.pop_back();
1236
1237 // "do {} while (0)" is common in macros, avoid extra blocks. Be sure
1238 // to correctly handle break/continue though.
1239 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1240 bool EmitBoolCondBranch = !C || !C->isZero();
1241
1242 const SourceRange &R = S.getSourceRange();
1243 LoopStack.push(Header: LoopBody, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: DoAttrs,
1244 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1245 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1246 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1247
1248 auto *LoopFalse = (hasSkipCounter(S: &S) ? createBasicBlock(name: "do.loopfalse")
1249 : LoopExit.getBlock());
1250
1251 // As long as the condition is true, iterate the loop.
1252 if (EmitBoolCondBranch) {
1253 uint64_t BackedgeCount = getProfileCount(S: S.getBody()) - ParentCount;
1254 auto *I = Builder.CreateCondBr(
1255 Cond: BoolCondVal, True: LoopBody, False: LoopFalse,
1256 BranchWeights: createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: BackedgeCount));
1257
1258 // Key Instructions: Emit the condition and branch as separate source
1259 // location atoms otherwise we may omit a step onto the loop condition in
1260 // favour of the closing brace.
1261 // FIXME: We could have the branch as the backup location for the condition,
1262 // which would probably be a better experience (no jumping to the brace).
1263 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1264 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1265 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1266 }
1267
1268 LoopStack.pop();
1269
1270 if (LoopFalse != LoopExit.getBlock()) {
1271 EmitBlock(BB: LoopFalse);
1272 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S, /*UseBoth=*/true);
1273 }
1274
1275 // Emit the exit block.
1276 EmitBlock(BB: LoopExit.getBlock());
1277
1278 // The DoCond block typically is just a branch if we skipped
1279 // emitting a branch, try to erase it.
1280 if (!EmitBoolCondBranch)
1281 SimplifyForwardingBlocks(BB: LoopCond.getBlock());
1282
1283 if (CGM.shouldEmitConvergenceTokens())
1284 ConvergenceTokenStack.pop_back();
1285}
1286
1287void CodeGenFunction::EmitForStmt(const ForStmt &S,
1288 ArrayRef<const Attr *> ForAttrs) {
1289 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1290
1291 std::optional<LexicalScope> ForScope;
1292 if (getLangOpts().C99 || getLangOpts().CPlusPlus)
1293 ForScope.emplace(args&: *this, args: S.getSourceRange());
1294
1295 // Evaluate the first part before the loop.
1296 if (S.getInit())
1297 EmitStmt(S: S.getInit());
1298
1299 // Start the loop with a block that tests the condition.
1300 // If there's an increment, the continue scope will be overwritten
1301 // later.
1302 JumpDest CondDest = getJumpDestInCurrentScope(Name: "for.cond");
1303 llvm::BasicBlock *CondBlock = CondDest.getBlock();
1304 EmitBlock(BB: CondBlock);
1305
1306 if (CGM.shouldEmitConvergenceTokens())
1307 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1308
1309 const SourceRange &R = S.getSourceRange();
1310 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1311 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1312 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1313 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1314
1315 // Create a cleanup scope for the condition variable cleanups.
1316 LexicalScope ConditionScope(*this, S.getSourceRange());
1317
1318 // If the for loop doesn't have an increment we can just use the condition as
1319 // the continue block. Otherwise, if there is no condition variable, we can
1320 // form the continue block now. If there is a condition variable, we can't
1321 // form the continue block until after we've emitted the condition, because
1322 // the condition is in scope in the increment, but Sema's jump diagnostics
1323 // ensure that there are no continues from the condition variable that jump
1324 // to the loop increment.
1325 JumpDest Continue;
1326 if (!S.getInc())
1327 Continue = CondDest;
1328 else if (!S.getConditionVariable())
1329 Continue = getJumpDestInCurrentScope(Name: "for.inc");
1330 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1331
1332 if (S.getCond()) {
1333 // If the for statement has a condition scope, emit the local variable
1334 // declaration.
1335 if (S.getConditionVariable()) {
1336 EmitDecl(D: *S.getConditionVariable());
1337
1338 // We have entered the condition variable's scope, so we're now able to
1339 // jump to the continue block.
1340 Continue = S.getInc() ? getJumpDestInCurrentScope(Name: "for.inc") : CondDest;
1341 BreakContinueStack.back().ContinueBlock = Continue;
1342 }
1343
1344 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1345 // If there are any cleanups between here and the loop-exit scope,
1346 // create a block to stage a loop exit along.
1347 if (hasSkipCounter(S: &S) || (ForScope && ForScope->requiresCleanups()))
1348 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1349
1350 // As long as the condition is true, iterate the loop.
1351 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1352
1353 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1354 // compares unequal to 0. The condition must be a scalar type.
1355 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1356
1357 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1358
1359 llvm::MDNode *Weights =
1360 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1361 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1362 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1363 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1364
1365 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1366 // Key Instructions: Emit the condition and branch as separate atoms to
1367 // match existing loop stepping behaviour. FIXME: We could have the branch
1368 // as the backup location for the condition, which would probably be a
1369 // better experience (no jumping to the brace).
1370 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1371 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1372 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1373
1374 if (ExitBlock != LoopExit.getBlock()) {
1375 EmitBlock(BB: ExitBlock);
1376 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1377 EmitBranchThroughCleanup(Dest: LoopExit);
1378 }
1379
1380 EmitBlock(BB: ForBody);
1381 } else {
1382 // Treat it as a non-zero constant. Don't even create a new block for the
1383 // body, just fall into it.
1384 PGO->markStmtAsUsed(Skipped: true, S: &S);
1385 }
1386
1387 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1388
1389 {
1390 // Create a separate cleanup scope for the body, in case it is not
1391 // a compound statement.
1392 RunCleanupsScope BodyScope(*this);
1393 EmitStmt(S: S.getBody());
1394 }
1395
1396 // The last block in the loop's body (which unconditionally branches to the
1397 // `inc` block if there is one).
1398 auto *FinalBodyBB = Builder.GetInsertBlock();
1399
1400 // If there is an increment, emit it next.
1401 if (S.getInc()) {
1402 EmitBlock(BB: Continue.getBlock());
1403 EmitStmt(S: S.getInc());
1404 }
1405
1406 BreakContinueStack.pop_back();
1407
1408 ConditionScope.ForceCleanup();
1409
1410 EmitStopPoint(S: &S);
1411 EmitBranch(Target: CondBlock);
1412
1413 if (ForScope)
1414 ForScope->ForceCleanup();
1415
1416 LoopStack.pop();
1417
1418 // Emit the fall-through block.
1419 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1420
1421 if (CGM.shouldEmitConvergenceTokens())
1422 ConvergenceTokenStack.pop_back();
1423
1424 if (FinalBodyBB) {
1425 // Key Instructions: We want the for closing brace to be step-able on to
1426 // match existing behaviour.
1427 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1428 }
1429}
1430
1431void
1432CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1433 ArrayRef<const Attr *> ForAttrs) {
1434 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1435
1436 LexicalScope ForScope(*this, S.getSourceRange());
1437
1438 // Evaluate the first pieces before the loop.
1439 if (S.getInit())
1440 EmitStmt(S: S.getInit());
1441 EmitStmt(S: S.getRangeStmt());
1442 EmitStmt(S: S.getBeginStmt());
1443 EmitStmt(S: S.getEndStmt());
1444
1445 // Start the loop with a block that tests the condition.
1446 // If there's an increment, the continue scope will be overwritten
1447 // later.
1448 llvm::BasicBlock *CondBlock = createBasicBlock(name: "for.cond");
1449 EmitBlock(BB: CondBlock);
1450
1451 if (CGM.shouldEmitConvergenceTokens())
1452 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1453
1454 const SourceRange &R = S.getSourceRange();
1455 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1456 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1457 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
1458
1459 // If there are any cleanups between here and the loop-exit scope,
1460 // create a block to stage a loop exit along.
1461 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1462 if (hasSkipCounter(S: &S) || ForScope.requiresCleanups())
1463 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1464
1465 // The loop body, consisting of the specified body and the loop variable.
1466 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1467
1468 // The body is executed if the expression, contextually converted
1469 // to bool, is true.
1470 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1471 llvm::MDNode *Weights =
1472 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1473 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1474 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1475 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1476 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1477 // Key Instructions: Emit the condition and branch as separate atoms to
1478 // match existing loop stepping behaviour. FIXME: We could have the branch as
1479 // the backup location for the condition, which would probably be a better
1480 // experience.
1481 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1482 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1483 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1484
1485 if (ExitBlock != LoopExit.getBlock()) {
1486 EmitBlock(BB: ExitBlock);
1487 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1488 EmitBranchThroughCleanup(Dest: LoopExit);
1489 }
1490
1491 EmitBlock(BB: ForBody);
1492 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1493
1494 // Create a block for the increment. In case of a 'continue', we jump there.
1495 JumpDest Continue = getJumpDestInCurrentScope(Name: "for.inc");
1496
1497 // Store the blocks to use for break and continue.
1498 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1499
1500 {
1501 // Create a separate cleanup scope for the loop variable and body.
1502 LexicalScope BodyScope(*this, S.getSourceRange());
1503 EmitStmt(S: S.getLoopVarStmt());
1504 EmitStmt(S: S.getBody());
1505 }
1506 // The last block in the loop's body (which unconditionally branches to the
1507 // `inc` block if there is one).
1508 auto *FinalBodyBB = Builder.GetInsertBlock();
1509
1510 EmitStopPoint(S: &S);
1511 // If there is an increment, emit it next.
1512 EmitBlock(BB: Continue.getBlock());
1513 EmitStmt(S: S.getInc());
1514
1515 BreakContinueStack.pop_back();
1516
1517 EmitBranch(Target: CondBlock);
1518
1519 ForScope.ForceCleanup();
1520
1521 LoopStack.pop();
1522
1523 // Emit the fall-through block.
1524 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1525
1526 if (CGM.shouldEmitConvergenceTokens())
1527 ConvergenceTokenStack.pop_back();
1528
1529 if (FinalBodyBB) {
1530 // We want the for closing brace to be step-able on to match existing
1531 // behaviour.
1532 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1533 }
1534}
1535
1536void CodeGenFunction::EmitCXXExpansionStmtInstantiation(
1537 const CXXExpansionStmtInstantiation &S) {
1538 LexicalScope Scope(*this, S.getSourceRange());
1539
1540 for (const Stmt *DS : S.getPreambleStmts())
1541 EmitStmt(S: DS);
1542
1543 if (S.getInstantiations().empty())
1544 return;
1545
1546 JumpDest ExpandExit = getJumpDestInCurrentScope(Name: "expand.end");
1547 JumpDest ContinueDest;
1548 for (auto [N, Inst] : enumerate(First: S.getInstantiations())) {
1549 if (N == S.getInstantiations().size() - 1)
1550 ContinueDest = ExpandExit;
1551 else
1552 ContinueDest = getJumpDestInCurrentScope(Name: "expand.next");
1553
1554 LexicalScope ExpansionScope(*this, Inst->getSourceRange());
1555 BreakContinueStack.push_back(Elt: BreakContinue(S, ExpandExit, ContinueDest));
1556 EmitStmt(S: Inst);
1557 BreakContinueStack.pop_back();
1558 EmitBlock(BB: ContinueDest.getBlock(), IsFinished: true);
1559 }
1560}
1561
1562void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1563 if (RV.isScalar()) {
1564 Builder.CreateStore(Val: RV.getScalarVal(), Addr: ReturnValue);
1565 } else if (RV.isAggregate()) {
1566 LValue Dest = MakeAddrLValue(Addr: ReturnValue, T: Ty);
1567 LValue Src = MakeAddrLValue(Addr: RV.getAggregateAddress(), T: Ty);
1568 EmitAggregateCopy(Dest, Src, EltTy: Ty, MayOverlap: getOverlapForReturnValue());
1569 } else {
1570 EmitStoreOfComplex(V: RV.getComplexVal(), dest: MakeAddrLValue(Addr: ReturnValue, T: Ty),
1571 /*init*/ isInit: true);
1572 }
1573 EmitBranchThroughCleanup(Dest: ReturnBlock);
1574}
1575
1576namespace {
1577// RAII struct used to save and restore a return statment's result expression.
1578struct SaveRetExprRAII {
1579 SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1580 : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1581 CGF.RetExpr = RetExpr;
1582 }
1583 ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1584 const Expr *OldRetExpr;
1585 CodeGenFunction &CGF;
1586};
1587} // namespace
1588
1589/// Determine if the given call uses the swiftasync calling convention.
1590static bool isSwiftAsyncCallee(const CallExpr *CE) {
1591 auto calleeQualType = CE->getCallee()->getType();
1592 const FunctionType *calleeType = nullptr;
1593 if (calleeQualType->isFunctionPointerType() ||
1594 calleeQualType->isFunctionReferenceType() ||
1595 calleeQualType->isBlockPointerType() ||
1596 calleeQualType->isMemberFunctionPointerType()) {
1597 calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1598 } else if (auto *ty = dyn_cast<FunctionType>(Val&: calleeQualType)) {
1599 calleeType = ty;
1600 } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(Val: CE)) {
1601 if (auto methodDecl = CMCE->getMethodDecl()) {
1602 // getMethodDecl() doesn't handle member pointers at the moment.
1603 calleeType = methodDecl->getType()->castAs<FunctionType>();
1604 } else {
1605 return false;
1606 }
1607 } else {
1608 return false;
1609 }
1610 return calleeType->getCallConv() == CallingConv::CC_SwiftAsync;
1611}
1612
1613/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1614/// if the function returns void, or may be missing one if the function returns
1615/// non-void. Fun stuff :).
1616void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1617 ApplyAtomGroup Grp(getDebugInfo());
1618 if (requiresReturnValueCheck()) {
1619 llvm::Constant *SLoc = EmitCheckSourceLocation(Loc: S.getBeginLoc());
1620 auto *SLocPtr =
1621 new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1622 llvm::GlobalVariable::PrivateLinkage, SLoc);
1623 SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1624 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: SLocPtr);
1625 assert(ReturnLocation.isValid() && "No valid return location");
1626 Builder.CreateStore(Val: SLocPtr, Addr: ReturnLocation);
1627 }
1628
1629 // Returning from an outlined SEH helper is UB, and we already warn on it.
1630 if (IsOutlinedSEHHelper) {
1631 Builder.CreateUnreachable();
1632 Builder.ClearInsertionPoint();
1633 }
1634
1635 // Emit the result value, even if unused, to evaluate the side effects.
1636 const Expr *RV = S.getRetValue();
1637
1638 // Record the result expression of the return statement. The recorded
1639 // expression is used to determine whether a block capture's lifetime should
1640 // end at the end of the full expression as opposed to the end of the scope
1641 // enclosing the block expression.
1642 //
1643 // This permits a small, easily-implemented exception to our over-conservative
1644 // rules about not jumping to statements following block literals with
1645 // non-trivial cleanups.
1646 SaveRetExprRAII SaveRetExpr(RV, *this);
1647
1648 RunCleanupsScope cleanupScope(*this);
1649 if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(Val: RV))
1650 RV = EWC->getSubExpr();
1651
1652 // If we're in a swiftasynccall function, and the return expression is a
1653 // call to a swiftasynccall function, mark the call as the musttail call.
1654 std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail;
1655 if (RV && CurFnInfo &&
1656 CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) {
1657 if (auto CE = dyn_cast<CallExpr>(Val: RV)) {
1658 if (isSwiftAsyncCallee(CE)) {
1659 SaveMustTail.emplace(args&: MustTailCall, args&: CE);
1660 }
1661 }
1662 }
1663
1664 // FIXME: Clean this up by using an LValue for ReturnTemp,
1665 // EmitStoreThroughLValue, and EmitAnyExpr.
1666 // Check if the NRVO candidate was not globalized in OpenMP mode.
1667 if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1668 S.getNRVOCandidate()->isNRVOVariable() &&
1669 (!getLangOpts().OpenMP ||
1670 !CGM.getOpenMPRuntime()
1671 .getAddressOfLocalVariable(CGF&: *this, VD: S.getNRVOCandidate())
1672 .isValid())) {
1673 // Apply the named return value optimization for this return statement,
1674 // which means doing nothing: the appropriate result has already been
1675 // constructed into the NRVO variable.
1676
1677 // If there is an NRVO flag for this variable, set it to 1 into indicate
1678 // that the cleanup code should not destroy the variable.
1679 if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1680 Builder.CreateFlagStore(Value: Builder.getTrue(), Addr: NRVOFlag);
1681 } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1682 // Make sure not to return anything, but evaluate the expression
1683 // for side effects.
1684 if (RV) {
1685 EmitAnyExpr(E: RV);
1686 }
1687 } else if (!RV) {
1688 // Do nothing (return value is left uninitialized)
1689 } else if (FnRetTy->isReferenceType()) {
1690 // If this function returns a reference, take the address of the expression
1691 // rather than the value.
1692 RValue Result = EmitReferenceBindingToExpr(E: RV);
1693 auto *I = Builder.CreateStore(Val: Result.getScalarVal(), Addr: ReturnValue);
1694 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1695 } else {
1696 switch (getEvaluationKind(T: RV->getType())) {
1697 case TEK_Scalar: {
1698 llvm::Value *Ret = EmitScalarExpr(E: RV);
1699 if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1700 EmitStoreOfScalar(value: Ret, lvalue: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1701 /*isInit*/ true);
1702 } else {
1703 auto *I = Builder.CreateStore(Val: Ret, Addr: ReturnValue);
1704 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1705 }
1706 break;
1707 }
1708 case TEK_Complex:
1709 EmitComplexExprIntoLValue(E: RV, dest: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1710 /*isInit*/ true);
1711 break;
1712 case TEK_Aggregate:
1713 EmitAggExpr(E: RV, AS: AggValueSlot::forAddr(
1714 addr: ReturnValue, quals: Qualifiers(),
1715 isDestructed: AggValueSlot::IsDestructed,
1716 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1717 isAliased: AggValueSlot::IsNotAliased,
1718 mayOverlap: getOverlapForReturnValue()));
1719 break;
1720 }
1721 }
1722
1723 ++NumReturnExprs;
1724 if (!RV || RV->isEvaluatable(Ctx: getContext()))
1725 ++NumSimpleReturnExprs;
1726
1727 cleanupScope.ForceCleanup();
1728 EmitBranchThroughCleanup(Dest: ReturnBlock);
1729}
1730
1731void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1732 // As long as debug info is modeled with instructions, we have to ensure we
1733 // have a place to insert here and write the stop point here.
1734 if (HaveInsertPoint())
1735 EmitStopPoint(S: &S);
1736
1737 for (const auto *I : S.decls())
1738 EmitDecl(D: *I, /*EvaluateConditionDecl=*/true);
1739}
1740
1741auto CodeGenFunction::GetDestForLoopControlStmt(const LoopControlStmt &S)
1742 -> const BreakContinue * {
1743 if (!S.hasLabelTarget())
1744 return &BreakContinueStack.back();
1745
1746 const Stmt *LoopOrSwitch = S.getNamedLoopOrSwitch();
1747 assert(LoopOrSwitch && "break/continue target not set?");
1748 for (const BreakContinue &BC : llvm::reverse(C&: BreakContinueStack))
1749 if (BC.LoopOrSwitch == LoopOrSwitch)
1750 return &BC;
1751
1752 llvm_unreachable("break/continue target not found");
1753}
1754
1755void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1756 assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1757
1758 // If this code is reachable then emit a stop point (if generating
1759 // debug info). We have to do this ourselves because we are on the
1760 // "simple" statement path.
1761 if (HaveInsertPoint())
1762 EmitStopPoint(S: &S);
1763
1764 ApplyAtomGroup Grp(getDebugInfo());
1765 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->BreakBlock);
1766}
1767
1768void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1769 assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1770
1771 // If this code is reachable then emit a stop point (if generating
1772 // debug info). We have to do this ourselves because we are on the
1773 // "simple" statement path.
1774 if (HaveInsertPoint())
1775 EmitStopPoint(S: &S);
1776
1777 ApplyAtomGroup Grp(getDebugInfo());
1778 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->ContinueBlock);
1779}
1780
1781/// EmitCaseStmtRange - If case statement range is not too big then
1782/// add multiple cases to switch instruction, one for each value within
1783/// the range. If range is too big then emit "if" condition check.
1784void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1785 ArrayRef<const Attr *> Attrs) {
1786 assert(S.getRHS() && "Expected RHS value in CaseStmt");
1787
1788 llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(Ctx: getContext());
1789 llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(Ctx: getContext());
1790
1791 // Emit the code for this case. We do this first to make sure it is
1792 // properly chained from our predecessor before generating the
1793 // switch machinery to enter this block.
1794 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1795 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1796 EmitStmt(S: S.getSubStmt());
1797
1798 // If range is empty, do nothing.
1799 if (LHS.isSigned() ? RHS.slt(RHS: LHS) : RHS.ult(RHS: LHS))
1800 return;
1801
1802 Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1803 llvm::APInt Range = RHS - LHS;
1804 // FIXME: parameters such as this should not be hardcoded.
1805 if (Range.getBitWidth() < 7 ||
1806 Range.ult(RHS: llvm::APInt(Range.getBitWidth(), 64))) {
1807 // Range is small enough to add multiple switch instruction cases.
1808 uint64_t Total = getProfileCount(S: &S);
1809 unsigned NCases = Range.getZExtValue() + 1;
1810 // We only have one region counter for the entire set of cases here, so we
1811 // need to divide the weights evenly between the generated cases, ensuring
1812 // that the total weight is preserved. E.g., a weight of 5 over three cases
1813 // will be distributed as weights of 2, 2, and 1.
1814 uint64_t Weight = Total / NCases, Rem = Total % NCases;
1815 for (unsigned I = 0; I != NCases; ++I) {
1816 if (SwitchWeights)
1817 SwitchWeights->push_back(Elt: Weight + (Rem ? 1 : 0));
1818 else if (SwitchLikelihood)
1819 SwitchLikelihood->push_back(Elt: LH);
1820
1821 if (Rem)
1822 Rem--;
1823 SwitchInsn->addCase(OnVal: Builder.getInt(AI: LHS), Dest: CaseDest);
1824 ++LHS;
1825 }
1826 return;
1827 }
1828
1829 // The range is too big. Emit "if" condition into a new block,
1830 // making sure to save and restore the current insertion point.
1831 llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1832
1833 // Push this test onto the chain of range checks (which terminates
1834 // in the default basic block). The switch's default will be changed
1835 // to the top of this chain after switch emission is complete.
1836 llvm::BasicBlock *FalseDest = CaseRangeBlock;
1837 CaseRangeBlock = createBasicBlock(name: "sw.caserange");
1838
1839 CurFn->insert(Position: CurFn->end(), BB: CaseRangeBlock);
1840 Builder.SetInsertPoint(CaseRangeBlock);
1841
1842 // Emit range check.
1843 llvm::Value *Diff =
1844 Builder.CreateSub(LHS: SwitchInsn->getCondition(), RHS: Builder.getInt(AI: LHS));
1845 llvm::Value *Cond =
1846 Builder.CreateICmpULE(LHS: Diff, RHS: Builder.getInt(AI: Range), Name: "inbounds");
1847
1848 llvm::MDNode *Weights = nullptr;
1849 if (SwitchWeights) {
1850 uint64_t ThisCount = getProfileCount(S: &S);
1851 uint64_t DefaultCount = (*SwitchWeights)[0];
1852 Weights = createProfileWeights(TrueCount: ThisCount, FalseCount: DefaultCount);
1853
1854 // Since we're chaining the switch default through each large case range, we
1855 // need to update the weight for the default, ie, the first case, to include
1856 // this case.
1857 (*SwitchWeights)[0] += ThisCount;
1858 } else if (SwitchLikelihood)
1859 Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1860
1861 Builder.CreateCondBr(Cond, True: CaseDest, False: FalseDest, BranchWeights: Weights);
1862
1863 // Restore the appropriate insertion point.
1864 if (RestoreBB)
1865 Builder.SetInsertPoint(RestoreBB);
1866 else
1867 Builder.ClearInsertionPoint();
1868}
1869
1870void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1871 ArrayRef<const Attr *> Attrs) {
1872 // If there is no enclosing switch instance that we're aware of, then this
1873 // case statement and its block can be elided. This situation only happens
1874 // when we've constant-folded the switch, are emitting the constant case,
1875 // and part of the constant case includes another case statement. For
1876 // instance: switch (4) { case 4: do { case 5: } while (1); }
1877 if (!SwitchInsn) {
1878 EmitStmt(S: S.getSubStmt());
1879 return;
1880 }
1881
1882 // Handle case ranges.
1883 if (S.getRHS()) {
1884 EmitCaseStmtRange(S, Attrs);
1885 return;
1886 }
1887
1888 llvm::ConstantInt *CaseVal =
1889 Builder.getInt(AI: S.getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1890
1891 // Emit debuginfo for the case value if it is an enum value.
1892 const ConstantExpr *CE;
1893 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: S.getLHS()))
1894 CE = dyn_cast<ConstantExpr>(Val: ICE->getSubExpr());
1895 else
1896 CE = dyn_cast<ConstantExpr>(Val: S.getLHS());
1897 if (CE) {
1898 if (auto DE = dyn_cast<DeclRefExpr>(Val: CE->getSubExpr()))
1899 if (CGDebugInfo *Dbg = getDebugInfo())
1900 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
1901 Dbg->EmitGlobalVariable(VD: DE->getDecl(),
1902 Init: APValue(llvm::APSInt(CaseVal->getValue())));
1903 }
1904
1905 if (SwitchLikelihood)
1906 SwitchLikelihood->push_back(Elt: Stmt::getLikelihood(Attrs));
1907
1908 // If the body of the case is just a 'break', try to not emit an empty block.
1909 // If we're profiling or we're not optimizing, leave the block in for better
1910 // debug and coverage analysis.
1911 if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1912 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1913 isa<BreakStmt>(Val: S.getSubStmt())) {
1914 JumpDest Block = BreakContinueStack.back().BreakBlock;
1915
1916 // Only do this optimization if there are no cleanups that need emitting.
1917 if (isObviouslyBranchWithoutCleanups(Dest: Block)) {
1918 if (SwitchWeights)
1919 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1920 SwitchInsn->addCase(OnVal: CaseVal, Dest: Block.getBlock());
1921
1922 // If there was a fallthrough into this case, make sure to redirect it to
1923 // the end of the switch as well.
1924 if (Builder.GetInsertBlock()) {
1925 Builder.CreateBr(Dest: Block.getBlock());
1926 Builder.ClearInsertionPoint();
1927 }
1928 return;
1929 }
1930 }
1931
1932 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1933 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1934 if (SwitchWeights)
1935 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1936 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1937
1938 // Recursively emitting the statement is acceptable, but is not wonderful for
1939 // code where we have many case statements nested together, i.e.:
1940 // case 1:
1941 // case 2:
1942 // case 3: etc.
1943 // Handling this recursively will create a new block for each case statement
1944 // that falls through to the next case which is IR intensive. It also causes
1945 // deep recursion which can run into stack depth limitations. Handle
1946 // sequential non-range case statements specially.
1947 //
1948 // TODO When the next case has a likelihood attribute the code returns to the
1949 // recursive algorithm. Maybe improve this case if it becomes common practice
1950 // to use a lot of attributes.
1951 const CaseStmt *CurCase = &S;
1952 const CaseStmt *NextCase = dyn_cast<CaseStmt>(Val: S.getSubStmt());
1953
1954 // Otherwise, iteratively add consecutive cases to this switch stmt.
1955 while (NextCase && NextCase->getRHS() == nullptr) {
1956 CurCase = NextCase;
1957 llvm::ConstantInt *CaseVal =
1958 Builder.getInt(AI: CurCase->getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1959
1960 if (SwitchWeights)
1961 SwitchWeights->push_back(Elt: getProfileCount(S: NextCase));
1962 if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1963 CaseDest = createBasicBlock(name: "sw.bb");
1964 EmitBlockWithFallThrough(BB: CaseDest, S: CurCase);
1965 }
1966 // Since this loop is only executed when the CaseStmt has no attributes
1967 // use a hard-coded value.
1968 if (SwitchLikelihood)
1969 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
1970
1971 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1972 NextCase = dyn_cast<CaseStmt>(Val: CurCase->getSubStmt());
1973 }
1974
1975 // Generate a stop point for debug info if the case statement is
1976 // followed by a default statement. A fallthrough case before a
1977 // default case gets its own branch target.
1978 if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1979 EmitStopPoint(S: CurCase);
1980
1981 // Normal default recursion for non-cases.
1982 EmitStmt(S: CurCase->getSubStmt());
1983}
1984
1985void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1986 ArrayRef<const Attr *> Attrs) {
1987 // If there is no enclosing switch instance that we're aware of, then this
1988 // default statement can be elided. This situation only happens when we've
1989 // constant-folded the switch.
1990 if (!SwitchInsn) {
1991 EmitStmt(S: S.getSubStmt());
1992 return;
1993 }
1994
1995 llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1996 assert(DefaultBlock->empty() &&
1997 "EmitDefaultStmt: Default block already defined?");
1998
1999 if (SwitchLikelihood)
2000 SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
2001
2002 EmitBlockWithFallThrough(BB: DefaultBlock, S: &S);
2003
2004 EmitStmt(S: S.getSubStmt());
2005}
2006
2007namespace {
2008struct EmitDeferredStatement final : EHScopeStack::Cleanup {
2009 const DeferStmt &Stmt;
2010 EmitDeferredStatement(const DeferStmt *Stmt) : Stmt(*Stmt) {}
2011
2012 void Emit(CodeGenFunction &CGF, Flags) override {
2013 // Take care that any cleanups pushed by the body of a '_Defer' statement
2014 // don't clobber the current cleanup slot value.
2015 //
2016 // Assume we have a scope that pushes a cleanup; when that scope is exited,
2017 // we need to run that cleanup; this is accomplished by emitting the cleanup
2018 // into a separate block and then branching to that block at scope exit.
2019 //
2020 // Where this gets complicated is if we exit the scope in multiple different
2021 // ways; e.g. in a 'for' loop, we may exit the scope of its body by falling
2022 // off the end (in which case we need to run the cleanup and then branch to
2023 // the increment), or by 'break'ing out of the loop (in which case we need
2024 // to run the cleanup and then branch to the loop exit block); in both cases
2025 // we first branch to the cleanup block to run the cleanup, but the block we
2026 // need to jump to *after* running the cleanup is different.
2027 //
2028 // This is accomplished using a local integer variable called the 'cleanup
2029 // slot': before branching to the cleanup block, we store a value into that
2030 // slot. Then, in the cleanup block, after running the cleanup, we load the
2031 // value of that variable and 'switch' on it to branch to the appropriate
2032 // continuation block.
2033 //
2034 // The problem that arises once '_Defer' statements are involved is that the
2035 // body of a '_Defer' is an arbitrary statement which itself can create more
2036 // cleanups. This means we may end up overwriting the cleanup slot before we
2037 // ever have a chance to 'switch' on it, which means that once we *do* get
2038 // to the 'switch', we end up in whatever block the cleanup code happened to
2039 // pick as the default 'switch' exit label!
2040 //
2041 // That is, what is normally supposed to happen is something like:
2042 //
2043 // 1. Store 'X' to cleanup slot.
2044 // 2. Branch to cleanup block.
2045 // 3. Execute cleanup.
2046 // 4. Read value from cleanup slot.
2047 // 5. Branch to the block associated with 'X'.
2048 //
2049 // But if we encounter a _Defer' statement that contains a cleanup, then
2050 // what might instead happen is:
2051 //
2052 // 1. Store 'X' to cleanup slot.
2053 // 2. Branch to cleanup block.
2054 // 3. Execute cleanup; this ends up pushing another cleanup, so:
2055 // 3a. Store 'Y' to cleanup slot.
2056 // 3b. Run steps 2–5 recursively.
2057 // 4. Read value from cleanup slot, which is now 'Y' instead of 'X'.
2058 // 5. Branch to the block associated with 'Y'... which doesn't even
2059 // exist because the value 'Y' is only meaningful for the inner
2060 // cleanup. The result is we just branch 'somewhere random'.
2061 //
2062 // The rest of the cleanup code simply isn't prepared to handle this case
2063 // because most other cleanups can't push more cleanups, and thus, emitting
2064 // other cleanups generally cannot clobber the cleanup slot.
2065 //
2066 // To prevent this from happening, save the current cleanup slot value and
2067 // restore it after emitting the '_Defer' statement.
2068 llvm::Value *SavedCleanupDest = nullptr;
2069 if (CGF.NormalCleanupDest.isValid())
2070 SavedCleanupDest =
2071 CGF.Builder.CreateLoad(Addr: CGF.NormalCleanupDest, Name: "cleanup.dest.saved");
2072
2073 CGF.EmitStmt(S: Stmt.getBody());
2074
2075 if (SavedCleanupDest && CGF.HaveInsertPoint())
2076 CGF.Builder.CreateStore(Val: SavedCleanupDest, Addr: CGF.NormalCleanupDest);
2077
2078 // Cleanups must end with an insert point.
2079 CGF.EnsureInsertPoint();
2080 }
2081};
2082} // namespace
2083
2084void CodeGenFunction::EmitDeferStmt(const DeferStmt &S) {
2085 EHStack.pushCleanup<EmitDeferredStatement>(Kind: NormalAndEHCleanup, A: &S);
2086}
2087
2088/// CollectStatementsForCase - Given the body of a 'switch' statement and a
2089/// constant value that is being switched on, see if we can dead code eliminate
2090/// the body of the switch to a simple series of statements to emit. Basically,
2091/// on a switch (5) we want to find these statements:
2092/// case 5:
2093/// printf(...); <--
2094/// ++i; <--
2095/// break;
2096///
2097/// and add them to the ResultStmts vector. If it is unsafe to do this
2098/// transformation (for example, one of the elided statements contains a label
2099/// that might be jumped to), return CSFC_Failure. If we handled it and 'S'
2100/// should include statements after it (e.g. the printf() line is a substmt of
2101/// the case) then return CSFC_FallThrough. If we handled it and found a break
2102/// statement, then return CSFC_Success.
2103///
2104/// If Case is non-null, then we are looking for the specified case, checking
2105/// that nothing we jump over contains labels. If Case is null, then we found
2106/// the case and are looking for the break.
2107///
2108/// If the recursive walk actually finds our Case, then we set FoundCase to
2109/// true.
2110///
2111enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
2112static CSFC_Result CollectStatementsForCase(const Stmt *S,
2113 const SwitchCase *Case,
2114 bool &FoundCase,
2115 SmallVectorImpl<const Stmt*> &ResultStmts) {
2116 // If this is a null statement, just succeed.
2117 if (!S)
2118 return Case ? CSFC_Success : CSFC_FallThrough;
2119
2120 // If this is the switchcase (case 4: or default) that we're looking for, then
2121 // we're in business. Just add the substatement.
2122 if (const SwitchCase *SC = dyn_cast<SwitchCase>(Val: S)) {
2123 if (S == Case) {
2124 FoundCase = true;
2125 return CollectStatementsForCase(S: SC->getSubStmt(), Case: nullptr, FoundCase,
2126 ResultStmts);
2127 }
2128
2129 // Otherwise, this is some other case or default statement, just ignore it.
2130 return CollectStatementsForCase(S: SC->getSubStmt(), Case, FoundCase,
2131 ResultStmts);
2132 }
2133
2134 // If we are in the live part of the code and we found our break statement,
2135 // return a success!
2136 if (!Case && isa<BreakStmt>(Val: S))
2137 return CSFC_Success;
2138
2139 // If this is a switch statement, then it might contain the SwitchCase, the
2140 // break, or neither.
2141 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: S)) {
2142 // Handle this as two cases: we might be looking for the SwitchCase (if so
2143 // the skipped statements must be skippable) or we might already have it.
2144 CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
2145 bool StartedInLiveCode = FoundCase;
2146 unsigned StartSize = ResultStmts.size();
2147
2148 // If we've not found the case yet, scan through looking for it.
2149 if (Case) {
2150 // Keep track of whether we see a skipped declaration. The code could be
2151 // using the declaration even if it is skipped, so we can't optimize out
2152 // the decl if the kept statements might refer to it.
2153 bool HadSkippedDecl = false;
2154
2155 // If we're looking for the case, just see if we can skip each of the
2156 // substatements.
2157 for (; Case && I != E; ++I) {
2158 HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(S: *I);
2159
2160 switch (CollectStatementsForCase(S: *I, Case, FoundCase, ResultStmts)) {
2161 case CSFC_Failure: return CSFC_Failure;
2162 case CSFC_Success:
2163 // A successful result means that either 1) that the statement doesn't
2164 // have the case and is skippable, or 2) does contain the case value
2165 // and also contains the break to exit the switch. In the later case,
2166 // we just verify the rest of the statements are elidable.
2167 if (FoundCase) {
2168 // If we found the case and skipped declarations, we can't do the
2169 // optimization.
2170 if (HadSkippedDecl)
2171 return CSFC_Failure;
2172
2173 for (++I; I != E; ++I)
2174 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2175 return CSFC_Failure;
2176 return CSFC_Success;
2177 }
2178 break;
2179 case CSFC_FallThrough:
2180 // If we have a fallthrough condition, then we must have found the
2181 // case started to include statements. Consider the rest of the
2182 // statements in the compound statement as candidates for inclusion.
2183 assert(FoundCase && "Didn't find case but returned fallthrough?");
2184 // We recursively found Case, so we're not looking for it anymore.
2185 Case = nullptr;
2186
2187 // If we found the case and skipped declarations, we can't do the
2188 // optimization.
2189 if (HadSkippedDecl)
2190 return CSFC_Failure;
2191 break;
2192 }
2193 }
2194
2195 if (!FoundCase)
2196 return CSFC_Success;
2197
2198 assert(!HadSkippedDecl && "fallthrough after skipping decl");
2199 }
2200
2201 // If we have statements in our range, then we know that the statements are
2202 // live and need to be added to the set of statements we're tracking.
2203 bool AnyDecls = false;
2204 for (; I != E; ++I) {
2205 AnyDecls |= CodeGenFunction::mightAddDeclToScope(S: *I);
2206
2207 switch (CollectStatementsForCase(S: *I, Case: nullptr, FoundCase, ResultStmts)) {
2208 case CSFC_Failure: return CSFC_Failure;
2209 case CSFC_FallThrough:
2210 // A fallthrough result means that the statement was simple and just
2211 // included in ResultStmt, keep adding them afterwards.
2212 break;
2213 case CSFC_Success:
2214 // A successful result means that we found the break statement and
2215 // stopped statement inclusion. We just ensure that any leftover stmts
2216 // are skippable and return success ourselves.
2217 for (++I; I != E; ++I)
2218 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2219 return CSFC_Failure;
2220 return CSFC_Success;
2221 }
2222 }
2223
2224 // If we're about to fall out of a scope without hitting a 'break;', we
2225 // can't perform the optimization if there were any decls in that scope
2226 // (we'd lose their end-of-lifetime).
2227 if (AnyDecls) {
2228 // If the entire compound statement was live, there's one more thing we
2229 // can try before giving up: emit the whole thing as a single statement.
2230 // We can do that unless the statement contains a 'break;'.
2231 // FIXME: Such a break must be at the end of a construct within this one.
2232 // We could emit this by just ignoring the BreakStmts entirely.
2233 if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
2234 ResultStmts.resize(N: StartSize);
2235 ResultStmts.push_back(Elt: S);
2236 } else {
2237 return CSFC_Failure;
2238 }
2239 }
2240
2241 return CSFC_FallThrough;
2242 }
2243
2244 // Okay, this is some other statement that we don't handle explicitly, like a
2245 // for statement or increment etc. If we are skipping over this statement,
2246 // just verify it doesn't have labels, which would make it invalid to elide.
2247 if (Case) {
2248 if (CodeGenFunction::ContainsLabel(S, IgnoreCaseStmts: true))
2249 return CSFC_Failure;
2250 return CSFC_Success;
2251 }
2252
2253 // Otherwise, we want to include this statement. Everything is cool with that
2254 // so long as it doesn't contain a break out of the switch we're in.
2255 if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
2256
2257 // Otherwise, everything is great. Include the statement and tell the caller
2258 // that we fall through and include the next statement as well.
2259 ResultStmts.push_back(Elt: S);
2260 return CSFC_FallThrough;
2261}
2262
2263/// FindCaseStatementsForValue - Find the case statement being jumped to and
2264/// then invoke CollectStatementsForCase to find the list of statements to emit
2265/// for a switch on constant. See the comment above CollectStatementsForCase
2266/// for more details.
2267static bool FindCaseStatementsForValue(const SwitchStmt &S,
2268 const llvm::APSInt &ConstantCondValue,
2269 SmallVectorImpl<const Stmt*> &ResultStmts,
2270 ASTContext &C,
2271 const SwitchCase *&ResultCase) {
2272 // First step, find the switch case that is being branched to. We can do this
2273 // efficiently by scanning the SwitchCase list.
2274 const SwitchCase *Case = S.getSwitchCaseList();
2275 const DefaultStmt *DefaultCase = nullptr;
2276
2277 for (; Case; Case = Case->getNextSwitchCase()) {
2278 // It's either a default or case. Just remember the default statement in
2279 // case we're not jumping to any numbered cases.
2280 if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Val: Case)) {
2281 DefaultCase = DS;
2282 continue;
2283 }
2284
2285 // Check to see if this case is the one we're looking for.
2286 const CaseStmt *CS = cast<CaseStmt>(Val: Case);
2287 // Don't handle case ranges yet.
2288 if (CS->getRHS()) return false;
2289
2290 // If we found our case, remember it as 'case'.
2291 if (CS->getLHS()->EvaluateKnownConstInt(Ctx: C) == ConstantCondValue)
2292 break;
2293 }
2294
2295 // If we didn't find a matching case, we use a default if it exists, or we
2296 // elide the whole switch body!
2297 if (!Case) {
2298 // It is safe to elide the body of the switch if it doesn't contain labels
2299 // etc. If it is safe, return successfully with an empty ResultStmts list.
2300 if (!DefaultCase)
2301 return !CodeGenFunction::ContainsLabel(S: &S);
2302 Case = DefaultCase;
2303 }
2304
2305 // Ok, we know which case is being jumped to, try to collect all the
2306 // statements that follow it. This can fail for a variety of reasons. Also,
2307 // check to see that the recursive walk actually found our case statement.
2308 // Insane cases like this can fail to find it in the recursive walk since we
2309 // don't handle every stmt kind:
2310 // switch (4) {
2311 // while (1) {
2312 // case 4: ...
2313 bool FoundCase = false;
2314 ResultCase = Case;
2315 return CollectStatementsForCase(S: S.getBody(), Case, FoundCase,
2316 ResultStmts) != CSFC_Failure &&
2317 FoundCase;
2318}
2319
2320static std::optional<SmallVector<uint64_t, 16>>
2321getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
2322 // Are there enough branches to weight them?
2323 if (Likelihoods.size() <= 1)
2324 return std::nullopt;
2325
2326 uint64_t NumUnlikely = 0;
2327 uint64_t NumNone = 0;
2328 uint64_t NumLikely = 0;
2329 for (const auto LH : Likelihoods) {
2330 switch (LH) {
2331 case Stmt::LH_Unlikely:
2332 ++NumUnlikely;
2333 break;
2334 case Stmt::LH_None:
2335 ++NumNone;
2336 break;
2337 case Stmt::LH_Likely:
2338 ++NumLikely;
2339 break;
2340 }
2341 }
2342
2343 // Is there a likelihood attribute used?
2344 if (NumUnlikely == 0 && NumLikely == 0)
2345 return std::nullopt;
2346
2347 // When multiple cases share the same code they can be combined during
2348 // optimization. In that case the weights of the branch will be the sum of
2349 // the individual weights. Make sure the combined sum of all neutral cases
2350 // doesn't exceed the value of a single likely attribute.
2351 // The additions both avoid divisions by 0 and make sure the weights of None
2352 // don't exceed the weight of Likely.
2353 const uint64_t Likely = INT32_MAX / (NumLikely + 2);
2354 const uint64_t None = Likely / (NumNone + 1);
2355 const uint64_t Unlikely = 0;
2356
2357 SmallVector<uint64_t, 16> Result;
2358 Result.reserve(N: Likelihoods.size());
2359 for (const auto LH : Likelihoods) {
2360 switch (LH) {
2361 case Stmt::LH_Unlikely:
2362 Result.push_back(Elt: Unlikely);
2363 break;
2364 case Stmt::LH_None:
2365 Result.push_back(Elt: None);
2366 break;
2367 case Stmt::LH_Likely:
2368 Result.push_back(Elt: Likely);
2369 break;
2370 }
2371 }
2372
2373 return Result;
2374}
2375
2376void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
2377 // Handle nested switch statements.
2378 llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
2379 SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
2380 SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
2381 llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
2382
2383 // See if we can constant fold the condition of the switch and therefore only
2384 // emit the live case statement (if any) of the switch.
2385 llvm::APSInt ConstantCondValue;
2386 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: ConstantCondValue)) {
2387 SmallVector<const Stmt*, 4> CaseStmts;
2388 const SwitchCase *Case = nullptr;
2389 if (FindCaseStatementsForValue(S, ConstantCondValue, ResultStmts&: CaseStmts,
2390 C&: getContext(), ResultCase&: Case)) {
2391 if (Case)
2392 incrementProfileCounter(S: Case);
2393 RunCleanupsScope ExecutedScope(*this);
2394
2395 if (S.getInit())
2396 EmitStmt(S: S.getInit());
2397
2398 // Emit the condition variable if needed inside the entire cleanup scope
2399 // used by this special case for constant folded switches.
2400 if (S.getConditionVariable())
2401 EmitDecl(D: *S.getConditionVariable(), /*EvaluateConditionDecl=*/true);
2402
2403 // At this point, we are no longer "within" a switch instance, so
2404 // we can temporarily enforce this to ensure that any embedded case
2405 // statements are not emitted.
2406 SwitchInsn = nullptr;
2407
2408 // Okay, we can dead code eliminate everything except this case. Emit the
2409 // specified series of statements and we're good.
2410 for (const Stmt *CaseStmt : CaseStmts)
2411 EmitStmt(S: CaseStmt);
2412 incrementProfileCounter(S: &S);
2413 PGO->markStmtMaybeUsed(S: S.getBody());
2414
2415 // Now we want to restore the saved switch instance so that nested
2416 // switches continue to function properly
2417 SwitchInsn = SavedSwitchInsn;
2418
2419 return;
2420 }
2421 }
2422
2423 JumpDest SwitchExit = getJumpDestInCurrentScope(Name: "sw.epilog");
2424
2425 RunCleanupsScope ConditionScope(*this);
2426
2427 if (S.getInit()) {
2428 EmitStmt(S: S.getInit());
2429
2430 // The init statement may have cleared the insertion point (e.g. it ended in
2431 // a 'noreturn' call); the condition emitted below needs a valid one.
2432 EnsureInsertPoint();
2433 }
2434
2435 if (S.getConditionVariable())
2436 EmitDecl(D: *S.getConditionVariable());
2437 llvm::Value *CondV = EmitScalarExpr(E: S.getCond());
2438 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
2439
2440 // Create basic block to hold stuff that comes after switch
2441 // statement. We also need to create a default block now so that
2442 // explicit case ranges tests can have a place to jump to on
2443 // failure.
2444 llvm::BasicBlock *DefaultBlock = createBasicBlock(name: "sw.default");
2445 SwitchInsn = Builder.CreateSwitch(V: CondV, Dest: DefaultBlock);
2446 addInstToNewSourceAtom(KeyInstruction: SwitchInsn, Backup: CondV);
2447
2448 if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {
2449 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2450 llvm::ConstantInt *BranchHintConstant =
2451 HLSLControlFlowAttr ==
2452 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2453 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2454 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2455 llvm::Metadata *Vals[] = {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2456 MDHelper.createConstant(C: BranchHintConstant)};
2457 SwitchInsn->setMetadata(Kind: "hlsl.controlflow.hint",
2458 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2459 }
2460
2461 if (PGO->haveRegionCounts()) {
2462 // Walk the SwitchCase list to find how many there are.
2463 uint64_t DefaultCount = 0;
2464 unsigned NumCases = 0;
2465 for (const SwitchCase *Case = S.getSwitchCaseList();
2466 Case;
2467 Case = Case->getNextSwitchCase()) {
2468 if (isa<DefaultStmt>(Val: Case))
2469 DefaultCount = getProfileCount(S: Case);
2470 NumCases += 1;
2471 }
2472 SwitchWeights = new SmallVector<uint64_t, 16>();
2473 SwitchWeights->reserve(N: NumCases);
2474 // The default needs to be first. We store the edge count, so we already
2475 // know the right weight.
2476 SwitchWeights->push_back(Elt: DefaultCount);
2477 } else if (CGM.getCodeGenOpts().OptimizationLevel) {
2478 SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
2479 // Initialize the default case.
2480 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
2481 }
2482
2483 CaseRangeBlock = DefaultBlock;
2484
2485 // Clear the insertion point to indicate we are in unreachable code.
2486 Builder.ClearInsertionPoint();
2487
2488 // All break statements jump to NextBlock. If BreakContinueStack is non-empty
2489 // then reuse last ContinueBlock.
2490 JumpDest OuterContinue;
2491 if (!BreakContinueStack.empty())
2492 OuterContinue = BreakContinueStack.back().ContinueBlock;
2493
2494 BreakContinueStack.push_back(Elt: BreakContinue(S, SwitchExit, OuterContinue));
2495
2496 // Emit switch body.
2497 EmitStmt(S: S.getBody());
2498
2499 BreakContinueStack.pop_back();
2500
2501 // Update the default block in case explicit case range tests have
2502 // been chained on top.
2503 SwitchInsn->setDefaultDest(CaseRangeBlock);
2504
2505 // If a default was never emitted:
2506 if (!DefaultBlock->getParent()) {
2507 // If we have cleanups, emit the default block so that there's a
2508 // place to jump through the cleanups from.
2509 if (ConditionScope.requiresCleanups()) {
2510 EmitBlock(BB: DefaultBlock);
2511
2512 // Otherwise, just forward the default block to the switch end.
2513 } else {
2514 DefaultBlock->replaceAllUsesWith(V: SwitchExit.getBlock());
2515 delete DefaultBlock;
2516 }
2517 }
2518
2519 ConditionScope.ForceCleanup();
2520
2521 // Close the last case (or DefaultBlock).
2522 EmitBranch(Target: SwitchExit.getBlock());
2523
2524 // Insert a False Counter if SwitchStmt doesn't have DefaultStmt.
2525 if (hasSkipCounter(S: S.getCond())) {
2526 auto *ImplicitDefaultBlock = createBasicBlock(name: "sw.false");
2527 EmitBlock(BB: ImplicitDefaultBlock);
2528 incrementProfileCounter(ExecSkip: UseSkipPath, S: S.getCond());
2529 Builder.CreateBr(Dest: SwitchInsn->getDefaultDest());
2530 SwitchInsn->setDefaultDest(ImplicitDefaultBlock);
2531 }
2532
2533 // Emit continuation.
2534 EmitBlock(BB: SwitchExit.getBlock(), IsFinished: true);
2535 incrementProfileCounter(S: &S);
2536
2537 // If the switch has a condition wrapped by __builtin_unpredictable,
2538 // create metadata that specifies that the switch is unpredictable.
2539 // Don't bother if not optimizing because that metadata would not be used.
2540 auto *Call = dyn_cast<CallExpr>(Val: S.getCond());
2541 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2542 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2543 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2544 llvm::MDBuilder MDHelper(getLLVMContext());
2545 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_unpredictable,
2546 Node: MDHelper.createUnpredictable());
2547 }
2548 }
2549
2550 if (SwitchWeights) {
2551 assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2552 "switch weights do not match switch cases");
2553 // If there's only one jump destination there's no sense weighting it.
2554 if (SwitchWeights->size() > 1)
2555 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2556 Node: createProfileWeights(Weights: *SwitchWeights));
2557 delete SwitchWeights;
2558 } else if (SwitchLikelihood) {
2559 assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2560 "switch likelihoods do not match switch cases");
2561 std::optional<SmallVector<uint64_t, 16>> LHW =
2562 getLikelihoodWeights(Likelihoods: *SwitchLikelihood);
2563 if (LHW) {
2564 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2565 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2566 Node: createProfileWeights(Weights: *LHW));
2567 }
2568 delete SwitchLikelihood;
2569 }
2570 SwitchInsn = SavedSwitchInsn;
2571 SwitchWeights = SavedSwitchWeights;
2572 SwitchLikelihood = SavedSwitchLikelihood;
2573 CaseRangeBlock = SavedCRBlock;
2574}
2575
2576std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2577 const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2578 QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2579 if (Info.allowsRegister() || !Info.allowsMemory()) {
2580 if (CodeGenFunction::hasScalarEvaluationKind(T: InputType))
2581 return {EmitLoadOfLValue(V: InputValue, Loc).getScalarVal(), nullptr};
2582
2583 llvm::Type *Ty = ConvertType(T: InputType);
2584 uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2585 if ((Size <= 64 && llvm::isPowerOf2_64(Value: Size)) ||
2586 getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty)) {
2587 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2588
2589 return {Builder.CreateLoad(Addr: InputValue.getAddress().withElementType(ElemTy: Ty)),
2590 nullptr};
2591 }
2592 }
2593
2594 Address Addr = InputValue.getAddress();
2595 ConstraintStr += '*';
2596 return {InputValue.getPointer(CGF&: *this), Addr.getElementType()};
2597}
2598std::pair<llvm::Value *, llvm::Type *>
2599CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2600 const Expr *InputExpr,
2601 std::string &ConstraintStr) {
2602 // If this can't be a register or memory, i.e., has to be a constant
2603 // (immediate or symbolic), try to emit it as such.
2604 if (!Info.allowsRegister() && !Info.allowsMemory()) {
2605 if (Info.requiresImmediateConstant()) {
2606 Expr::EvalResult EVResult;
2607 InputExpr->EvaluateAsRValue(Result&: EVResult, Ctx: getContext(), InConstantContext: true);
2608
2609 llvm::APSInt IntResult;
2610 if (EVResult.Val.toIntegralConstant(Result&: IntResult, SrcTy: InputExpr->getType(),
2611 Ctx: getContext()))
2612 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: IntResult), nullptr};
2613 }
2614
2615 Expr::EvalResult Result;
2616 if (InputExpr->EvaluateAsInt(Result, Ctx: getContext()))
2617 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: Result.Val.getInt()),
2618 nullptr};
2619 }
2620
2621 if (Info.allowsRegister() || !Info.allowsMemory())
2622 if (CodeGenFunction::hasScalarEvaluationKind(T: InputExpr->getType()))
2623 return {EmitScalarExpr(E: InputExpr), nullptr};
2624 if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2625 return {EmitScalarExpr(E: InputExpr), nullptr};
2626 InputExpr = InputExpr->IgnoreParenNoopCasts(Ctx: getContext());
2627 LValue Dest = EmitLValue(E: InputExpr);
2628 return EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr,
2629 Loc: InputExpr->getExprLoc());
2630}
2631
2632/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2633/// asm call instruction. The !srcloc MDNode contains a list of constant
2634/// integers which are the source locations of the start of each line in the
2635/// asm.
2636static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2637 CodeGenFunction &CGF) {
2638 SmallVector<llvm::Metadata *, 8> Locs;
2639 // Add the location of the first line to the MDNode.
2640 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
2641 Ty: CGF.Int64Ty, V: Str->getBeginLoc().getRawEncoding())));
2642 StringRef StrVal = Str->getString();
2643 if (!StrVal.empty()) {
2644 const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2645 const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2646 unsigned StartToken = 0;
2647 unsigned ByteOffset = 0;
2648
2649 // Add the location of the start of each subsequent line of the asm to the
2650 // MDNode.
2651 for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2652 if (StrVal[i] != '\n') continue;
2653 SourceLocation LineLoc = Str->getLocationOfByte(
2654 ByteNo: i + 1, SM, Features: LangOpts, Target: CGF.getTarget(), StartToken: &StartToken, StartTokenByteOffset: &ByteOffset);
2655 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(
2656 C: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: LineLoc.getRawEncoding())));
2657 }
2658 }
2659
2660 return llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: Locs);
2661}
2662
2663namespace clang {
2664
2665/// This structure holds the information gathered about the constraints for an
2666/// inline assembly statement. It helps in separating the constraint processing
2667/// from the code generation.
2668class AsmConstraintsInfo {
2669 CodeGenFunction &CGF;
2670 CodeGenModule &CGM; // Per-module state.
2671 const AsmStmt &S;
2672 CGBuilderTy &Builder;
2673
2674 // The final asm string.
2675 std::string AsmString;
2676
2677 // The output and input constraints.
2678 SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2679 SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2680
2681 // Constraint strings.
2682 std::string Constraints;
2683 std::string InOutConstraints;
2684
2685 // Keep track of out constraints for tied input operand.
2686 std::vector<std::string> OutputConstraints;
2687
2688 // Keep track of argument types.
2689 std::vector<llvm::Value *> Args;
2690 std::vector<llvm::Type *> ArgTypes;
2691 std::vector<llvm::Type *> ArgElemTypes;
2692
2693 // Keep track of result register constraints.
2694 std::vector<LValue> ResultRegDests;
2695 std::vector<QualType> ResultRegQualTys;
2696 std::vector<llvm::Type *> ResultRegTypes;
2697 std::vector<llvm::Type *> ResultTruncRegTypes;
2698
2699 llvm::BitVector ResultTypeRequiresCast;
2700
2701 // Keep track of in/out constraints.
2702 std::vector<llvm::Value *> InOutArgs;
2703 std::vector<llvm::Type *> InOutArgTypes;
2704 std::vector<llvm::Type *> InOutArgElemTypes;
2705
2706 // Destination blocks for 'asm gotos'.
2707 llvm::BasicBlock *DefaultDest = nullptr;
2708 SmallVector<llvm::BasicBlock *, 3> IndirectDests;
2709
2710 std::vector<std::optional<std::pair<unsigned, unsigned>>> ResultBounds;
2711
2712 // An inline asm can be marked readonly if it meets the following
2713 // conditions:
2714 //
2715 // - it doesn't have any sideeffects
2716 // - it doesn't clobber memory
2717 // - it doesn't return a value by-reference
2718 //
2719 // It can be marked readnone if it doesn't have any input memory
2720 // constraints in addition to meeting the conditions listed above.
2721 bool ReadOnly = true;
2722 bool ReadNone = true;
2723
2724 bool GetOutputAndInputConstraints();
2725 void HandleOutputConstraints();
2726 void HandleMSStyleAsmBlob();
2727 void HandleInputConstraints();
2728 bool HandleLabels();
2729 bool HandleClobbers();
2730 void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2731 bool HasUnwindClobber, bool NoMerge, bool NoConvergent,
2732 std::vector<llvm::Value *> &RegResults);
2733 void EmitAsmStores(const llvm::ArrayRef<llvm::Value *> RegResults);
2734
2735 void EmitHipStdParUnsupportedAsm() {
2736 constexpr auto Name = "__ASM__hipstdpar_unsupported";
2737
2738 std::string Asm;
2739 if (auto GCCAsm = dyn_cast<GCCAsmStmt>(Val: &S))
2740 Asm = GCCAsm->getAsmString();
2741
2742 auto &Ctx = getLLVMContext();
2743 auto StrTy = llvm::ConstantDataArray::getString(Context&: Ctx, Initializer: Asm);
2744 auto FnTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx),
2745 Params: {StrTy->getType()}, isVarArg: false);
2746 auto UBF = CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2747
2748 Builder.CreateCall(Callee: UBF, Args: {StrTy});
2749 }
2750
2751 ASTContext &getContext() { return CGF.getContext(); }
2752 llvm::LLVMContext &getLLVMContext() { return CGF.getLLVMContext(); }
2753 const TargetInfo &getTarget() const { return CGF.getTarget(); }
2754 const LangOptions &getLangOpts() const { return CGF.getLangOpts(); }
2755 const TargetCodeGenInfo &getTargetHooks() const {
2756 return CGM.getTargetCodeGenInfo();
2757 }
2758
2759public:
2760 AsmConstraintsInfo(CodeGenFunction &CGF, const AsmStmt &S)
2761 : CGF(CGF), CGM(CGF.CGM), S(S), Builder(CGF.Builder),
2762 AsmString(S.generateAsmString(C: CGF.getContext())) {}
2763
2764 void EmitAsmStmt();
2765};
2766
2767} // namespace clang
2768
2769void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2770 // Pop all cleanup blocks at the end of the asm statement.
2771 CodeGenFunction::RunCleanupsScope Cleanups(*this);
2772
2773 // Get all the output and input constraints together.
2774 AsmConstraintsInfo AsmInfo(*this, S);
2775 AsmInfo.EmitAsmStmt();
2776}
2777
2778void AsmConstraintsInfo::EmitAsmStmt() {
2779 if (!GetOutputAndInputConstraints())
2780 return EmitHipStdParUnsupportedAsm();
2781
2782 // Handle output constraints.
2783 HandleOutputConstraints();
2784
2785 // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2786 // to the return value slot. Only do this when returning in registers.
2787 HandleMSStyleAsmBlob();
2788
2789 // Handle input constraints.
2790 HandleInputConstraints();
2791
2792 // Handle 'asm goto' labels.
2793 bool IsGCCAsmGoto = HandleLabels();
2794
2795 // Handle any clobbers.
2796 bool HasUnwindClobber = HandleClobbers();
2797 assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
2798 "unwind clobber can't be used with asm goto");
2799
2800 // Add machine specific clobbers
2801 std::string_view MachineClobbers = getTarget().getClobbers();
2802 if (!MachineClobbers.empty()) {
2803 if (!Constraints.empty())
2804 Constraints += ',';
2805 Constraints += MachineClobbers;
2806 }
2807
2808 llvm::Type *ResultType;
2809 if (ResultRegTypes.empty())
2810 ResultType = CGF.VoidTy;
2811 else if (ResultRegTypes.size() == 1)
2812 ResultType = ResultRegTypes[0];
2813 else
2814 ResultType = llvm::StructType::get(Context&: getLLVMContext(), Elements: ResultRegTypes);
2815
2816 llvm::FunctionType *FTy =
2817 llvm::FunctionType::get(Result: ResultType, Params: ArgTypes, isVarArg: false);
2818
2819 bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
2820
2821 llvm::InlineAsm::AsmDialect GnuAsmDialect =
2822 CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
2823 ? llvm::InlineAsm::AD_ATT
2824 : llvm::InlineAsm::AD_Intel;
2825 llvm::InlineAsm::AsmDialect AsmDialect =
2826 isa<MSAsmStmt>(Val: &S) ? llvm::InlineAsm::AD_Intel : GnuAsmDialect;
2827
2828 llvm::InlineAsm *IA = llvm::InlineAsm::get(
2829 Ty: FTy, AsmString, Constraints, hasSideEffects: HasSideEffect,
2830 /* IsAlignStack */ isAlignStack: false, asmDialect: AsmDialect, canThrow: HasUnwindClobber);
2831 std::vector<llvm::Value *> RegResults;
2832 llvm::CallBrInst *CBR;
2833 llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>
2834 CBRRegResults;
2835
2836 if (IsGCCAsmGoto) {
2837 CBR = Builder.CreateCallBr(Callee: IA, DefaultDest, IndirectDests, Args);
2838 CGF.EmitBlock(BB: DefaultDest);
2839 UpdateAsmCallInst(Result&: *CBR, HasSideEffect,
2840 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2841 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2842
2843 // Because we are emitting code top to bottom, we don't have enough
2844 // information at this point to know precisely whether we have a critical
2845 // edge. If we have outputs, split all indirect destinations.
2846 if (!RegResults.empty()) {
2847 unsigned I = 0;
2848 for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {
2849 llvm::Twine SynthName = Dest->getName() + ".split";
2850 llvm::BasicBlock *SynthBB = CGF.createBasicBlock(name: SynthName);
2851 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2852 Builder.SetInsertPoint(SynthBB);
2853
2854 if (ResultRegTypes.size() == 1) {
2855 CBRRegResults[SynthBB].push_back(Elt: CBR);
2856 } else {
2857 for (unsigned J = 0, E = ResultRegTypes.size(); J != E; ++J) {
2858 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: CBR, Idxs: J, Name: "asmresult");
2859 CBRRegResults[SynthBB].push_back(Elt: Tmp);
2860 }
2861 }
2862
2863 CGF.EmitBranch(Target: Dest);
2864 CGF.EmitBlock(BB: SynthBB);
2865 CBR->setIndirectDest(i: I++, B: SynthBB);
2866 }
2867 }
2868 } else if (HasUnwindClobber) {
2869 llvm::CallBase *Result = CGF.EmitCallOrInvoke(Callee: IA, Args, Name: "");
2870 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2871 /*HasUnwindClobber=*/true, NoMerge: CGF.InNoMergeAttributedStmt,
2872 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2873 } else {
2874 llvm::CallInst *Result =
2875 Builder.CreateCall(Callee: IA, Args, OpBundles: CGF.getBundlesForFunclet(Callee: IA));
2876 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2877 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2878 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2879 }
2880
2881 EmitAsmStores(RegResults);
2882
2883 // If this is an asm goto with outputs, repeat EmitAsmStores, but with a
2884 // different insertion point; one for each indirect destination and with
2885 // CBRRegResults rather than RegResults.
2886 if (IsGCCAsmGoto && !CBRRegResults.empty()) {
2887 for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {
2888 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2889 Builder.SetInsertPoint(TheBB: Succ, IP: --(Succ->end()));
2890 EmitAsmStores(RegResults: CBRRegResults[Succ]);
2891 }
2892 }
2893}
2894
2895/// Gather and validate the output and input constraints for the given inline
2896/// assembly statement. This ensures that the constraints are valid for the
2897/// target and prepares them for further processing.
2898bool AsmConstraintsInfo::GetOutputAndInputConstraints() {
2899 bool IsValidTargetAsm = true;
2900 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2901 for (unsigned I = 0, E = S.getNumOutputs(); I != E && IsValidTargetAsm; I++) {
2902 StringRef Name;
2903 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2904 Name = GAS->getOutputName(i: I);
2905
2906 TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i: I), Name);
2907
2908 bool IsValid = getTarget().validateOutputConstraint(Info);
2909 if (IsHipStdPar && !IsValid)
2910 IsValidTargetAsm = false;
2911 else
2912 assert(IsValid && "Failed to parse output constraint");
2913
2914 OutputConstraintInfos.push_back(Elt: Info);
2915 }
2916
2917 for (unsigned I = 0, E = S.getNumInputs(); I != E && IsValidTargetAsm; I++) {
2918 StringRef Name;
2919 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2920 Name = GAS->getInputName(i: I);
2921
2922 TargetInfo::ConstraintInfo Info(S.getInputConstraint(i: I), Name);
2923
2924 bool IsValid =
2925 getTarget().validateInputConstraint(OutputConstraints: OutputConstraintInfos, info&: Info);
2926 if (IsHipStdPar && !IsValid)
2927 IsValidTargetAsm = false;
2928 else
2929 assert(IsValid && "Failed to parse input constraint");
2930
2931 InputConstraintInfos.push_back(Elt: Info);
2932 }
2933
2934 return IsValidTargetAsm;
2935}
2936
2937/// Process the output constraints of an inline assembly statement. This method
2938/// handles the complexity of determining whether an output should be a
2939/// register or memory operand, manages tied operands, and prepares the
2940/// necessary arguments for the LLVM inline asm call.
2941void AsmConstraintsInfo::HandleOutputConstraints() {
2942 // Keep track of defined physregs.
2943 llvm::SmallSet<std::string, 8> PhysRegOutputs;
2944
2945 for (unsigned I = 0, E = S.getNumOutputs(); I != E; I++) {
2946 TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[I];
2947
2948 // Simplify the output constraint.
2949 std::string OutputConstraint(S.getOutputConstraint(i: I));
2950 OutputConstraint = getTarget().simplifyConstraint(
2951 Constraint: StringRef(OutputConstraint).substr(Start: 1), OutCons: &OutputConstraintInfos);
2952
2953 const Expr *OutExpr = S.getOutputExpr(i: I);
2954 OutExpr = OutExpr->IgnoreParenNoopCasts(Ctx: getContext());
2955
2956 std::string GCCReg;
2957 OutputConstraint = S.addVariableConstraints(
2958 Constraint: OutputConstraint, AsmExpr: *OutExpr, Target: getTarget(), EarlyClobber: Info.earlyClobber(),
2959 UnsupportedCB: [&](const Stmt *UnspStmt, StringRef Msg) {
2960 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
2961 },
2962 GCCReg: &GCCReg);
2963
2964 // Give an error on multiple outputs to same physreg.
2965 if (!GCCReg.empty() && !PhysRegOutputs.insert(V: GCCReg).second)
2966 CGM.Error(loc: S.getAsmLoc(), error: "multiple outputs to hard register: " + GCCReg);
2967
2968 OutputConstraints.push_back(x: OutputConstraint);
2969 LValue Dest = CGF.EmitLValue(E: OutExpr);
2970 if (!Constraints.empty())
2971 Constraints += ',';
2972
2973 // If this is a register output, then make the inline asm return it
2974 // by-value. If this is a memory result, return the value by-reference.
2975 QualType QTy = OutExpr->getType();
2976 const bool IsScalarOrAggregate =
2977 CodeGenFunction::hasScalarEvaluationKind(T: QTy) ||
2978 CodeGenFunction::hasAggregateEvaluationKind(T: QTy);
2979
2980 if (!Info.allowsMemory() && IsScalarOrAggregate) {
2981 Constraints += "=" + OutputConstraint;
2982 ResultRegQualTys.push_back(x: QTy);
2983 ResultRegDests.push_back(x: Dest);
2984
2985 ResultBounds.emplace_back(args: Info.getOutputOperandBounds());
2986
2987 llvm::Type *Ty = CGF.ConvertTypeForMem(T: QTy);
2988 const bool RequiresCast =
2989 Info.allowsRegister() &&
2990 (getTargetHooks().isScalarizableAsmOperand(CGF, Ty) ||
2991 Ty->isAggregateType());
2992
2993 ResultTruncRegTypes.push_back(x: Ty);
2994 ResultTypeRequiresCast.push_back(Val: RequiresCast);
2995
2996 if (RequiresCast) {
2997 if (unsigned Size = getContext().getTypeSize(T: QTy))
2998 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2999 else
3000 CGM.Error(loc: OutExpr->getExprLoc(), error: "output size should not be zero");
3001 }
3002
3003 ResultRegTypes.push_back(x: Ty);
3004
3005 // If this output is tied to an input, and if the input is larger, then
3006 // we need to set the actual result type of the inline asm node to be the
3007 // same as the input type.
3008 if (Info.hasMatchingInput()) {
3009 unsigned InputNo;
3010 for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
3011 TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
3012 if (Input.hasTiedOperand() && Input.getTiedOperand() == I)
3013 break;
3014 }
3015 assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
3016
3017 QualType InputTy = S.getInputExpr(i: InputNo)->getType();
3018 QualType OutputType = OutExpr->getType();
3019
3020 uint64_t InputSize = getContext().getTypeSize(T: InputTy);
3021 if (getContext().getTypeSize(T: OutputType) < InputSize)
3022 // Form the asm to return the value as a larger integer or fp type.
3023 ResultRegTypes.back() = CGF.ConvertType(T: InputTy);
3024 }
3025
3026 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3027 CGF, Constraint: OutputConstraint, Ty: ResultRegTypes.back()))
3028 ResultRegTypes.back() = AdjTy;
3029 else
3030 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3031 DiagID: diag::err_asm_invalid_type_in_input)
3032 << OutExpr->getType() << OutputConstraint;
3033
3034 // Update largest vector width for any vector types.
3035 if (auto *VT = dyn_cast<llvm::VectorType>(Val: ResultRegTypes.back()))
3036 CGF.LargestVectorWidth =
3037 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3038 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3039 } else {
3040 Address DestAddr = Dest.getAddress();
3041
3042 // Matrix types in memory are represented by arrays, but accessed through
3043 // vector pointers, with the alignment specified on the access operation.
3044 // For inline assembly, update pointer arguments to use vector pointers.
3045 // Otherwise there will be a mis-match if the matrix is also an
3046 // input-argument which is represented as vector.
3047 if (isa<MatrixType>(Val: OutExpr->getType().getCanonicalType()))
3048 DestAddr =
3049 DestAddr.withElementType(ElemTy: CGF.ConvertType(T: OutExpr->getType()));
3050
3051 ArgTypes.push_back(x: DestAddr.getType());
3052 ArgElemTypes.push_back(x: DestAddr.getElementType());
3053 Args.push_back(x: DestAddr.emitRawPointer(CGF));
3054
3055 Constraints += "=*" + OutputConstraint;
3056 ReadOnly = false;
3057 ReadNone = false;
3058 }
3059
3060 if (!Info.isReadWrite())
3061 continue;
3062
3063 InOutConstraints += ',';
3064
3065 const Expr *InputExpr = S.getOutputExpr(i: I);
3066 llvm::Value *Arg;
3067 llvm::Type *ArgElemType;
3068 std::tie(args&: Arg, args&: ArgElemType) =
3069 CGF.EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(),
3070 ConstraintStr&: InOutConstraints, Loc: InputExpr->getExprLoc());
3071
3072 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3073 CGF, Constraint: OutputConstraint, Ty: Arg->getType()))
3074 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3075
3076 // Update largest vector width for any vector types.
3077 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3078 CGF.LargestVectorWidth =
3079 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3080 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3081
3082 // Only tie earlyclobber physregs.
3083 if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
3084 InOutConstraints += llvm::utostr(X: I);
3085 else
3086 InOutConstraints += OutputConstraint;
3087
3088 InOutArgTypes.push_back(x: Arg->getType());
3089 InOutArgElemTypes.push_back(x: ArgElemType);
3090 InOutArgs.push_back(x: Arg);
3091 }
3092}
3093
3094/// Special handling for Microsoft-style inline assembly blocks. This ensures
3095/// that return registers (like EAX:EDX) are correctly mapped to the function's
3096/// return value slot when necessary.
3097void AsmConstraintsInfo::HandleMSStyleAsmBlob() {
3098 if (!isa<MSAsmStmt>(Val: &S))
3099 return;
3100
3101 const ABIArgInfo &RetAI = CGF.CurFnInfo->getReturnInfo();
3102 if (!RetAI.isDirect() && !RetAI.isExtend())
3103 return;
3104
3105 // Make a fake lvalue for the return value slot.
3106 LValue ReturnSlot =
3107 CGF.MakeAddrLValueWithoutTBAA(Addr: CGF.ReturnValue, T: CGF.FnRetTy);
3108 CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
3109 CGF, ReturnValue: ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
3110 ResultRegDests, AsmString, NumOutputs: S.getNumOutputs());
3111 CGF.SawAsmBlock = true;
3112}
3113
3114/// Process the input constraints of an inline assembly statement. It handles
3115/// type conversions, extensions for tied operands, and collects the necessary
3116/// LLVM values to be passed to the inline assembly call.
3117void AsmConstraintsInfo::HandleInputConstraints() {
3118 ASTContext &Ctx = getContext();
3119
3120 for (unsigned I = 0, E = S.getNumInputs(); I != E; I++) {
3121 TargetInfo::ConstraintInfo &Info = InputConstraintInfos[I];
3122 const Expr *InputExpr = S.getInputExpr(i: I);
3123
3124 if (Info.allowsMemory())
3125 ReadNone = false;
3126
3127 if (!Constraints.empty())
3128 Constraints += ',';
3129
3130 // Simplify the input constraint.
3131 std::string InputConstraint(S.getInputConstraint(i: I));
3132 InputConstraint =
3133 getTarget().simplifyConstraint(Constraint: InputConstraint, OutCons: &OutputConstraintInfos);
3134
3135 InputConstraint = S.addVariableConstraints(
3136 Constraint: InputConstraint, AsmExpr: *InputExpr->IgnoreParenNoopCasts(Ctx), Target: getTarget(),
3137 EarlyClobber: false /* No EarlyClobber */,
3138 UnsupportedCB: [&](const Stmt *UnspStmt, std::string_view Msg) {
3139 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
3140 });
3141
3142 std::string ReplaceConstraint(InputConstraint);
3143 llvm::Value *Arg;
3144 llvm::Type *ArgElemType;
3145 std::tie(args&: Arg, args&: ArgElemType) = CGF.EmitAsmInput(Info, InputExpr, ConstraintStr&: Constraints);
3146
3147 // If this input argument is tied to a larger output result, extend the
3148 // input to be the same size as the output. The LLVM backend wants to see
3149 // the input and output of a matching constraint be the same size. Note
3150 // that GCC does not define what the top bits are here. We use zext because
3151 // that is usually cheaper, but LLVM IR should really get an anyext someday.
3152 if (Info.hasTiedOperand()) {
3153 unsigned Output = Info.getTiedOperand();
3154 QualType OutputType = S.getOutputExpr(i: Output)->getType();
3155 QualType InputTy = InputExpr->getType();
3156
3157 if (Ctx.getTypeSize(T: OutputType) > Ctx.getTypeSize(T: InputTy)) {
3158 // Use ptrtoint as appropriate so that we can do our extension.
3159 if (isa<llvm::PointerType>(Val: Arg->getType()))
3160 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: CGF.IntPtrTy);
3161
3162 llvm::Type *OutputTy = CGF.ConvertType(T: OutputType);
3163 if (isa<llvm::IntegerType>(Val: OutputTy))
3164 Arg = Builder.CreateZExt(V: Arg, DestTy: OutputTy);
3165 else if (isa<llvm::PointerType>(Val: OutputTy))
3166 Arg = Builder.CreateZExt(V: Arg, DestTy: CGF.IntPtrTy);
3167 else if (OutputTy->isFloatingPointTy())
3168 Arg = Builder.CreateFPExt(V: Arg, DestTy: OutputTy);
3169 }
3170
3171 // Deal with the tied operands' constraint code in adjustInlineAsmType.
3172 ReplaceConstraint = OutputConstraints[Output];
3173 }
3174
3175 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3176 CGF, Constraint: ReplaceConstraint, Ty: Arg->getType()))
3177 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3178 else
3179 CGM.getDiags().Report(Loc: S.getAsmLoc(), DiagID: diag::err_asm_invalid_type_in_input)
3180 << InputExpr->getType() << InputConstraint;
3181
3182 // Update largest vector width for any vector types.
3183 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3184 CGF.LargestVectorWidth =
3185 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3186 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3187
3188 ArgTypes.push_back(x: Arg->getType());
3189 ArgElemTypes.push_back(x: ArgElemType);
3190 Args.push_back(x: Arg);
3191
3192 Constraints += InputConstraint;
3193 }
3194
3195 // Append the "input" part of in/out constraints.
3196 for (unsigned I = 0, E = InOutArgs.size(); I != E; I++) {
3197 ArgTypes.push_back(x: InOutArgTypes[I]);
3198 ArgElemTypes.push_back(x: InOutArgElemTypes[I]);
3199 Args.push_back(x: InOutArgs[I]);
3200 }
3201
3202 Constraints += InOutConstraints;
3203}
3204
3205/// Handle labels in an 'asm goto' statement. This method resolves the symbolic
3206/// labels to LLVM basic blocks and updates the constraint string to reflect
3207/// the indirect jump targets.
3208bool AsmConstraintsInfo::HandleLabels() {
3209 if (const auto *GS = dyn_cast<GCCAsmStmt>(Val: &S); GS && GS->isAsmGoto()) {
3210 for (const auto *E : GS->labels()) {
3211 CodeGenFunction::JumpDest Dest = CGF.getJumpDestForLabel(D: E->getLabel());
3212 IndirectDests.push_back(Elt: Dest.getBlock());
3213
3214 if (!Constraints.empty())
3215 Constraints += ',';
3216
3217 Constraints += "!i";
3218 }
3219
3220 DefaultDest = CGF.createBasicBlock(name: "asm.fallthrough");
3221 return true;
3222 }
3223
3224 return false;
3225}
3226
3227/// Process clobber constraints for an inline assembly statement. This
3228/// identifies which registers or system state (like "memory" or "cc") are
3229/// modified by the assembly block, which is crucial for correct optimization
3230/// and side-effect modeling.
3231bool AsmConstraintsInfo::HandleClobbers() {
3232 bool HasUnwindClobber = false;
3233 for (unsigned I = 0, E = S.getNumClobbers(); I != E; I++) {
3234 std::string Clobber = S.getClobber(i: I);
3235
3236 if (Clobber == "unwind") {
3237 HasUnwindClobber = true;
3238 continue;
3239 }
3240
3241 if (Clobber == "memory") {
3242 ReadOnly = false;
3243 ReadNone = false;
3244 } else if (Clobber != "cc") {
3245 Clobber = getTarget().getNormalizedGCCRegisterName(Name: Clobber);
3246 if (CGM.getCodeGenOpts().StackClashProtector &&
3247 getTarget().isSPRegName(Clobber)) {
3248 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3249 DiagID: diag::warn_stack_clash_protection_inline_asm);
3250 }
3251 }
3252
3253 if (isa<MSAsmStmt>(Val: &S)) {
3254 if (Clobber == "eax" || Clobber == "edx") {
3255 if (Constraints.find(s: "=&A") != std::string::npos)
3256 continue;
3257
3258 std::string::size_type position1 =
3259 Constraints.find(str: "={" + Clobber + "}");
3260 if (position1 != std::string::npos) {
3261 Constraints.insert(pos: position1 + 1, s: "&");
3262 continue;
3263 }
3264
3265 std::string::size_type position2 = Constraints.find(s: "=A");
3266 if (position2 != std::string::npos) {
3267 Constraints.insert(pos: position2 + 1, s: "&");
3268 continue;
3269 }
3270 }
3271 }
3272
3273 if (!Constraints.empty())
3274 Constraints += ',';
3275
3276 Constraints += "~{" + Clobber + '}';
3277 }
3278
3279 return HasUnwindClobber;
3280}
3281
3282void AsmConstraintsInfo::UpdateAsmCallInst(
3283 llvm::CallBase &Result, bool HasSideEffect, bool HasUnwindClobber,
3284 bool NoMerge, bool NoConvergent, std::vector<llvm::Value *> &RegResults) {
3285 if (!HasUnwindClobber)
3286 Result.addFnAttr(Kind: llvm::Attribute::NoUnwind);
3287
3288 if (NoMerge)
3289 Result.addFnAttr(Kind: llvm::Attribute::NoMerge);
3290
3291 // Attach readnone and readonly attributes.
3292 if (!HasSideEffect) {
3293 if (ReadNone)
3294 Result.setDoesNotAccessMemory();
3295 else if (ReadOnly)
3296 Result.setOnlyReadsMemory();
3297 }
3298
3299 // Add elementtype attribute for indirect constraints.
3300 for (auto Pair : llvm::enumerate(First&: ArgElemTypes)) {
3301 if (Pair.value()) {
3302 auto Attr = llvm::Attribute::get(
3303 Context&: getLLVMContext(), Kind: llvm::Attribute::ElementType, Ty: Pair.value());
3304 Result.addParamAttr(ArgNo: Pair.index(), Attr);
3305 }
3306 }
3307
3308 // Slap the source location of the inline asm into a !srcloc metadata on the
3309 // call.
3310 const StringLiteral *SL;
3311 if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(Val: &S);
3312 gccAsmStmt &&
3313 (SL = dyn_cast<StringLiteral>(Val: gccAsmStmt->getAsmStringExpr()))) {
3314 Result.setMetadata(Kind: "srcloc", Node: getAsmSrcLocInfo(Str: SL, CGF));
3315 } else {
3316 // At least put the line number on MS inline asm blobs and GCC asm constexpr
3317 // strings.
3318 llvm::Constant *Loc =
3319 llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: S.getAsmLoc().getRawEncoding());
3320 Result.setMetadata(Kind: "srcloc",
3321 Node: llvm::MDNode::get(Context&: getLLVMContext(),
3322 MDs: llvm::ConstantAsMetadata::get(C: Loc)));
3323 }
3324
3325 // Make inline-asm calls Key for the debug info feature Key Instructions.
3326 CGF.addInstToNewSourceAtom(KeyInstruction: &Result, Backup: nullptr);
3327
3328 if (!NoConvergent && getLangOpts().assumeFunctionsAreConvergent())
3329 // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
3330 // convergent (meaning, they may call an intrinsically convergent op, such
3331 // as bar.sync, and so can't have certain optimizations applied around
3332 // them) unless it's explicitly marked 'noconvergent'.
3333 Result.addFnAttr(Kind: llvm::Attribute::Convergent);
3334
3335 // Extract all of the register value results from the asm.
3336 if (ResultRegTypes.size() == 1) {
3337 RegResults.push_back(x: &Result);
3338 } else {
3339 for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
3340 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: &Result, Idxs: i, Name: "asmresult");
3341 RegResults.push_back(x: Tmp);
3342 }
3343 }
3344}
3345
3346void AsmConstraintsInfo::EmitAsmStores(
3347 const llvm::ArrayRef<llvm::Value *> RegResults) {
3348 llvm::LLVMContext &CTX = getLLVMContext();
3349
3350 assert(RegResults.size() == ResultRegTypes.size());
3351 assert(RegResults.size() == ResultTruncRegTypes.size());
3352 assert(RegResults.size() == ResultRegDests.size());
3353
3354 // ResultRegDests can also be populated by addReturnRegisterOutputs() above,
3355 // in which case its size may grow.
3356 assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
3357 assert(ResultBounds.size() <= ResultRegDests.size());
3358
3359 for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
3360 llvm::Value *Tmp = RegResults[i];
3361 llvm::Type *TruncTy = ResultTruncRegTypes[i];
3362
3363 if (i < ResultBounds.size() && ResultBounds[i].has_value()) {
3364 const auto [LowerBound, UpperBound] = ResultBounds[i].value();
3365
3366 // FIXME: Support for nonzero lower bounds not yet implemented.
3367 assert(LowerBound == 0 && "Output operand lower bound is not zero.");
3368
3369 llvm::Constant *UpperBoundConst =
3370 llvm::ConstantInt::get(Ty: Tmp->getType(), V: UpperBound);
3371 llvm::Value *IsBooleanValue =
3372 Builder.CreateCmp(Pred: llvm::CmpInst::ICMP_ULT, LHS: Tmp, RHS: UpperBoundConst);
3373 llvm::Function *FnAssume = CGM.getIntrinsic(IID: llvm::Intrinsic::assume);
3374
3375 Builder.CreateCall(Callee: FnAssume, Args: IsBooleanValue);
3376 }
3377
3378 // If the result type of the LLVM IR asm doesn't match the result type of
3379 // the expression, do the conversion.
3380 if (ResultRegTypes[i] != TruncTy) {
3381 // Truncate the integer result to the right size, note that TruncTy can be
3382 // a pointer.
3383 if (TruncTy->isFloatingPointTy())
3384 Tmp = Builder.CreateFPTrunc(V: Tmp, DestTy: TruncTy);
3385 else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
3386 uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(Ty: TruncTy);
3387 Tmp = Builder.CreateTrunc(
3388 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)ResSize));
3389 Tmp = Builder.CreateIntToPtr(V: Tmp, DestTy: TruncTy);
3390 } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
3391 uint64_t TmpSize =
3392 CGM.getDataLayout().getTypeSizeInBits(Ty: Tmp->getType());
3393 Tmp = Builder.CreatePtrToInt(
3394 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)TmpSize));
3395 Tmp = Builder.CreateTrunc(V: Tmp, DestTy: TruncTy);
3396 } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
3397 Tmp = Builder.CreateZExtOrTrunc(V: Tmp, DestTy: TruncTy);
3398 } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
3399 Tmp = Builder.CreateBitCast(V: Tmp, DestTy: TruncTy);
3400 }
3401 }
3402
3403 ApplyAtomGroup Grp(CGF.getDebugInfo());
3404 LValue Dest = ResultRegDests[i];
3405
3406 // ResultTypeRequiresCast elements correspond to the first
3407 // ResultTypeRequiresCast.size() elements of RegResults.
3408 if (i < ResultTypeRequiresCast.size() && ResultTypeRequiresCast[i]) {
3409 unsigned Size = getContext().getTypeSize(T: ResultRegQualTys[i]);
3410 Address A = Dest.getAddress().withElementType(ElemTy: ResultRegTypes[i]);
3411
3412 if (getTargetHooks().isScalarizableAsmOperand(CGF, Ty: TruncTy)) {
3413 llvm::StoreInst *S = Builder.CreateStore(Val: Tmp, Addr: A);
3414 CGF.addInstToCurrentSourceAtom(KeyInstruction: S, Backup: S->getValueOperand());
3415 continue;
3416 }
3417
3418 QualType Ty = getContext().getIntTypeForBitwidth(DestWidth: Size, /*Signed=*/false);
3419 if (Ty.isNull()) {
3420 const Expr *OutExpr = S.getOutputExpr(i);
3421 CGM.getDiags().Report(Loc: OutExpr->getExprLoc(),
3422 DiagID: diag::err_store_value_to_reg);
3423 return;
3424 }
3425
3426 Dest = CGF.MakeAddrLValue(Addr: A, T: Ty);
3427 }
3428
3429 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Tmp), Dst: Dest);
3430 }
3431}
3432
3433LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
3434 const RecordDecl *RD = S.getCapturedRecordDecl();
3435 CanQualType RecordTy = getContext().getCanonicalTagType(TD: RD);
3436
3437 // Initialize the captured struct.
3438 LValue SlotLV =
3439 MakeAddrLValue(Addr: CreateMemTemp(T: RecordTy, Name: "agg.captured"), T: RecordTy);
3440
3441 RecordDecl::field_iterator CurField = RD->field_begin();
3442 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
3443 E = S.capture_init_end();
3444 I != E; ++I, ++CurField) {
3445 LValue LV = EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
3446 if (CurField->hasCapturedVLAType()) {
3447 EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
3448 } else {
3449 EmitInitializerForField(Field: *CurField, LHS: LV, Init: *I);
3450 }
3451 }
3452
3453 return SlotLV;
3454}
3455
3456/// Generate an outlined function for the body of a CapturedStmt, store any
3457/// captured variables into the captured struct, and call the outlined function.
3458llvm::Function *
3459CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
3460 LValue CapStruct = InitCapturedStruct(S);
3461
3462 // Emit the CapturedDecl
3463 CodeGenFunction CGF(CGM, true);
3464 CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
3465 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
3466 delete CGF.CapturedStmtInfo;
3467
3468 // Emit call to the helper function.
3469 EmitCallOrInvoke(Callee: F, Args: CapStruct.getPointer(CGF&: *this));
3470
3471 return F;
3472}
3473
3474Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
3475 LValue CapStruct = InitCapturedStruct(S);
3476 return CapStruct.getAddress();
3477}
3478
3479/// Creates the outlined function for a CapturedStmt.
3480llvm::Function *
3481CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
3482 assert(CapturedStmtInfo &&
3483 "CapturedStmtInfo should be set when generating the captured function");
3484 const CapturedDecl *CD = S.getCapturedDecl();
3485 const RecordDecl *RD = S.getCapturedRecordDecl();
3486 SourceLocation Loc = S.getBeginLoc();
3487 assert(CD->hasBody() && "missing CapturedDecl body");
3488
3489 // Build the argument list.
3490 ASTContext &Ctx = CGM.getContext();
3491 FunctionArgList Args;
3492 Args.append(in_start: CD->param_begin(), in_end: CD->param_end());
3493
3494 // Create the function declaration.
3495 const CGFunctionInfo &FuncInfo =
3496 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
3497 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
3498
3499 llvm::Function *F =
3500 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
3501 N: CapturedStmtInfo->getHelperName(), M: &CGM.getModule());
3502 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
3503 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3504 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3505 if (CD->isNothrow())
3506 F->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3507
3508 // Generate the function.
3509 StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc: CD->getLocation(),
3510 StartLoc: CD->getBody()->getBeginLoc());
3511 // Set the context parameter in CapturedStmtInfo.
3512 Address DeclPtr = GetAddrOfLocalVar(VD: CD->getContextParam());
3513 CapturedStmtInfo->setContextValue(Builder.CreateLoad(Addr: DeclPtr));
3514
3515 // Initialize variable-length arrays.
3516 LValue Base = MakeNaturalAlignRawAddrLValue(
3517 V: CapturedStmtInfo->getContextValue(), T: Ctx.getCanonicalTagType(TD: RD));
3518 for (auto *FD : RD->fields()) {
3519 if (FD->hasCapturedVLAType()) {
3520 auto *ExprArg =
3521 EmitLoadOfLValue(V: EmitLValueForField(Base, Field: FD), Loc: S.getBeginLoc())
3522 .getScalarVal();
3523 auto VAT = FD->getCapturedVLAType();
3524 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
3525 }
3526 }
3527
3528 // If 'this' is captured, load it into CXXThisValue.
3529 if (CapturedStmtInfo->isCXXThisExprCaptured()) {
3530 FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
3531 LValue ThisLValue = EmitLValueForField(Base, Field: FD);
3532 CXXThisValue = EmitLoadOfLValue(V: ThisLValue, Loc).getScalarVal();
3533 }
3534
3535 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
3536 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
3537 FinishFunction(EndLoc: CD->getBodyRBrace());
3538
3539 return F;
3540}
3541
3542// Returns the first convergence entry/loop/anchor instruction found in |BB|.
3543// std::nullptr otherwise.
3544static llvm::ConvergenceControlInst *getConvergenceToken(llvm::BasicBlock *BB) {
3545 for (auto &I : *BB) {
3546 if (auto *CI = dyn_cast<llvm::ConvergenceControlInst>(Val: &I))
3547 return CI;
3548 }
3549 return nullptr;
3550}
3551
3552llvm::CallBase *
3553CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input) {
3554 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3555 assert(ParentToken);
3556
3557 llvm::Value *bundleArgs[] = {ParentToken};
3558 llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
3559 auto *Output = llvm::CallBase::addOperandBundle(
3560 CB: Input, ID: llvm::LLVMContext::OB_convergencectrl, OB, InsertPt: Input->getIterator());
3561 Input->replaceAllUsesWith(V: Output);
3562 Input->eraseFromParent();
3563 return Output;
3564}
3565
3566llvm::ConvergenceControlInst *
3567CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {
3568 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3569 assert(ParentToken);
3570 return llvm::ConvergenceControlInst::CreateLoop(BB&: *BB, Parent: ParentToken);
3571}
3572
3573llvm::ConvergenceControlInst *
3574CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
3575 llvm::BasicBlock *BB = &F->getEntryBlock();
3576 llvm::ConvergenceControlInst *Token = getConvergenceToken(BB);
3577 if (Token)
3578 return Token;
3579
3580 // Adding a convergence token requires the function to be marked as
3581 // convergent.
3582 F->setConvergent();
3583 return llvm::ConvergenceControlInst::CreateEntry(BB&: *BB);
3584}
3585