1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Stmt nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGDebugInfo.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "CodeGenPGO.h"
18#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Expr.h"
21#include "clang/AST/Stmt.h"
22#include "clang/AST/StmtSYCL.h"
23#include "clang/AST/StmtVisitor.h"
24#include "clang/Basic/Builtins.h"
25#include "clang/Basic/DiagnosticSema.h"
26#include "clang/Basic/PrettyStackTrace.h"
27#include "clang/Basic/SourceManager.h"
28#include "clang/Basic/TargetInfo.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/IR/Assumptions.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/InlineAsm.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/MDBuilder.h"
38#include "llvm/Support/SaveAndRestore.h"
39#include <optional>
40
41using namespace clang;
42using namespace CodeGen;
43
44//===----------------------------------------------------------------------===//
45// Statement Emission
46//===----------------------------------------------------------------------===//
47
48void CodeGenFunction::EmitStopPoint(const Stmt *S) {
49 if (CGDebugInfo *DI = getDebugInfo()) {
50 SourceLocation Loc;
51 Loc = S->getBeginLoc();
52 DI->EmitLocation(Builder, Loc);
53
54 LastStopPoint = Loc;
55 }
56}
57
58void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
59 assert(S && "Null statement?");
60 PGO->setCurrentStmt(S);
61
62 // These statements have their own debug info handling.
63 if (EmitSimpleStmt(S, Attrs))
64 return;
65
66 // Check if we are generating unreachable code.
67 if (!HaveInsertPoint()) {
68 // If so, and the statement doesn't contain a label, then we do not need to
69 // generate actual code. This is safe because (1) the current point is
70 // unreachable, so we don't need to execute the code, and (2) we've already
71 // handled the statements which update internal data structures (like the
72 // local variable map) which could be used by subsequent statements.
73 if (!ContainsLabel(S)) {
74 // Verify that any decl statements were handled as simple, they may be in
75 // scope of subsequent reachable statements.
76 assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
77 PGO->markStmtMaybeUsed(S);
78 return;
79 }
80
81 // Otherwise, make a new block to hold the code.
82 EnsureInsertPoint();
83 }
84
85 // Generate a stoppoint if we are emitting debug info.
86 EmitStopPoint(S);
87
88 // Ignore all OpenMP directives except for simd if OpenMP with Simd is
89 // enabled.
90 if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
91 if (const auto *D = dyn_cast<OMPExecutableDirective>(Val: S)) {
92 EmitSimpleOMPExecutableDirective(D: *D);
93 return;
94 }
95 }
96
97 switch (S->getStmtClass()) {
98 case Stmt::NoStmtClass:
99 case Stmt::CXXCatchStmtClass:
100 case Stmt::SEHExceptStmtClass:
101 case Stmt::SEHFinallyStmtClass:
102 case Stmt::MSDependentExistsStmtClass:
103 case Stmt::UnresolvedSYCLKernelCallStmtClass:
104 llvm_unreachable("invalid statement class to emit generically");
105 case Stmt::NullStmtClass:
106 case Stmt::CompoundStmtClass:
107 case Stmt::DeclStmtClass:
108 case Stmt::LabelStmtClass:
109 case Stmt::AttributedStmtClass:
110 case Stmt::GotoStmtClass:
111 case Stmt::BreakStmtClass:
112 case Stmt::ContinueStmtClass:
113 case Stmt::DefaultStmtClass:
114 case Stmt::CaseStmtClass:
115 case Stmt::DeferStmtClass:
116 case Stmt::SEHLeaveStmtClass:
117 case Stmt::SYCLKernelCallStmtClass:
118 llvm_unreachable("should have emitted these statements as simple");
119
120#define STMT(Type, Base)
121#define ABSTRACT_STMT(Op)
122#define EXPR(Type, Base) \
123 case Stmt::Type##Class:
124#include "clang/AST/StmtNodes.inc"
125 {
126 // Remember the block we came in on.
127 llvm::BasicBlock *incoming = Builder.GetInsertBlock();
128 assert(incoming && "expression emission must have an insertion point");
129
130 EmitIgnoredExpr(E: cast<Expr>(Val: S));
131
132 llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
133 assert(outgoing && "expression emission cleared block!");
134
135 // The expression emitters assume (reasonably!) that the insertion
136 // point is always set. To maintain that, the call-emission code
137 // for noreturn functions has to enter a new block with no
138 // predecessors. We want to kill that block and mark the current
139 // insertion point unreachable in the common case of a call like
140 // "exit();". Since expression emission doesn't otherwise create
141 // blocks with no predecessors, we can just test for that.
142 // However, we must be careful not to do this to our incoming
143 // block, because *statement* emission does sometimes create
144 // reachable blocks which will have no predecessors until later in
145 // the function. This occurs with, e.g., labels that are not
146 // reachable by fallthrough.
147 if (incoming != outgoing && outgoing->use_empty()) {
148 outgoing->eraseFromParent();
149 Builder.ClearInsertionPoint();
150 }
151 break;
152 }
153
154 case Stmt::IndirectGotoStmtClass:
155 EmitIndirectGotoStmt(S: cast<IndirectGotoStmt>(Val: *S)); break;
156
157 case Stmt::IfStmtClass: EmitIfStmt(S: cast<IfStmt>(Val: *S)); break;
158 case Stmt::WhileStmtClass: EmitWhileStmt(S: cast<WhileStmt>(Val: *S), Attrs); break;
159 case Stmt::DoStmtClass: EmitDoStmt(S: cast<DoStmt>(Val: *S), Attrs); break;
160 case Stmt::ForStmtClass: EmitForStmt(S: cast<ForStmt>(Val: *S), Attrs); break;
161
162 case Stmt::ReturnStmtClass: EmitReturnStmt(S: cast<ReturnStmt>(Val: *S)); break;
163
164 case Stmt::SwitchStmtClass: EmitSwitchStmt(S: cast<SwitchStmt>(Val: *S)); break;
165 case Stmt::GCCAsmStmtClass: // Intentional fall-through.
166 case Stmt::MSAsmStmtClass: EmitAsmStmt(S: cast<AsmStmt>(Val: *S)); break;
167 case Stmt::CoroutineBodyStmtClass:
168 EmitCoroutineBody(S: cast<CoroutineBodyStmt>(Val: *S));
169 break;
170 case Stmt::CoreturnStmtClass:
171 EmitCoreturnStmt(S: cast<CoreturnStmt>(Val: *S));
172 break;
173 case Stmt::CapturedStmtClass: {
174 const CapturedStmt *CS = cast<CapturedStmt>(Val: S);
175 EmitCapturedStmt(S: *CS, K: CS->getCapturedRegionKind());
176 }
177 break;
178 case Stmt::ObjCAtTryStmtClass:
179 EmitObjCAtTryStmt(S: cast<ObjCAtTryStmt>(Val: *S));
180 break;
181 case Stmt::ObjCAtCatchStmtClass:
182 llvm_unreachable(
183 "@catch statements should be handled by EmitObjCAtTryStmt");
184 case Stmt::ObjCAtFinallyStmtClass:
185 llvm_unreachable(
186 "@finally statements should be handled by EmitObjCAtTryStmt");
187 case Stmt::ObjCAtThrowStmtClass:
188 EmitObjCAtThrowStmt(S: cast<ObjCAtThrowStmt>(Val: *S));
189 break;
190 case Stmt::ObjCAtSynchronizedStmtClass:
191 EmitObjCAtSynchronizedStmt(S: cast<ObjCAtSynchronizedStmt>(Val: *S));
192 break;
193 case Stmt::ObjCForCollectionStmtClass:
194 EmitObjCForCollectionStmt(S: cast<ObjCForCollectionStmt>(Val: *S));
195 break;
196 case Stmt::ObjCAutoreleasePoolStmtClass:
197 EmitObjCAutoreleasePoolStmt(S: cast<ObjCAutoreleasePoolStmt>(Val: *S));
198 break;
199
200 case Stmt::CXXTryStmtClass:
201 EmitCXXTryStmt(S: cast<CXXTryStmt>(Val: *S));
202 break;
203 case Stmt::CXXForRangeStmtClass:
204 EmitCXXForRangeStmt(S: cast<CXXForRangeStmt>(Val: *S), Attrs);
205 break;
206 case Stmt::CXXExpansionStmtPatternClass:
207 llvm_unreachable("unexpanded expansion statements should not be emitted");
208 case Stmt::CXXExpansionStmtInstantiationClass:
209 EmitCXXExpansionStmtInstantiation(S: cast<CXXExpansionStmtInstantiation>(Val: *S));
210 break;
211 case Stmt::SEHTryStmtClass:
212 EmitSEHTryStmt(S: cast<SEHTryStmt>(Val: *S));
213 break;
214 case Stmt::OMPMetaDirectiveClass:
215 EmitOMPMetaDirective(S: cast<OMPMetaDirective>(Val: *S));
216 break;
217 case Stmt::OMPCanonicalLoopClass:
218 EmitOMPCanonicalLoop(S: cast<OMPCanonicalLoop>(Val: S));
219 break;
220 case Stmt::OMPParallelDirectiveClass:
221 EmitOMPParallelDirective(S: cast<OMPParallelDirective>(Val: *S));
222 break;
223 case Stmt::OMPSimdDirectiveClass:
224 EmitOMPSimdDirective(S: cast<OMPSimdDirective>(Val: *S));
225 break;
226 case Stmt::OMPTileDirectiveClass:
227 EmitOMPTileDirective(S: cast<OMPTileDirective>(Val: *S));
228 break;
229 case Stmt::OMPStripeDirectiveClass:
230 EmitOMPStripeDirective(S: cast<OMPStripeDirective>(Val: *S));
231 break;
232 case Stmt::OMPUnrollDirectiveClass:
233 EmitOMPUnrollDirective(S: cast<OMPUnrollDirective>(Val: *S));
234 break;
235 case Stmt::OMPReverseDirectiveClass:
236 EmitOMPReverseDirective(S: cast<OMPReverseDirective>(Val: *S));
237 break;
238 case Stmt::OMPSplitDirectiveClass:
239 EmitOMPSplitDirective(S: cast<OMPSplitDirective>(Val: *S));
240 break;
241 case Stmt::OMPInterchangeDirectiveClass:
242 EmitOMPInterchangeDirective(S: cast<OMPInterchangeDirective>(Val: *S));
243 break;
244 case Stmt::OMPFuseDirectiveClass:
245 EmitOMPFuseDirective(S: cast<OMPFuseDirective>(Val: *S));
246 break;
247 case Stmt::OMPForDirectiveClass:
248 EmitOMPForDirective(S: cast<OMPForDirective>(Val: *S));
249 break;
250 case Stmt::OMPForSimdDirectiveClass:
251 EmitOMPForSimdDirective(S: cast<OMPForSimdDirective>(Val: *S));
252 break;
253 case Stmt::OMPSectionsDirectiveClass:
254 EmitOMPSectionsDirective(S: cast<OMPSectionsDirective>(Val: *S));
255 break;
256 case Stmt::OMPSectionDirectiveClass:
257 EmitOMPSectionDirective(S: cast<OMPSectionDirective>(Val: *S));
258 break;
259 case Stmt::OMPSingleDirectiveClass:
260 EmitOMPSingleDirective(S: cast<OMPSingleDirective>(Val: *S));
261 break;
262 case Stmt::OMPMasterDirectiveClass:
263 EmitOMPMasterDirective(S: cast<OMPMasterDirective>(Val: *S));
264 break;
265 case Stmt::OMPCriticalDirectiveClass:
266 EmitOMPCriticalDirective(S: cast<OMPCriticalDirective>(Val: *S));
267 break;
268 case Stmt::OMPParallelForDirectiveClass:
269 EmitOMPParallelForDirective(S: cast<OMPParallelForDirective>(Val: *S));
270 break;
271 case Stmt::OMPParallelForSimdDirectiveClass:
272 EmitOMPParallelForSimdDirective(S: cast<OMPParallelForSimdDirective>(Val: *S));
273 break;
274 case Stmt::OMPParallelMasterDirectiveClass:
275 EmitOMPParallelMasterDirective(S: cast<OMPParallelMasterDirective>(Val: *S));
276 break;
277 case Stmt::OMPParallelSectionsDirectiveClass:
278 EmitOMPParallelSectionsDirective(S: cast<OMPParallelSectionsDirective>(Val: *S));
279 break;
280 case Stmt::OMPTaskDirectiveClass:
281 EmitOMPTaskDirective(S: cast<OMPTaskDirective>(Val: *S));
282 break;
283 case Stmt::OMPTaskyieldDirectiveClass:
284 EmitOMPTaskyieldDirective(S: cast<OMPTaskyieldDirective>(Val: *S));
285 break;
286 case Stmt::OMPErrorDirectiveClass:
287 EmitOMPErrorDirective(S: cast<OMPErrorDirective>(Val: *S));
288 break;
289 case Stmt::OMPBarrierDirectiveClass:
290 EmitOMPBarrierDirective(S: cast<OMPBarrierDirective>(Val: *S));
291 break;
292 case Stmt::OMPTaskwaitDirectiveClass:
293 EmitOMPTaskwaitDirective(S: cast<OMPTaskwaitDirective>(Val: *S));
294 break;
295 case Stmt::OMPTaskgroupDirectiveClass:
296 EmitOMPTaskgroupDirective(S: cast<OMPTaskgroupDirective>(Val: *S));
297 break;
298 case Stmt::OMPFlushDirectiveClass:
299 EmitOMPFlushDirective(S: cast<OMPFlushDirective>(Val: *S));
300 break;
301 case Stmt::OMPDepobjDirectiveClass:
302 EmitOMPDepobjDirective(S: cast<OMPDepobjDirective>(Val: *S));
303 break;
304 case Stmt::OMPScanDirectiveClass:
305 EmitOMPScanDirective(S: cast<OMPScanDirective>(Val: *S));
306 break;
307 case Stmt::OMPOrderedDirectiveClass:
308 EmitOMPOrderedDirective(S: cast<OMPOrderedDirective>(Val: *S));
309 break;
310 case Stmt::OMPAtomicDirectiveClass:
311 EmitOMPAtomicDirective(S: cast<OMPAtomicDirective>(Val: *S));
312 break;
313 case Stmt::OMPTargetDirectiveClass:
314 EmitOMPTargetDirective(S: cast<OMPTargetDirective>(Val: *S));
315 break;
316 case Stmt::OMPTeamsDirectiveClass:
317 EmitOMPTeamsDirective(S: cast<OMPTeamsDirective>(Val: *S));
318 break;
319 case Stmt::OMPCancellationPointDirectiveClass:
320 EmitOMPCancellationPointDirective(S: cast<OMPCancellationPointDirective>(Val: *S));
321 break;
322 case Stmt::OMPCancelDirectiveClass:
323 EmitOMPCancelDirective(S: cast<OMPCancelDirective>(Val: *S));
324 break;
325 case Stmt::OMPTargetDataDirectiveClass:
326 EmitOMPTargetDataDirective(S: cast<OMPTargetDataDirective>(Val: *S));
327 break;
328 case Stmt::OMPTargetEnterDataDirectiveClass:
329 EmitOMPTargetEnterDataDirective(S: cast<OMPTargetEnterDataDirective>(Val: *S));
330 break;
331 case Stmt::OMPTargetExitDataDirectiveClass:
332 EmitOMPTargetExitDataDirective(S: cast<OMPTargetExitDataDirective>(Val: *S));
333 break;
334 case Stmt::OMPTargetParallelDirectiveClass:
335 EmitOMPTargetParallelDirective(S: cast<OMPTargetParallelDirective>(Val: *S));
336 break;
337 case Stmt::OMPTargetParallelForDirectiveClass:
338 EmitOMPTargetParallelForDirective(S: cast<OMPTargetParallelForDirective>(Val: *S));
339 break;
340 case Stmt::OMPTaskLoopDirectiveClass:
341 EmitOMPTaskLoopDirective(S: cast<OMPTaskLoopDirective>(Val: *S));
342 break;
343 case Stmt::OMPTaskLoopSimdDirectiveClass:
344 EmitOMPTaskLoopSimdDirective(S: cast<OMPTaskLoopSimdDirective>(Val: *S));
345 break;
346 case Stmt::OMPMasterTaskLoopDirectiveClass:
347 EmitOMPMasterTaskLoopDirective(S: cast<OMPMasterTaskLoopDirective>(Val: *S));
348 break;
349 case Stmt::OMPMaskedTaskLoopDirectiveClass:
350 EmitOMPMaskedTaskLoopDirective(S: cast<OMPMaskedTaskLoopDirective>(Val: *S));
351 break;
352 case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
353 EmitOMPMasterTaskLoopSimdDirective(
354 S: cast<OMPMasterTaskLoopSimdDirective>(Val: *S));
355 break;
356 case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
357 EmitOMPMaskedTaskLoopSimdDirective(
358 S: cast<OMPMaskedTaskLoopSimdDirective>(Val: *S));
359 break;
360 case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
361 EmitOMPParallelMasterTaskLoopDirective(
362 S: cast<OMPParallelMasterTaskLoopDirective>(Val: *S));
363 break;
364 case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
365 EmitOMPParallelMaskedTaskLoopDirective(
366 S: cast<OMPParallelMaskedTaskLoopDirective>(Val: *S));
367 break;
368 case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
369 EmitOMPParallelMasterTaskLoopSimdDirective(
370 S: cast<OMPParallelMasterTaskLoopSimdDirective>(Val: *S));
371 break;
372 case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
373 EmitOMPParallelMaskedTaskLoopSimdDirective(
374 S: cast<OMPParallelMaskedTaskLoopSimdDirective>(Val: *S));
375 break;
376 case Stmt::OMPDistributeDirectiveClass:
377 EmitOMPDistributeDirective(S: cast<OMPDistributeDirective>(Val: *S));
378 break;
379 case Stmt::OMPTargetUpdateDirectiveClass:
380 EmitOMPTargetUpdateDirective(S: cast<OMPTargetUpdateDirective>(Val: *S));
381 break;
382 case Stmt::OMPDistributeParallelForDirectiveClass:
383 EmitOMPDistributeParallelForDirective(
384 S: cast<OMPDistributeParallelForDirective>(Val: *S));
385 break;
386 case Stmt::OMPDistributeParallelForSimdDirectiveClass:
387 EmitOMPDistributeParallelForSimdDirective(
388 S: cast<OMPDistributeParallelForSimdDirective>(Val: *S));
389 break;
390 case Stmt::OMPDistributeSimdDirectiveClass:
391 EmitOMPDistributeSimdDirective(S: cast<OMPDistributeSimdDirective>(Val: *S));
392 break;
393 case Stmt::OMPTargetParallelForSimdDirectiveClass:
394 EmitOMPTargetParallelForSimdDirective(
395 S: cast<OMPTargetParallelForSimdDirective>(Val: *S));
396 break;
397 case Stmt::OMPTargetSimdDirectiveClass:
398 EmitOMPTargetSimdDirective(S: cast<OMPTargetSimdDirective>(Val: *S));
399 break;
400 case Stmt::OMPTeamsDistributeDirectiveClass:
401 EmitOMPTeamsDistributeDirective(S: cast<OMPTeamsDistributeDirective>(Val: *S));
402 break;
403 case Stmt::OMPTeamsDistributeSimdDirectiveClass:
404 EmitOMPTeamsDistributeSimdDirective(
405 S: cast<OMPTeamsDistributeSimdDirective>(Val: *S));
406 break;
407 case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
408 EmitOMPTeamsDistributeParallelForSimdDirective(
409 S: cast<OMPTeamsDistributeParallelForSimdDirective>(Val: *S));
410 break;
411 case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
412 EmitOMPTeamsDistributeParallelForDirective(
413 S: cast<OMPTeamsDistributeParallelForDirective>(Val: *S));
414 break;
415 case Stmt::OMPTargetTeamsDirectiveClass:
416 EmitOMPTargetTeamsDirective(S: cast<OMPTargetTeamsDirective>(Val: *S));
417 break;
418 case Stmt::OMPTargetTeamsDistributeDirectiveClass:
419 EmitOMPTargetTeamsDistributeDirective(
420 S: cast<OMPTargetTeamsDistributeDirective>(Val: *S));
421 break;
422 case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
423 EmitOMPTargetTeamsDistributeParallelForDirective(
424 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: *S));
425 break;
426 case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
427 EmitOMPTargetTeamsDistributeParallelForSimdDirective(
428 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: *S));
429 break;
430 case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
431 EmitOMPTargetTeamsDistributeSimdDirective(
432 S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: *S));
433 break;
434 case Stmt::OMPInteropDirectiveClass:
435 EmitOMPInteropDirective(S: cast<OMPInteropDirective>(Val: *S));
436 break;
437 case Stmt::OMPDispatchDirectiveClass:
438 CGM.ErrorUnsupported(S, Type: "OpenMP dispatch directive");
439 break;
440 case Stmt::OMPScopeDirectiveClass:
441 EmitOMPScopeDirective(S: cast<OMPScopeDirective>(Val: *S));
442 break;
443 case Stmt::OMPMaskedDirectiveClass:
444 EmitOMPMaskedDirective(S: cast<OMPMaskedDirective>(Val: *S));
445 break;
446 case Stmt::OMPGenericLoopDirectiveClass:
447 EmitOMPGenericLoopDirective(S: cast<OMPGenericLoopDirective>(Val: *S));
448 break;
449 case Stmt::OMPTeamsGenericLoopDirectiveClass:
450 EmitOMPTeamsGenericLoopDirective(S: cast<OMPTeamsGenericLoopDirective>(Val: *S));
451 break;
452 case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
453 EmitOMPTargetTeamsGenericLoopDirective(
454 S: cast<OMPTargetTeamsGenericLoopDirective>(Val: *S));
455 break;
456 case Stmt::OMPParallelGenericLoopDirectiveClass:
457 EmitOMPParallelGenericLoopDirective(
458 S: cast<OMPParallelGenericLoopDirective>(Val: *S));
459 break;
460 case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
461 EmitOMPTargetParallelGenericLoopDirective(
462 S: cast<OMPTargetParallelGenericLoopDirective>(Val: *S));
463 break;
464 case Stmt::OMPParallelMaskedDirectiveClass:
465 EmitOMPParallelMaskedDirective(S: cast<OMPParallelMaskedDirective>(Val: *S));
466 break;
467 case Stmt::OMPAssumeDirectiveClass:
468 EmitOMPAssumeDirective(S: cast<OMPAssumeDirective>(Val: *S));
469 break;
470 case Stmt::OpenACCComputeConstructClass:
471 EmitOpenACCComputeConstruct(S: cast<OpenACCComputeConstruct>(Val: *S));
472 break;
473 case Stmt::OpenACCLoopConstructClass:
474 EmitOpenACCLoopConstruct(S: cast<OpenACCLoopConstruct>(Val: *S));
475 break;
476 case Stmt::OpenACCCombinedConstructClass:
477 EmitOpenACCCombinedConstruct(S: cast<OpenACCCombinedConstruct>(Val: *S));
478 break;
479 case Stmt::OpenACCDataConstructClass:
480 EmitOpenACCDataConstruct(S: cast<OpenACCDataConstruct>(Val: *S));
481 break;
482 case Stmt::OpenACCEnterDataConstructClass:
483 EmitOpenACCEnterDataConstruct(S: cast<OpenACCEnterDataConstruct>(Val: *S));
484 break;
485 case Stmt::OpenACCExitDataConstructClass:
486 EmitOpenACCExitDataConstruct(S: cast<OpenACCExitDataConstruct>(Val: *S));
487 break;
488 case Stmt::OpenACCHostDataConstructClass:
489 EmitOpenACCHostDataConstruct(S: cast<OpenACCHostDataConstruct>(Val: *S));
490 break;
491 case Stmt::OpenACCWaitConstructClass:
492 EmitOpenACCWaitConstruct(S: cast<OpenACCWaitConstruct>(Val: *S));
493 break;
494 case Stmt::OpenACCInitConstructClass:
495 EmitOpenACCInitConstruct(S: cast<OpenACCInitConstruct>(Val: *S));
496 break;
497 case Stmt::OpenACCShutdownConstructClass:
498 EmitOpenACCShutdownConstruct(S: cast<OpenACCShutdownConstruct>(Val: *S));
499 break;
500 case Stmt::OpenACCSetConstructClass:
501 EmitOpenACCSetConstruct(S: cast<OpenACCSetConstruct>(Val: *S));
502 break;
503 case Stmt::OpenACCUpdateConstructClass:
504 EmitOpenACCUpdateConstruct(S: cast<OpenACCUpdateConstruct>(Val: *S));
505 break;
506 case Stmt::OpenACCAtomicConstructClass:
507 EmitOpenACCAtomicConstruct(S: cast<OpenACCAtomicConstruct>(Val: *S));
508 break;
509 case Stmt::OpenACCCacheConstructClass:
510 EmitOpenACCCacheConstruct(S: cast<OpenACCCacheConstruct>(Val: *S));
511 break;
512 }
513}
514
515bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
516 ArrayRef<const Attr *> Attrs) {
517 switch (S->getStmtClass()) {
518 default:
519 return false;
520 case Stmt::NullStmtClass:
521 break;
522 case Stmt::CompoundStmtClass:
523 EmitCompoundStmt(S: cast<CompoundStmt>(Val: *S));
524 break;
525 case Stmt::DeclStmtClass:
526 EmitDeclStmt(S: cast<DeclStmt>(Val: *S));
527 break;
528 case Stmt::LabelStmtClass:
529 EmitLabelStmt(S: cast<LabelStmt>(Val: *S));
530 break;
531 case Stmt::AttributedStmtClass:
532 EmitAttributedStmt(S: cast<AttributedStmt>(Val: *S));
533 break;
534 case Stmt::GotoStmtClass:
535 EmitGotoStmt(S: cast<GotoStmt>(Val: *S));
536 break;
537 case Stmt::BreakStmtClass:
538 EmitBreakStmt(S: cast<BreakStmt>(Val: *S));
539 break;
540 case Stmt::ContinueStmtClass:
541 EmitContinueStmt(S: cast<ContinueStmt>(Val: *S));
542 break;
543 case Stmt::DefaultStmtClass:
544 EmitDefaultStmt(S: cast<DefaultStmt>(Val: *S), Attrs);
545 break;
546 case Stmt::CaseStmtClass:
547 EmitCaseStmt(S: cast<CaseStmt>(Val: *S), Attrs);
548 break;
549 case Stmt::DeferStmtClass:
550 EmitDeferStmt(S: cast<DeferStmt>(Val: *S));
551 break;
552 case Stmt::SEHLeaveStmtClass:
553 EmitSEHLeaveStmt(S: cast<SEHLeaveStmt>(Val: *S));
554 break;
555 case Stmt::SYCLKernelCallStmtClass:
556 EmitSYCLKernelCallStmt(S: cast<SYCLKernelCallStmt>(Val: *S));
557 break;
558 }
559 return true;
560}
561
562/// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true,
563/// this captures the expression result of the last sub-statement and returns it
564/// (for use by the statement expression extension).
565Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
566 AggValueSlot AggSlot) {
567 PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
568 "LLVM IR generation of compound statement ('{}')");
569
570 // Keep track of the current cleanup stack depth, including debug scopes.
571 LexicalScope Scope(*this, S.getSourceRange());
572
573 return EmitCompoundStmtWithoutScope(S, GetLast, AVS: AggSlot);
574}
575
576Address
577CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
578 bool GetLast,
579 AggValueSlot AggSlot) {
580
581 for (CompoundStmt::const_body_iterator I = S.body_begin(),
582 E = S.body_end() - GetLast;
583 I != E; ++I)
584 EmitStmt(S: *I);
585
586 Address RetAlloca = Address::invalid();
587 if (GetLast) {
588 // We have to special case labels here. They are statements, but when put
589 // at the end of a statement expression, they yield the value of their
590 // subexpression. Handle this by walking through all labels we encounter,
591 // emitting them before we evaluate the subexpr.
592 // Similar issues arise for attributed statements.
593 const Stmt *LastStmt = S.body_back();
594 while (!isa<Expr>(Val: LastStmt)) {
595 if (const auto *LS = dyn_cast<LabelStmt>(Val: LastStmt)) {
596 EmitLabel(D: LS->getDecl());
597 LastStmt = LS->getSubStmt();
598 } else if (const auto *AS = dyn_cast<AttributedStmt>(Val: LastStmt)) {
599 // FIXME: Update this if we ever have attributes that affect the
600 // semantics of an expression.
601 LastStmt = AS->getSubStmt();
602 } else {
603 llvm_unreachable("unknown value statement");
604 }
605 }
606
607 EnsureInsertPoint();
608
609 const Expr *E = cast<Expr>(Val: LastStmt);
610 QualType ExprTy = E->getType();
611 if (hasAggregateEvaluationKind(T: ExprTy)) {
612 EmitAggExpr(E, AS: AggSlot);
613 } else {
614 // We can't return an RValue here because there might be cleanups at
615 // the end of the StmtExpr. Because of that, we have to emit the result
616 // here into a temporary alloca.
617 RetAlloca = CreateMemTempWithoutCast(T: ExprTy);
618 EmitAnyExprToMem(E, Location: RetAlloca, Quals: Qualifiers(),
619 /*IsInit*/ IsInitializer: false);
620 }
621 }
622
623 return RetAlloca;
624}
625
626void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
627 llvm::UncondBrInst *BI = dyn_cast<llvm::UncondBrInst>(Val: BB->getTerminator());
628
629 // If there is a cleanup stack, then we it isn't worth trying to
630 // simplify this block (we would need to remove it from the scope map
631 // and cleanup entry).
632 if (!EHStack.empty())
633 return;
634
635 // Can only simplify direct branches.
636 if (!BI)
637 return;
638
639 // Can only simplify empty blocks.
640 if (BI->getIterator() != BB->begin())
641 return;
642
643 BB->replaceAllUsesWith(V: BI->getSuccessor());
644 BI->eraseFromParent();
645 BB->eraseFromParent();
646}
647
648void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
649 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
650
651 // Fall out of the current block (if necessary).
652 EmitBranch(Block: BB);
653
654 if (IsFinished && BB->use_empty()) {
655 delete BB;
656 return;
657 }
658
659 // Place the block after the current block, if possible, or else at
660 // the end of the function.
661 if (CurBB && CurBB->getParent())
662 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
663 else
664 CurFn->insert(Position: CurFn->end(), BB);
665 Builder.SetInsertPoint(BB);
666}
667
668void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
669 // Emit a branch from the current block to the target one if this
670 // was a real block. If this was just a fall-through block after a
671 // terminator, don't emit it.
672 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
673
674 if (!CurBB || CurBB->hasTerminator()) {
675 // If there is no insert point or the previous block is already
676 // terminated, don't touch it.
677 } else {
678 // Otherwise, create a fall-through branch.
679 Builder.CreateBr(Dest: Target);
680 }
681
682 Builder.ClearInsertionPoint();
683}
684
685void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
686 bool inserted = false;
687 for (llvm::User *u : block->users()) {
688 if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(Val: u)) {
689 CurFn->insert(Position: std::next(x: insn->getParent()->getIterator()), BB: block);
690 inserted = true;
691 break;
692 }
693 }
694
695 if (!inserted)
696 CurFn->insert(Position: CurFn->end(), BB: block);
697
698 Builder.SetInsertPoint(block);
699}
700
701CodeGenFunction::JumpDest
702CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
703 JumpDest &Dest = LabelMap[D];
704 if (Dest.isValid()) return Dest;
705
706 // Create, but don't insert, the new block.
707 Dest = JumpDest(createBasicBlock(name: D->getName()),
708 EHScopeStack::stable_iterator::invalid(),
709 NextCleanupDestIndex++);
710 return Dest;
711}
712
713void CodeGenFunction::EmitLabel(const LabelDecl *D) {
714 // Add this label to the current lexical scope if we're within any
715 // normal cleanups. Jumps "in" to this label --- when permitted by
716 // the language --- may need to be routed around such cleanups.
717 if (EHStack.hasNormalCleanups() && CurLexicalScope)
718 CurLexicalScope->addLabel(label: D);
719
720 JumpDest &Dest = LabelMap[D];
721
722 // If we didn't need a forward reference to this label, just go
723 // ahead and create a destination at the current scope.
724 if (!Dest.isValid()) {
725 Dest = getJumpDestInCurrentScope(Name: D->getName());
726
727 // Otherwise, we need to give this label a target depth and remove
728 // it from the branch-fixups list.
729 } else {
730 assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
731 Dest.setScopeDepth(EHStack.stable_begin());
732 ResolveBranchFixups(Target: Dest.getBlock());
733 }
734
735 EmitBlock(BB: Dest.getBlock());
736
737 // Emit debug info for labels.
738 if (CGDebugInfo *DI = getDebugInfo()) {
739 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
740 DI->setLocation(D->getLocation());
741 DI->EmitLabel(D, Builder);
742 }
743 }
744
745 incrementProfileCounter(S: D->getStmt());
746}
747
748/// Change the cleanup scope of the labels in this lexical scope to
749/// match the scope of the enclosing context.
750void CodeGenFunction::LexicalScope::rescopeLabels() {
751 assert(!Labels.empty());
752 EHScopeStack::stable_iterator innermostScope
753 = CGF.EHStack.getInnermostNormalCleanup();
754
755 // Change the scope depth of all the labels.
756 for (const LabelDecl *Label : Labels) {
757 assert(CGF.LabelMap.count(Label));
758 JumpDest &dest = CGF.LabelMap.find(Val: Label)->second;
759 assert(dest.getScopeDepth().isValid());
760 assert(innermostScope.encloses(dest.getScopeDepth()));
761 dest.setScopeDepth(innermostScope);
762 }
763
764 // Reparent the labels if the new scope also has cleanups.
765 if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
766 ParentScope->Labels.append(in_start: Labels.begin(), in_end: Labels.end());
767 }
768}
769
770
771void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
772 EmitLabel(D: S.getDecl());
773
774 // IsEHa - emit eha.scope.begin if it's a side entry of a scope
775 if (getLangOpts().EHAsynch && S.isSideEntry())
776 EmitSehCppScopeBegin();
777
778 EmitStmt(S: S.getSubStmt());
779}
780
781void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
782 bool nomerge = false;
783 bool noinline = false;
784 bool alwaysinline = false;
785 bool noconvergent = false;
786 HLSLControlFlowHintAttr::Spelling flattenOrBranch =
787 HLSLControlFlowHintAttr::SpellingNotCalculated;
788 const CallExpr *musttail = nullptr;
789 const AtomicAttr *AA = nullptr;
790
791 for (const auto *A : S.getAttrs()) {
792 switch (A->getKind()) {
793 default:
794 break;
795 case attr::NoMerge:
796 nomerge = true;
797 break;
798 case attr::NoInline:
799 noinline = true;
800 break;
801 case attr::AlwaysInline:
802 alwaysinline = true;
803 break;
804 case attr::NoConvergent:
805 noconvergent = true;
806 break;
807 case attr::MustTail: {
808 const Stmt *Sub = S.getSubStmt();
809 const ReturnStmt *R = cast<ReturnStmt>(Val: Sub);
810 musttail = cast<CallExpr>(Val: R->getRetValue()->IgnoreParens());
811 } break;
812 case attr::CXXAssume: {
813 const Expr *Assumption = cast<CXXAssumeAttr>(Val: A)->getAssumption();
814 if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() &&
815 !Assumption->HasSideEffects(Ctx: getContext())) {
816 llvm::Value *AssumptionVal = EmitCheckedArgForAssume(E: Assumption);
817 Builder.CreateAssumption(Cond: AssumptionVal);
818 }
819 } break;
820 case attr::Atomic:
821 AA = cast<AtomicAttr>(Val: A);
822 break;
823 case attr::HLSLControlFlowHint: {
824 flattenOrBranch = cast<HLSLControlFlowHintAttr>(Val: A)->getSemanticSpelling();
825 } break;
826 }
827 }
828 SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);
829 SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);
830 SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);
831 SaveAndRestore save_noconvergent(InNoConvergentAttributedStmt, noconvergent);
832 SaveAndRestore save_musttail(MustTailCall, musttail);
833 SaveAndRestore save_flattenOrBranch(HLSLControlFlowAttr, flattenOrBranch);
834 CGAtomicOptionsRAII AORAII(CGM, AA);
835 EmitStmt(S: S.getSubStmt(), Attrs: S.getAttrs());
836}
837
838void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
839 // If this code is reachable then emit a stop point (if generating
840 // debug info). We have to do this ourselves because we are on the
841 // "simple" statement path.
842 if (HaveInsertPoint())
843 EmitStopPoint(S: &S);
844
845 ApplyAtomGroup Grp(getDebugInfo());
846 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: S.getLabel()));
847}
848
849
850void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
851 ApplyAtomGroup Grp(getDebugInfo());
852 if (const LabelDecl *Target = S.getConstantTarget()) {
853 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: Target));
854 return;
855 }
856
857 // Ensure that we have an i8* for our PHI node.
858 llvm::Value *V = Builder.CreateBitCast(V: EmitScalarExpr(E: S.getTarget()),
859 DestTy: Int8PtrTy, Name: "addr");
860 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
861
862 // Get the basic block for the indirect goto.
863 llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
864
865 // The first instruction in the block has to be the PHI for the switch dest,
866 // add an entry for this branch.
867 cast<llvm::PHINode>(Val: IndGotoBB->begin())->addIncoming(V, BB: CurBB);
868
869 EmitBranch(Target: IndGotoBB);
870 if (CurBB && CurBB->hasTerminator())
871 addInstToCurrentSourceAtom(KeyInstruction: CurBB->getTerminator(), Backup: nullptr);
872}
873
874void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
875 const Stmt *Else = S.getElse();
876
877 // The else branch of a consteval if statement is always the only branch that
878 // can be runtime evaluated.
879 if (S.isConsteval()) {
880 const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;
881 if (Executed) {
882 RunCleanupsScope ExecutedScope(*this);
883 EmitStmt(S: Executed);
884 }
885 return;
886 }
887
888 // C99 6.8.4.1: The first substatement is executed if the expression compares
889 // unequal to 0. The condition must be a scalar type.
890 LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
891 ApplyDebugLocation DL(*this, S.getCond());
892
893 if (S.getInit()) {
894 EmitStmt(S: S.getInit());
895
896 // The init statement may have cleared the insertion point (e.g. it ended in
897 // a 'noreturn' call); the condition emitted below needs a valid one.
898 EnsureInsertPoint();
899 }
900
901 if (S.getConditionVariable())
902 EmitDecl(D: *S.getConditionVariable());
903
904 // If the condition constant folds and can be elided, try to avoid emitting
905 // the condition and the dead arm of the if/else.
906 bool CondConstant;
907 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: CondConstant,
908 AllowLabels: S.isConstexpr())) {
909 // Figure out which block (then or else) is executed.
910 const Stmt *Executed = S.getThen();
911 const Stmt *Skipped = Else;
912 if (!CondConstant) // Condition false?
913 std::swap(a&: Executed, b&: Skipped);
914
915 // If the skipped block has no labels in it, just emit the executed block.
916 // This avoids emitting dead code and simplifies the CFG substantially.
917 if (S.isConstexpr() || !ContainsLabel(S: Skipped)) {
918 incrementProfileCounter(ExecSkip: CondConstant ? UseExecPath : UseSkipPath, S: &S,
919 /*UseBoth=*/true);
920 if (Executed) {
921 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
922 RunCleanupsScope ExecutedScope(*this);
923 EmitStmt(S: Executed);
924 }
925 PGO->markStmtMaybeUsed(S: Skipped);
926 return;
927 }
928 }
929
930 auto HasSkip = hasSkipCounter(S: &S);
931
932 // Otherwise, the condition did not fold, or we couldn't elide it. Just emit
933 // the conditional branch.
934 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.then");
935 llvm::BasicBlock *ContBlock = createBasicBlock(name: "if.end");
936 llvm::BasicBlock *ElseBlock =
937 (Else || HasSkip ? createBasicBlock(name: "if.else") : ContBlock);
938 // Prefer the PGO based weights over the likelihood attribute.
939 // When the build isn't optimized the metadata isn't used, so don't generate
940 // it.
941 // Also, differentiate between disabled PGO and a never executed branch with
942 // PGO. Assuming PGO is in use:
943 // - we want to ignore the [[likely]] attribute if the branch is never
944 // executed,
945 // - assuming the profile is poor, preserving the attribute may still be
946 // beneficial.
947 // As an approximation, preserve the attribute only if both the branch and the
948 // parent context were not executed.
949 Stmt::Likelihood LH = Stmt::LH_None;
950 uint64_t ThenCount = getProfileCount(S: S.getThen());
951 if (!ThenCount && !getCurrentProfileCount() &&
952 CGM.getCodeGenOpts().OptimizationLevel)
953 LH = Stmt::getLikelihood(Then: S.getThen(), Else);
954
955 // When measuring MC/DC, always fully evaluate the condition up front using
956 // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
957 // executing the body of the if.then or if.else. This is useful for when
958 // there is a 'return' within the body, but this is particularly beneficial
959 // when one if-stmt is nested within another if-stmt so that all of the MC/DC
960 // updates are kept linear and consistent.
961 if (!CGM.getCodeGenOpts().MCDCCoverage) {
962 EmitBranchOnBoolExpr(Cond: S.getCond(), TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount: ThenCount, LH,
963 /*ConditionalOp=*/nullptr,
964 /*ConditionalDecl=*/S.getConditionVariable());
965 } else {
966 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
967 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
968 Builder.CreateCondBr(Cond: BoolCondVal, True: ThenBlock, False: ElseBlock);
969 }
970
971 // Emit the 'then' code.
972 EmitBlock(BB: ThenBlock);
973 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
974 {
975 RunCleanupsScope ThenScope(*this);
976 EmitStmt(S: S.getThen());
977 }
978 EmitBranch(Target: ContBlock);
979
980 // Emit the 'else' code if present.
981 if (Else) {
982 {
983 // There is no need to emit line number for an unconditional branch.
984 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
985 EmitBlock(BB: ElseBlock);
986 }
987 // Add a counter to else block unless it has CounterExpr.
988 if (HasSkip)
989 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
990 {
991 RunCleanupsScope ElseScope(*this);
992 EmitStmt(S: Else);
993 }
994 {
995 // There is no need to emit line number for an unconditional branch.
996 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
997 EmitBranch(Target: ContBlock);
998 }
999 } else if (HasSkip) {
1000 EmitBlock(BB: ElseBlock);
1001 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1002 EmitBranch(Target: ContBlock);
1003 }
1004
1005 // Emit the continuation block for code after the if.
1006 EmitBlock(BB: ContBlock, IsFinished: true);
1007}
1008
1009bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression,
1010 bool HasEmptyBody) {
1011 if (CGM.getCodeGenOpts().getFiniteLoops() ==
1012 CodeGenOptions::FiniteLoopsKind::Never)
1013 return false;
1014
1015 // Now apply rules for plain C (see 6.8.5.6 in C11).
1016 // Loops with constant conditions do not have to make progress in any C
1017 // version.
1018 // As an extension, we consisider loops whose constant expression
1019 // can be constant-folded.
1020 Expr::EvalResult Result;
1021 bool CondIsConstInt =
1022 !ControllingExpression ||
1023 (ControllingExpression->EvaluateAsInt(Result, Ctx: getContext()) &&
1024 Result.Val.isInt());
1025
1026 bool CondIsTrue = CondIsConstInt && (!ControllingExpression ||
1027 Result.Val.getInt().getBoolValue());
1028
1029 // Loops with non-constant conditions must make progress in C11 and later.
1030 if (getLangOpts().C11 && !CondIsConstInt)
1031 return true;
1032
1033 // [C++26][intro.progress] (DR)
1034 // The implementation may assume that any thread will eventually do one of the
1035 // following:
1036 // [...]
1037 // - continue execution of a trivial infinite loop ([stmt.iter.general]).
1038 if (CGM.getCodeGenOpts().getFiniteLoops() ==
1039 CodeGenOptions::FiniteLoopsKind::Always ||
1040 getLangOpts().CPlusPlus11) {
1041 if (HasEmptyBody && CondIsTrue) {
1042 CurFn->removeFnAttr(Kind: llvm::Attribute::MustProgress);
1043 return false;
1044 }
1045 return true;
1046 }
1047 return false;
1048}
1049
1050// [C++26][stmt.iter.general] (DR)
1051// A trivially empty iteration statement is an iteration statement matching one
1052// of the following forms:
1053// - while ( expression ) ;
1054// - while ( expression ) { }
1055// - do ; while ( expression ) ;
1056// - do { } while ( expression ) ;
1057// - for ( init-statement expression(opt); ) ;
1058// - for ( init-statement expression(opt); ) { }
1059template <typename LoopStmt> static bool hasEmptyLoopBody(const LoopStmt &S) {
1060 if constexpr (std::is_same_v<LoopStmt, ForStmt>) {
1061 if (S.getInc())
1062 return false;
1063 }
1064 const Stmt *Body = S.getBody();
1065 if (!Body || isa<NullStmt>(Val: Body))
1066 return true;
1067 if (const CompoundStmt *Compound = dyn_cast<CompoundStmt>(Val: Body))
1068 return Compound->body_empty();
1069 return false;
1070}
1071
1072void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
1073 ArrayRef<const Attr *> WhileAttrs) {
1074 // Emit the header for the loop, which will also become
1075 // the continue target.
1076 JumpDest LoopHeader = getJumpDestInCurrentScope(Name: "while.cond");
1077 EmitBlock(BB: LoopHeader.getBlock());
1078
1079 if (CGM.shouldEmitConvergenceTokens())
1080 ConvergenceTokenStack.push_back(
1081 Elt: emitConvergenceLoopToken(BB: LoopHeader.getBlock()));
1082
1083 // Create an exit block for when the condition fails, which will
1084 // also become the break target.
1085 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "while.end");
1086
1087 // Store the blocks to use for break and continue.
1088 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopHeader));
1089
1090 // C++ [stmt.while]p2:
1091 // When the condition of a while statement is a declaration, the
1092 // scope of the variable that is declared extends from its point
1093 // of declaration (3.3.2) to the end of the while statement.
1094 // [...]
1095 // The object created in a condition is destroyed and created
1096 // with each iteration of the loop.
1097 RunCleanupsScope ConditionScope(*this);
1098
1099 if (S.getConditionVariable())
1100 EmitDecl(D: *S.getConditionVariable());
1101
1102 // Evaluate the conditional in the while header. C99 6.8.5.1: The
1103 // evaluation of the controlling expression takes place before each
1104 // execution of the loop body.
1105 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1106
1107 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1108
1109 // while(1) is common, avoid extra exit blocks. Be sure
1110 // to correctly handle break/continue though.
1111 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1112 bool EmitBoolCondBranch = !C || !C->isOne();
1113 const SourceRange &R = S.getSourceRange();
1114 LoopStack.push(Header: LoopHeader.getBlock(), Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
1115 Attrs: WhileAttrs, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1116 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1117 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1118
1119 // As long as the condition is true, go to the loop body.
1120 llvm::BasicBlock *LoopBody = createBasicBlock(name: "while.body");
1121 if (EmitBoolCondBranch) {
1122 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1123 if (hasSkipCounter(S: &S) || ConditionScope.requiresCleanups())
1124 ExitBlock = createBasicBlock(name: "while.exit");
1125 llvm::MDNode *Weights =
1126 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1127 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1128 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1129 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1130 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock, BranchWeights: Weights);
1131 // Key Instructions: Emit the condition and branch as separate source
1132 // location atoms otherwise we may omit a step onto the loop condition in
1133 // favour of the `while` keyword.
1134 // FIXME: We could have the branch as the backup location for the condition,
1135 // which would probably be a better experience. Explore this later.
1136 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1137 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1138 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1139
1140 if (ExitBlock != LoopExit.getBlock()) {
1141 EmitBlock(BB: ExitBlock);
1142 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1143 EmitBranchThroughCleanup(Dest: LoopExit);
1144 }
1145 } else if (const Attr *A = Stmt::getLikelihoodAttr(S: S.getBody())) {
1146 CGM.getDiags().Report(Loc: A->getLocation(),
1147 DiagID: diag::warn_attribute_has_no_effect_on_infinite_loop)
1148 << A << A->getRange();
1149 CGM.getDiags().Report(
1150 Loc: S.getWhileLoc(),
1151 DiagID: diag::note_attribute_has_no_effect_on_infinite_loop_here)
1152 << SourceRange(S.getWhileLoc(), S.getRParenLoc());
1153 }
1154
1155 // Emit the loop body. We have to emit this in a cleanup scope
1156 // because it might be a singleton DeclStmt.
1157 {
1158 RunCleanupsScope BodyScope(*this);
1159 EmitBlock(BB: LoopBody);
1160 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1161 EmitStmt(S: S.getBody());
1162 }
1163
1164 BreakContinueStack.pop_back();
1165
1166 // Immediately force cleanup.
1167 ConditionScope.ForceCleanup();
1168
1169 EmitStopPoint(S: &S);
1170 // Branch to the loop header again.
1171 EmitBranch(Target: LoopHeader.getBlock());
1172
1173 LoopStack.pop();
1174
1175 // Emit the exit block.
1176 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1177
1178 // The LoopHeader typically is just a branch if we skipped emitting
1179 // a branch, try to erase it.
1180 if (!EmitBoolCondBranch) {
1181 SimplifyForwardingBlocks(BB: LoopHeader.getBlock());
1182 PGO->markStmtAsUsed(Skipped: true, S: &S);
1183 }
1184
1185 if (CGM.shouldEmitConvergenceTokens())
1186 ConvergenceTokenStack.pop_back();
1187}
1188
1189void CodeGenFunction::EmitDoStmt(const DoStmt &S,
1190 ArrayRef<const Attr *> DoAttrs) {
1191 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "do.end");
1192 JumpDest LoopCond = getJumpDestInCurrentScope(Name: "do.cond");
1193
1194 uint64_t ParentCount = getCurrentProfileCount();
1195
1196 // Store the blocks to use for break and continue.
1197 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopCond));
1198
1199 // Emit the body of the loop.
1200 llvm::BasicBlock *LoopBody = createBasicBlock(name: "do.body");
1201
1202 EmitBlockWithFallThrough(BB: LoopBody, S: &S);
1203
1204 if (CGM.shouldEmitConvergenceTokens())
1205 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: LoopBody));
1206
1207 {
1208 RunCleanupsScope BodyScope(*this);
1209 EmitStmt(S: S.getBody());
1210 }
1211
1212 EmitBlock(BB: LoopCond.getBlock());
1213
1214 // C99 6.8.5.2: "The evaluation of the controlling expression takes place
1215 // after each execution of the loop body."
1216
1217 // Evaluate the conditional in the while header.
1218 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1219 // compares unequal to 0. The condition must be a scalar type.
1220 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1221
1222 BreakContinueStack.pop_back();
1223
1224 // "do {} while (0)" is common in macros, avoid extra blocks. Be sure
1225 // to correctly handle break/continue though.
1226 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1227 bool EmitBoolCondBranch = !C || !C->isZero();
1228
1229 const SourceRange &R = S.getSourceRange();
1230 LoopStack.push(Header: LoopBody, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: DoAttrs,
1231 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1232 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1233 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1234
1235 auto *LoopFalse = (hasSkipCounter(S: &S) ? createBasicBlock(name: "do.loopfalse")
1236 : LoopExit.getBlock());
1237
1238 // As long as the condition is true, iterate the loop.
1239 if (EmitBoolCondBranch) {
1240 uint64_t BackedgeCount = getProfileCount(S: S.getBody()) - ParentCount;
1241 auto *I = Builder.CreateCondBr(
1242 Cond: BoolCondVal, True: LoopBody, False: LoopFalse,
1243 BranchWeights: createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: BackedgeCount));
1244
1245 // Key Instructions: Emit the condition and branch as separate source
1246 // location atoms otherwise we may omit a step onto the loop condition in
1247 // favour of the closing brace.
1248 // FIXME: We could have the branch as the backup location for the condition,
1249 // which would probably be a better experience (no jumping to the brace).
1250 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1251 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1252 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1253 }
1254
1255 LoopStack.pop();
1256
1257 if (LoopFalse != LoopExit.getBlock()) {
1258 EmitBlock(BB: LoopFalse);
1259 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S, /*UseBoth=*/true);
1260 }
1261
1262 // Emit the exit block.
1263 EmitBlock(BB: LoopExit.getBlock());
1264
1265 // The DoCond block typically is just a branch if we skipped
1266 // emitting a branch, try to erase it.
1267 if (!EmitBoolCondBranch)
1268 SimplifyForwardingBlocks(BB: LoopCond.getBlock());
1269
1270 if (CGM.shouldEmitConvergenceTokens())
1271 ConvergenceTokenStack.pop_back();
1272}
1273
1274void CodeGenFunction::EmitForStmt(const ForStmt &S,
1275 ArrayRef<const Attr *> ForAttrs) {
1276 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1277
1278 std::optional<LexicalScope> ForScope;
1279 if (getLangOpts().C99 || getLangOpts().CPlusPlus)
1280 ForScope.emplace(args&: *this, args: S.getSourceRange());
1281
1282 // Evaluate the first part before the loop.
1283 if (S.getInit())
1284 EmitStmt(S: S.getInit());
1285
1286 // Start the loop with a block that tests the condition.
1287 // If there's an increment, the continue scope will be overwritten
1288 // later.
1289 JumpDest CondDest = getJumpDestInCurrentScope(Name: "for.cond");
1290 llvm::BasicBlock *CondBlock = CondDest.getBlock();
1291 EmitBlock(BB: CondBlock);
1292
1293 if (CGM.shouldEmitConvergenceTokens())
1294 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1295
1296 const SourceRange &R = S.getSourceRange();
1297 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1298 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1299 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1300 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1301
1302 // Create a cleanup scope for the condition variable cleanups.
1303 LexicalScope ConditionScope(*this, S.getSourceRange());
1304
1305 // If the for loop doesn't have an increment we can just use the condition as
1306 // the continue block. Otherwise, if there is no condition variable, we can
1307 // form the continue block now. If there is a condition variable, we can't
1308 // form the continue block until after we've emitted the condition, because
1309 // the condition is in scope in the increment, but Sema's jump diagnostics
1310 // ensure that there are no continues from the condition variable that jump
1311 // to the loop increment.
1312 JumpDest Continue;
1313 if (!S.getInc())
1314 Continue = CondDest;
1315 else if (!S.getConditionVariable())
1316 Continue = getJumpDestInCurrentScope(Name: "for.inc");
1317 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1318
1319 if (S.getCond()) {
1320 // If the for statement has a condition scope, emit the local variable
1321 // declaration.
1322 if (S.getConditionVariable()) {
1323 EmitDecl(D: *S.getConditionVariable());
1324
1325 // We have entered the condition variable's scope, so we're now able to
1326 // jump to the continue block.
1327 Continue = S.getInc() ? getJumpDestInCurrentScope(Name: "for.inc") : CondDest;
1328 BreakContinueStack.back().ContinueBlock = Continue;
1329 }
1330
1331 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1332 // If there are any cleanups between here and the loop-exit scope,
1333 // create a block to stage a loop exit along.
1334 if (hasSkipCounter(S: &S) || (ForScope && ForScope->requiresCleanups()))
1335 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1336
1337 // As long as the condition is true, iterate the loop.
1338 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1339
1340 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1341 // compares unequal to 0. The condition must be a scalar type.
1342 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1343
1344 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1345
1346 llvm::MDNode *Weights =
1347 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1348 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1349 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1350 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1351
1352 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1353 // Key Instructions: Emit the condition and branch as separate atoms to
1354 // match existing loop stepping behaviour. FIXME: We could have the branch
1355 // as the backup location for the condition, which would probably be a
1356 // better experience (no jumping to the brace).
1357 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1358 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1359 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1360
1361 if (ExitBlock != LoopExit.getBlock()) {
1362 EmitBlock(BB: ExitBlock);
1363 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1364 EmitBranchThroughCleanup(Dest: LoopExit);
1365 }
1366
1367 EmitBlock(BB: ForBody);
1368 } else {
1369 // Treat it as a non-zero constant. Don't even create a new block for the
1370 // body, just fall into it.
1371 PGO->markStmtAsUsed(Skipped: true, S: &S);
1372 }
1373
1374 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1375
1376 {
1377 // Create a separate cleanup scope for the body, in case it is not
1378 // a compound statement.
1379 RunCleanupsScope BodyScope(*this);
1380 EmitStmt(S: S.getBody());
1381 }
1382
1383 // The last block in the loop's body (which unconditionally branches to the
1384 // `inc` block if there is one).
1385 auto *FinalBodyBB = Builder.GetInsertBlock();
1386
1387 // If there is an increment, emit it next.
1388 if (S.getInc()) {
1389 EmitBlock(BB: Continue.getBlock());
1390 EmitStmt(S: S.getInc());
1391 }
1392
1393 BreakContinueStack.pop_back();
1394
1395 ConditionScope.ForceCleanup();
1396
1397 EmitStopPoint(S: &S);
1398 EmitBranch(Target: CondBlock);
1399
1400 if (ForScope)
1401 ForScope->ForceCleanup();
1402
1403 LoopStack.pop();
1404
1405 // Emit the fall-through block.
1406 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1407
1408 if (CGM.shouldEmitConvergenceTokens())
1409 ConvergenceTokenStack.pop_back();
1410
1411 if (FinalBodyBB) {
1412 // Key Instructions: We want the for closing brace to be step-able on to
1413 // match existing behaviour.
1414 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1415 }
1416}
1417
1418void
1419CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1420 ArrayRef<const Attr *> ForAttrs) {
1421 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1422
1423 LexicalScope ForScope(*this, S.getSourceRange());
1424
1425 // Evaluate the first pieces before the loop.
1426 if (S.getInit())
1427 EmitStmt(S: S.getInit());
1428 EmitStmt(S: S.getRangeStmt());
1429 EmitStmt(S: S.getBeginStmt());
1430 EmitStmt(S: S.getEndStmt());
1431
1432 // Start the loop with a block that tests the condition.
1433 // If there's an increment, the continue scope will be overwritten
1434 // later.
1435 llvm::BasicBlock *CondBlock = createBasicBlock(name: "for.cond");
1436 EmitBlock(BB: CondBlock);
1437
1438 if (CGM.shouldEmitConvergenceTokens())
1439 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1440
1441 const SourceRange &R = S.getSourceRange();
1442 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1443 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1444 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
1445
1446 // If there are any cleanups between here and the loop-exit scope,
1447 // create a block to stage a loop exit along.
1448 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1449 if (hasSkipCounter(S: &S) || ForScope.requiresCleanups())
1450 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1451
1452 // The loop body, consisting of the specified body and the loop variable.
1453 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1454
1455 // The body is executed if the expression, contextually converted
1456 // to bool, is true.
1457 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1458 llvm::MDNode *Weights =
1459 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1460 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1461 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1462 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1463 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1464 // Key Instructions: Emit the condition and branch as separate atoms to
1465 // match existing loop stepping behaviour. FIXME: We could have the branch as
1466 // the backup location for the condition, which would probably be a better
1467 // experience.
1468 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1469 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1470 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1471
1472 if (ExitBlock != LoopExit.getBlock()) {
1473 EmitBlock(BB: ExitBlock);
1474 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1475 EmitBranchThroughCleanup(Dest: LoopExit);
1476 }
1477
1478 EmitBlock(BB: ForBody);
1479 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1480
1481 // Create a block for the increment. In case of a 'continue', we jump there.
1482 JumpDest Continue = getJumpDestInCurrentScope(Name: "for.inc");
1483
1484 // Store the blocks to use for break and continue.
1485 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1486
1487 {
1488 // Create a separate cleanup scope for the loop variable and body.
1489 LexicalScope BodyScope(*this, S.getSourceRange());
1490 EmitStmt(S: S.getLoopVarStmt());
1491 EmitStmt(S: S.getBody());
1492 }
1493 // The last block in the loop's body (which unconditionally branches to the
1494 // `inc` block if there is one).
1495 auto *FinalBodyBB = Builder.GetInsertBlock();
1496
1497 EmitStopPoint(S: &S);
1498 // If there is an increment, emit it next.
1499 EmitBlock(BB: Continue.getBlock());
1500 EmitStmt(S: S.getInc());
1501
1502 BreakContinueStack.pop_back();
1503
1504 EmitBranch(Target: CondBlock);
1505
1506 ForScope.ForceCleanup();
1507
1508 LoopStack.pop();
1509
1510 // Emit the fall-through block.
1511 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1512
1513 if (CGM.shouldEmitConvergenceTokens())
1514 ConvergenceTokenStack.pop_back();
1515
1516 if (FinalBodyBB) {
1517 // We want the for closing brace to be step-able on to match existing
1518 // behaviour.
1519 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1520 }
1521}
1522
1523void CodeGenFunction::EmitCXXExpansionStmtInstantiation(
1524 const CXXExpansionStmtInstantiation &S) {
1525 LexicalScope Scope(*this, S.getSourceRange());
1526
1527 for (const Stmt *DS : S.getPreambleStmts())
1528 EmitStmt(S: DS);
1529
1530 if (S.getInstantiations().empty())
1531 return;
1532
1533 JumpDest ExpandExit = getJumpDestInCurrentScope(Name: "expand.end");
1534 JumpDest ContinueDest;
1535 for (auto [N, Inst] : enumerate(First: S.getInstantiations())) {
1536 if (N == S.getInstantiations().size() - 1)
1537 ContinueDest = ExpandExit;
1538 else
1539 ContinueDest = getJumpDestInCurrentScope(Name: "expand.next");
1540
1541 LexicalScope ExpansionScope(*this, Inst->getSourceRange());
1542 BreakContinueStack.push_back(Elt: BreakContinue(S, ExpandExit, ContinueDest));
1543 EmitStmt(S: Inst);
1544 BreakContinueStack.pop_back();
1545 EmitBlock(BB: ContinueDest.getBlock(), IsFinished: true);
1546 }
1547}
1548
1549void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1550 if (RV.isScalar()) {
1551 Builder.CreateStore(Val: RV.getScalarVal(), Addr: ReturnValue);
1552 } else if (RV.isAggregate()) {
1553 LValue Dest = MakeAddrLValue(Addr: ReturnValue, T: Ty);
1554 LValue Src = MakeAddrLValue(Addr: RV.getAggregateAddress(), T: Ty);
1555 EmitAggregateCopy(Dest, Src, EltTy: Ty, MayOverlap: getOverlapForReturnValue());
1556 } else {
1557 EmitStoreOfComplex(V: RV.getComplexVal(), dest: MakeAddrLValue(Addr: ReturnValue, T: Ty),
1558 /*init*/ isInit: true);
1559 }
1560 EmitBranchThroughCleanup(Dest: ReturnBlock);
1561}
1562
1563namespace {
1564// RAII struct used to save and restore a return statment's result expression.
1565struct SaveRetExprRAII {
1566 SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1567 : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1568 CGF.RetExpr = RetExpr;
1569 }
1570 ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1571 const Expr *OldRetExpr;
1572 CodeGenFunction &CGF;
1573};
1574} // namespace
1575
1576/// Determine if the given call uses the swiftasync calling convention.
1577static bool isSwiftAsyncCallee(const CallExpr *CE) {
1578 auto calleeQualType = CE->getCallee()->getType();
1579 const FunctionType *calleeType = nullptr;
1580 if (calleeQualType->isFunctionPointerType() ||
1581 calleeQualType->isFunctionReferenceType() ||
1582 calleeQualType->isBlockPointerType() ||
1583 calleeQualType->isMemberFunctionPointerType()) {
1584 calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1585 } else if (auto *ty = dyn_cast<FunctionType>(Val&: calleeQualType)) {
1586 calleeType = ty;
1587 } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(Val: CE)) {
1588 if (auto methodDecl = CMCE->getMethodDecl()) {
1589 // getMethodDecl() doesn't handle member pointers at the moment.
1590 calleeType = methodDecl->getType()->castAs<FunctionType>();
1591 } else {
1592 return false;
1593 }
1594 } else {
1595 return false;
1596 }
1597 return calleeType->getCallConv() == CallingConv::CC_SwiftAsync;
1598}
1599
1600/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1601/// if the function returns void, or may be missing one if the function returns
1602/// non-void. Fun stuff :).
1603void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1604 ApplyAtomGroup Grp(getDebugInfo());
1605 if (requiresReturnValueCheck()) {
1606 llvm::Constant *SLoc = EmitCheckSourceLocation(Loc: S.getBeginLoc());
1607 auto *SLocPtr =
1608 new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1609 llvm::GlobalVariable::PrivateLinkage, SLoc);
1610 SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1611 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: SLocPtr);
1612 assert(ReturnLocation.isValid() && "No valid return location");
1613 Builder.CreateStore(Val: SLocPtr, Addr: ReturnLocation);
1614 }
1615
1616 // Returning from an outlined SEH helper is UB, and we already warn on it.
1617 if (IsOutlinedSEHHelper) {
1618 Builder.CreateUnreachable();
1619 Builder.ClearInsertionPoint();
1620 }
1621
1622 // Emit the result value, even if unused, to evaluate the side effects.
1623 const Expr *RV = S.getRetValue();
1624
1625 // Record the result expression of the return statement. The recorded
1626 // expression is used to determine whether a block capture's lifetime should
1627 // end at the end of the full expression as opposed to the end of the scope
1628 // enclosing the block expression.
1629 //
1630 // This permits a small, easily-implemented exception to our over-conservative
1631 // rules about not jumping to statements following block literals with
1632 // non-trivial cleanups.
1633 SaveRetExprRAII SaveRetExpr(RV, *this);
1634
1635 RunCleanupsScope cleanupScope(*this);
1636 if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(Val: RV))
1637 RV = EWC->getSubExpr();
1638
1639 // If we're in a swiftasynccall function, and the return expression is a
1640 // call to a swiftasynccall function, mark the call as the musttail call.
1641 std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail;
1642 if (RV && CurFnInfo &&
1643 CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) {
1644 if (auto CE = dyn_cast<CallExpr>(Val: RV)) {
1645 if (isSwiftAsyncCallee(CE)) {
1646 SaveMustTail.emplace(args&: MustTailCall, args&: CE);
1647 }
1648 }
1649 }
1650
1651 // FIXME: Clean this up by using an LValue for ReturnTemp,
1652 // EmitStoreThroughLValue, and EmitAnyExpr.
1653 // Check if the NRVO candidate was not globalized in OpenMP mode.
1654 if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1655 S.getNRVOCandidate()->isNRVOVariable() &&
1656 (!getLangOpts().OpenMP ||
1657 !CGM.getOpenMPRuntime()
1658 .getAddressOfLocalVariable(CGF&: *this, VD: S.getNRVOCandidate())
1659 .isValid())) {
1660 // Apply the named return value optimization for this return statement,
1661 // which means doing nothing: the appropriate result has already been
1662 // constructed into the NRVO variable.
1663
1664 // If there is an NRVO flag for this variable, set it to 1 into indicate
1665 // that the cleanup code should not destroy the variable.
1666 if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1667 Builder.CreateFlagStore(Value: Builder.getTrue(), Addr: NRVOFlag);
1668 } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1669 // Make sure not to return anything, but evaluate the expression
1670 // for side effects.
1671 if (RV) {
1672 EmitAnyExpr(E: RV);
1673 }
1674 } else if (!RV) {
1675 // Do nothing (return value is left uninitialized)
1676 } else if (FnRetTy->isReferenceType()) {
1677 // If this function returns a reference, take the address of the expression
1678 // rather than the value.
1679 RValue Result = EmitReferenceBindingToExpr(E: RV);
1680 auto *I = Builder.CreateStore(Val: Result.getScalarVal(), Addr: ReturnValue);
1681 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1682 } else {
1683 switch (getEvaluationKind(T: RV->getType())) {
1684 case TEK_Scalar: {
1685 llvm::Value *Ret = EmitScalarExpr(E: RV);
1686 if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1687 EmitStoreOfScalar(value: Ret, lvalue: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1688 /*isInit*/ true);
1689 } else {
1690 auto *I = Builder.CreateStore(Val: Ret, Addr: ReturnValue);
1691 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1692 }
1693 break;
1694 }
1695 case TEK_Complex:
1696 EmitComplexExprIntoLValue(E: RV, dest: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1697 /*isInit*/ true);
1698 break;
1699 case TEK_Aggregate:
1700 EmitAggExpr(E: RV, AS: AggValueSlot::forAddr(
1701 addr: ReturnValue, quals: Qualifiers(),
1702 isDestructed: AggValueSlot::IsDestructed,
1703 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1704 isAliased: AggValueSlot::IsNotAliased,
1705 mayOverlap: getOverlapForReturnValue()));
1706 break;
1707 }
1708 }
1709
1710 ++NumReturnExprs;
1711 if (!RV || RV->isEvaluatable(Ctx: getContext()))
1712 ++NumSimpleReturnExprs;
1713
1714 cleanupScope.ForceCleanup();
1715 EmitBranchThroughCleanup(Dest: ReturnBlock);
1716}
1717
1718void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1719 // As long as debug info is modeled with instructions, we have to ensure we
1720 // have a place to insert here and write the stop point here.
1721 if (HaveInsertPoint())
1722 EmitStopPoint(S: &S);
1723
1724 for (const auto *I : S.decls())
1725 EmitDecl(D: *I, /*EvaluateConditionDecl=*/true);
1726}
1727
1728auto CodeGenFunction::GetDestForLoopControlStmt(const LoopControlStmt &S)
1729 -> const BreakContinue * {
1730 if (!S.hasLabelTarget())
1731 return &BreakContinueStack.back();
1732
1733 const Stmt *LoopOrSwitch = S.getNamedLoopOrSwitch();
1734 assert(LoopOrSwitch && "break/continue target not set?");
1735 for (const BreakContinue &BC : llvm::reverse(C&: BreakContinueStack))
1736 if (BC.LoopOrSwitch == LoopOrSwitch)
1737 return &BC;
1738
1739 llvm_unreachable("break/continue target not found");
1740}
1741
1742void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1743 assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1744
1745 // If this code is reachable then emit a stop point (if generating
1746 // debug info). We have to do this ourselves because we are on the
1747 // "simple" statement path.
1748 if (HaveInsertPoint())
1749 EmitStopPoint(S: &S);
1750
1751 ApplyAtomGroup Grp(getDebugInfo());
1752 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->BreakBlock);
1753}
1754
1755void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1756 assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1757
1758 // If this code is reachable then emit a stop point (if generating
1759 // debug info). We have to do this ourselves because we are on the
1760 // "simple" statement path.
1761 if (HaveInsertPoint())
1762 EmitStopPoint(S: &S);
1763
1764 ApplyAtomGroup Grp(getDebugInfo());
1765 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->ContinueBlock);
1766}
1767
1768/// EmitCaseStmtRange - If case statement range is not too big then
1769/// add multiple cases to switch instruction, one for each value within
1770/// the range. If range is too big then emit "if" condition check.
1771void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1772 ArrayRef<const Attr *> Attrs) {
1773 assert(S.getRHS() && "Expected RHS value in CaseStmt");
1774
1775 llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(Ctx: getContext());
1776 llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(Ctx: getContext());
1777
1778 // Emit the code for this case. We do this first to make sure it is
1779 // properly chained from our predecessor before generating the
1780 // switch machinery to enter this block.
1781 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1782 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1783 EmitStmt(S: S.getSubStmt());
1784
1785 // If range is empty, do nothing.
1786 if (LHS.isSigned() ? RHS.slt(RHS: LHS) : RHS.ult(RHS: LHS))
1787 return;
1788
1789 Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1790 llvm::APInt Range = RHS - LHS;
1791 // FIXME: parameters such as this should not be hardcoded.
1792 if (Range.ult(RHS: llvm::APInt(Range.getBitWidth(), 64))) {
1793 // Range is small enough to add multiple switch instruction cases.
1794 uint64_t Total = getProfileCount(S: &S);
1795 unsigned NCases = Range.getZExtValue() + 1;
1796 // We only have one region counter for the entire set of cases here, so we
1797 // need to divide the weights evenly between the generated cases, ensuring
1798 // that the total weight is preserved. E.g., a weight of 5 over three cases
1799 // will be distributed as weights of 2, 2, and 1.
1800 uint64_t Weight = Total / NCases, Rem = Total % NCases;
1801 for (unsigned I = 0; I != NCases; ++I) {
1802 if (SwitchWeights)
1803 SwitchWeights->push_back(Elt: Weight + (Rem ? 1 : 0));
1804 else if (SwitchLikelihood)
1805 SwitchLikelihood->push_back(Elt: LH);
1806
1807 if (Rem)
1808 Rem--;
1809 SwitchInsn->addCase(OnVal: Builder.getInt(AI: LHS), Dest: CaseDest);
1810 ++LHS;
1811 }
1812 return;
1813 }
1814
1815 // The range is too big. Emit "if" condition into a new block,
1816 // making sure to save and restore the current insertion point.
1817 llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1818
1819 // Push this test onto the chain of range checks (which terminates
1820 // in the default basic block). The switch's default will be changed
1821 // to the top of this chain after switch emission is complete.
1822 llvm::BasicBlock *FalseDest = CaseRangeBlock;
1823 CaseRangeBlock = createBasicBlock(name: "sw.caserange");
1824
1825 CurFn->insert(Position: CurFn->end(), BB: CaseRangeBlock);
1826 Builder.SetInsertPoint(CaseRangeBlock);
1827
1828 // Emit range check.
1829 llvm::Value *Diff =
1830 Builder.CreateSub(LHS: SwitchInsn->getCondition(), RHS: Builder.getInt(AI: LHS));
1831 llvm::Value *Cond =
1832 Builder.CreateICmpULE(LHS: Diff, RHS: Builder.getInt(AI: Range), Name: "inbounds");
1833
1834 llvm::MDNode *Weights = nullptr;
1835 if (SwitchWeights) {
1836 uint64_t ThisCount = getProfileCount(S: &S);
1837 uint64_t DefaultCount = (*SwitchWeights)[0];
1838 Weights = createProfileWeights(TrueCount: ThisCount, FalseCount: DefaultCount);
1839
1840 // Since we're chaining the switch default through each large case range, we
1841 // need to update the weight for the default, ie, the first case, to include
1842 // this case.
1843 (*SwitchWeights)[0] += ThisCount;
1844 } else if (SwitchLikelihood)
1845 Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1846
1847 Builder.CreateCondBr(Cond, True: CaseDest, False: FalseDest, BranchWeights: Weights);
1848
1849 // Restore the appropriate insertion point.
1850 if (RestoreBB)
1851 Builder.SetInsertPoint(RestoreBB);
1852 else
1853 Builder.ClearInsertionPoint();
1854}
1855
1856void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1857 ArrayRef<const Attr *> Attrs) {
1858 // If there is no enclosing switch instance that we're aware of, then this
1859 // case statement and its block can be elided. This situation only happens
1860 // when we've constant-folded the switch, are emitting the constant case,
1861 // and part of the constant case includes another case statement. For
1862 // instance: switch (4) { case 4: do { case 5: } while (1); }
1863 if (!SwitchInsn) {
1864 EmitStmt(S: S.getSubStmt());
1865 return;
1866 }
1867
1868 // Handle case ranges.
1869 if (S.getRHS()) {
1870 EmitCaseStmtRange(S, Attrs);
1871 return;
1872 }
1873
1874 llvm::ConstantInt *CaseVal =
1875 Builder.getInt(AI: S.getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1876
1877 // Emit debuginfo for the case value if it is an enum value.
1878 const ConstantExpr *CE;
1879 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: S.getLHS()))
1880 CE = dyn_cast<ConstantExpr>(Val: ICE->getSubExpr());
1881 else
1882 CE = dyn_cast<ConstantExpr>(Val: S.getLHS());
1883 if (CE) {
1884 if (auto DE = dyn_cast<DeclRefExpr>(Val: CE->getSubExpr()))
1885 if (CGDebugInfo *Dbg = getDebugInfo())
1886 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
1887 Dbg->EmitGlobalVariable(VD: DE->getDecl(),
1888 Init: APValue(llvm::APSInt(CaseVal->getValue())));
1889 }
1890
1891 if (SwitchLikelihood)
1892 SwitchLikelihood->push_back(Elt: Stmt::getLikelihood(Attrs));
1893
1894 // If the body of the case is just a 'break', try to not emit an empty block.
1895 // If we're profiling or we're not optimizing, leave the block in for better
1896 // debug and coverage analysis.
1897 if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1898 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1899 isa<BreakStmt>(Val: S.getSubStmt())) {
1900 JumpDest Block = BreakContinueStack.back().BreakBlock;
1901
1902 // Only do this optimization if there are no cleanups that need emitting.
1903 if (isObviouslyBranchWithoutCleanups(Dest: Block)) {
1904 if (SwitchWeights)
1905 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1906 SwitchInsn->addCase(OnVal: CaseVal, Dest: Block.getBlock());
1907
1908 // If there was a fallthrough into this case, make sure to redirect it to
1909 // the end of the switch as well.
1910 if (Builder.GetInsertBlock()) {
1911 Builder.CreateBr(Dest: Block.getBlock());
1912 Builder.ClearInsertionPoint();
1913 }
1914 return;
1915 }
1916 }
1917
1918 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1919 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1920 if (SwitchWeights)
1921 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1922 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1923
1924 // Recursively emitting the statement is acceptable, but is not wonderful for
1925 // code where we have many case statements nested together, i.e.:
1926 // case 1:
1927 // case 2:
1928 // case 3: etc.
1929 // Handling this recursively will create a new block for each case statement
1930 // that falls through to the next case which is IR intensive. It also causes
1931 // deep recursion which can run into stack depth limitations. Handle
1932 // sequential non-range case statements specially.
1933 //
1934 // TODO When the next case has a likelihood attribute the code returns to the
1935 // recursive algorithm. Maybe improve this case if it becomes common practice
1936 // to use a lot of attributes.
1937 const CaseStmt *CurCase = &S;
1938 const CaseStmt *NextCase = dyn_cast<CaseStmt>(Val: S.getSubStmt());
1939
1940 // Otherwise, iteratively add consecutive cases to this switch stmt.
1941 while (NextCase && NextCase->getRHS() == nullptr) {
1942 CurCase = NextCase;
1943 llvm::ConstantInt *CaseVal =
1944 Builder.getInt(AI: CurCase->getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1945
1946 if (SwitchWeights)
1947 SwitchWeights->push_back(Elt: getProfileCount(S: NextCase));
1948 if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1949 CaseDest = createBasicBlock(name: "sw.bb");
1950 EmitBlockWithFallThrough(BB: CaseDest, S: CurCase);
1951 }
1952 // Since this loop is only executed when the CaseStmt has no attributes
1953 // use a hard-coded value.
1954 if (SwitchLikelihood)
1955 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
1956
1957 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1958 NextCase = dyn_cast<CaseStmt>(Val: CurCase->getSubStmt());
1959 }
1960
1961 // Generate a stop point for debug info if the case statement is
1962 // followed by a default statement. A fallthrough case before a
1963 // default case gets its own branch target.
1964 if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1965 EmitStopPoint(S: CurCase);
1966
1967 // Normal default recursion for non-cases.
1968 EmitStmt(S: CurCase->getSubStmt());
1969}
1970
1971void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1972 ArrayRef<const Attr *> Attrs) {
1973 // If there is no enclosing switch instance that we're aware of, then this
1974 // default statement can be elided. This situation only happens when we've
1975 // constant-folded the switch.
1976 if (!SwitchInsn) {
1977 EmitStmt(S: S.getSubStmt());
1978 return;
1979 }
1980
1981 llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1982 assert(DefaultBlock->empty() &&
1983 "EmitDefaultStmt: Default block already defined?");
1984
1985 if (SwitchLikelihood)
1986 SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1987
1988 EmitBlockWithFallThrough(BB: DefaultBlock, S: &S);
1989
1990 EmitStmt(S: S.getSubStmt());
1991}
1992
1993namespace {
1994struct EmitDeferredStatement final : EHScopeStack::Cleanup {
1995 const DeferStmt &Stmt;
1996 EmitDeferredStatement(const DeferStmt *Stmt) : Stmt(*Stmt) {}
1997
1998 void Emit(CodeGenFunction &CGF, Flags) override {
1999 // Take care that any cleanups pushed by the body of a '_Defer' statement
2000 // don't clobber the current cleanup slot value.
2001 //
2002 // Assume we have a scope that pushes a cleanup; when that scope is exited,
2003 // we need to run that cleanup; this is accomplished by emitting the cleanup
2004 // into a separate block and then branching to that block at scope exit.
2005 //
2006 // Where this gets complicated is if we exit the scope in multiple different
2007 // ways; e.g. in a 'for' loop, we may exit the scope of its body by falling
2008 // off the end (in which case we need to run the cleanup and then branch to
2009 // the increment), or by 'break'ing out of the loop (in which case we need
2010 // to run the cleanup and then branch to the loop exit block); in both cases
2011 // we first branch to the cleanup block to run the cleanup, but the block we
2012 // need to jump to *after* running the cleanup is different.
2013 //
2014 // This is accomplished using a local integer variable called the 'cleanup
2015 // slot': before branching to the cleanup block, we store a value into that
2016 // slot. Then, in the cleanup block, after running the cleanup, we load the
2017 // value of that variable and 'switch' on it to branch to the appropriate
2018 // continuation block.
2019 //
2020 // The problem that arises once '_Defer' statements are involved is that the
2021 // body of a '_Defer' is an arbitrary statement which itself can create more
2022 // cleanups. This means we may end up overwriting the cleanup slot before we
2023 // ever have a chance to 'switch' on it, which means that once we *do* get
2024 // to the 'switch', we end up in whatever block the cleanup code happened to
2025 // pick as the default 'switch' exit label!
2026 //
2027 // That is, what is normally supposed to happen is something like:
2028 //
2029 // 1. Store 'X' to cleanup slot.
2030 // 2. Branch to cleanup block.
2031 // 3. Execute cleanup.
2032 // 4. Read value from cleanup slot.
2033 // 5. Branch to the block associated with 'X'.
2034 //
2035 // But if we encounter a _Defer' statement that contains a cleanup, then
2036 // what might instead happen is:
2037 //
2038 // 1. Store 'X' to cleanup slot.
2039 // 2. Branch to cleanup block.
2040 // 3. Execute cleanup; this ends up pushing another cleanup, so:
2041 // 3a. Store 'Y' to cleanup slot.
2042 // 3b. Run steps 2–5 recursively.
2043 // 4. Read value from cleanup slot, which is now 'Y' instead of 'X'.
2044 // 5. Branch to the block associated with 'Y'... which doesn't even
2045 // exist because the value 'Y' is only meaningful for the inner
2046 // cleanup. The result is we just branch 'somewhere random'.
2047 //
2048 // The rest of the cleanup code simply isn't prepared to handle this case
2049 // because most other cleanups can't push more cleanups, and thus, emitting
2050 // other cleanups generally cannot clobber the cleanup slot.
2051 //
2052 // To prevent this from happening, save the current cleanup slot value and
2053 // restore it after emitting the '_Defer' statement.
2054 llvm::Value *SavedCleanupDest = nullptr;
2055 if (CGF.NormalCleanupDest.isValid())
2056 SavedCleanupDest =
2057 CGF.Builder.CreateLoad(Addr: CGF.NormalCleanupDest, Name: "cleanup.dest.saved");
2058
2059 CGF.EmitStmt(S: Stmt.getBody());
2060
2061 if (SavedCleanupDest && CGF.HaveInsertPoint())
2062 CGF.Builder.CreateStore(Val: SavedCleanupDest, Addr: CGF.NormalCleanupDest);
2063
2064 // Cleanups must end with an insert point.
2065 CGF.EnsureInsertPoint();
2066 }
2067};
2068} // namespace
2069
2070void CodeGenFunction::EmitDeferStmt(const DeferStmt &S) {
2071 EHStack.pushCleanup<EmitDeferredStatement>(Kind: NormalAndEHCleanup, A: &S);
2072}
2073
2074/// CollectStatementsForCase - Given the body of a 'switch' statement and a
2075/// constant value that is being switched on, see if we can dead code eliminate
2076/// the body of the switch to a simple series of statements to emit. Basically,
2077/// on a switch (5) we want to find these statements:
2078/// case 5:
2079/// printf(...); <--
2080/// ++i; <--
2081/// break;
2082///
2083/// and add them to the ResultStmts vector. If it is unsafe to do this
2084/// transformation (for example, one of the elided statements contains a label
2085/// that might be jumped to), return CSFC_Failure. If we handled it and 'S'
2086/// should include statements after it (e.g. the printf() line is a substmt of
2087/// the case) then return CSFC_FallThrough. If we handled it and found a break
2088/// statement, then return CSFC_Success.
2089///
2090/// If Case is non-null, then we are looking for the specified case, checking
2091/// that nothing we jump over contains labels. If Case is null, then we found
2092/// the case and are looking for the break.
2093///
2094/// If the recursive walk actually finds our Case, then we set FoundCase to
2095/// true.
2096///
2097enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
2098static CSFC_Result CollectStatementsForCase(const Stmt *S,
2099 const SwitchCase *Case,
2100 bool &FoundCase,
2101 SmallVectorImpl<const Stmt*> &ResultStmts) {
2102 // If this is a null statement, just succeed.
2103 if (!S)
2104 return Case ? CSFC_Success : CSFC_FallThrough;
2105
2106 // If this is the switchcase (case 4: or default) that we're looking for, then
2107 // we're in business. Just add the substatement.
2108 if (const SwitchCase *SC = dyn_cast<SwitchCase>(Val: S)) {
2109 if (S == Case) {
2110 FoundCase = true;
2111 return CollectStatementsForCase(S: SC->getSubStmt(), Case: nullptr, FoundCase,
2112 ResultStmts);
2113 }
2114
2115 // Otherwise, this is some other case or default statement, just ignore it.
2116 return CollectStatementsForCase(S: SC->getSubStmt(), Case, FoundCase,
2117 ResultStmts);
2118 }
2119
2120 // If we are in the live part of the code and we found our break statement,
2121 // return a success!
2122 if (!Case && isa<BreakStmt>(Val: S))
2123 return CSFC_Success;
2124
2125 // If this is a switch statement, then it might contain the SwitchCase, the
2126 // break, or neither.
2127 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: S)) {
2128 // Handle this as two cases: we might be looking for the SwitchCase (if so
2129 // the skipped statements must be skippable) or we might already have it.
2130 CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
2131 bool StartedInLiveCode = FoundCase;
2132 unsigned StartSize = ResultStmts.size();
2133
2134 // If we've not found the case yet, scan through looking for it.
2135 if (Case) {
2136 // Keep track of whether we see a skipped declaration. The code could be
2137 // using the declaration even if it is skipped, so we can't optimize out
2138 // the decl if the kept statements might refer to it.
2139 bool HadSkippedDecl = false;
2140
2141 // If we're looking for the case, just see if we can skip each of the
2142 // substatements.
2143 for (; Case && I != E; ++I) {
2144 HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(S: *I);
2145
2146 switch (CollectStatementsForCase(S: *I, Case, FoundCase, ResultStmts)) {
2147 case CSFC_Failure: return CSFC_Failure;
2148 case CSFC_Success:
2149 // A successful result means that either 1) that the statement doesn't
2150 // have the case and is skippable, or 2) does contain the case value
2151 // and also contains the break to exit the switch. In the later case,
2152 // we just verify the rest of the statements are elidable.
2153 if (FoundCase) {
2154 // If we found the case and skipped declarations, we can't do the
2155 // optimization.
2156 if (HadSkippedDecl)
2157 return CSFC_Failure;
2158
2159 for (++I; I != E; ++I)
2160 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2161 return CSFC_Failure;
2162 return CSFC_Success;
2163 }
2164 break;
2165 case CSFC_FallThrough:
2166 // If we have a fallthrough condition, then we must have found the
2167 // case started to include statements. Consider the rest of the
2168 // statements in the compound statement as candidates for inclusion.
2169 assert(FoundCase && "Didn't find case but returned fallthrough?");
2170 // We recursively found Case, so we're not looking for it anymore.
2171 Case = nullptr;
2172
2173 // If we found the case and skipped declarations, we can't do the
2174 // optimization.
2175 if (HadSkippedDecl)
2176 return CSFC_Failure;
2177 break;
2178 }
2179 }
2180
2181 if (!FoundCase)
2182 return CSFC_Success;
2183
2184 assert(!HadSkippedDecl && "fallthrough after skipping decl");
2185 }
2186
2187 // If we have statements in our range, then we know that the statements are
2188 // live and need to be added to the set of statements we're tracking.
2189 bool AnyDecls = false;
2190 for (; I != E; ++I) {
2191 AnyDecls |= CodeGenFunction::mightAddDeclToScope(S: *I);
2192
2193 switch (CollectStatementsForCase(S: *I, Case: nullptr, FoundCase, ResultStmts)) {
2194 case CSFC_Failure: return CSFC_Failure;
2195 case CSFC_FallThrough:
2196 // A fallthrough result means that the statement was simple and just
2197 // included in ResultStmt, keep adding them afterwards.
2198 break;
2199 case CSFC_Success:
2200 // A successful result means that we found the break statement and
2201 // stopped statement inclusion. We just ensure that any leftover stmts
2202 // are skippable and return success ourselves.
2203 for (++I; I != E; ++I)
2204 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2205 return CSFC_Failure;
2206 return CSFC_Success;
2207 }
2208 }
2209
2210 // If we're about to fall out of a scope without hitting a 'break;', we
2211 // can't perform the optimization if there were any decls in that scope
2212 // (we'd lose their end-of-lifetime).
2213 if (AnyDecls) {
2214 // If the entire compound statement was live, there's one more thing we
2215 // can try before giving up: emit the whole thing as a single statement.
2216 // We can do that unless the statement contains a 'break;'.
2217 // FIXME: Such a break must be at the end of a construct within this one.
2218 // We could emit this by just ignoring the BreakStmts entirely.
2219 if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
2220 ResultStmts.resize(N: StartSize);
2221 ResultStmts.push_back(Elt: S);
2222 } else {
2223 return CSFC_Failure;
2224 }
2225 }
2226
2227 return CSFC_FallThrough;
2228 }
2229
2230 // Okay, this is some other statement that we don't handle explicitly, like a
2231 // for statement or increment etc. If we are skipping over this statement,
2232 // just verify it doesn't have labels, which would make it invalid to elide.
2233 if (Case) {
2234 if (CodeGenFunction::ContainsLabel(S, IgnoreCaseStmts: true))
2235 return CSFC_Failure;
2236 return CSFC_Success;
2237 }
2238
2239 // Otherwise, we want to include this statement. Everything is cool with that
2240 // so long as it doesn't contain a break out of the switch we're in.
2241 if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
2242
2243 // Otherwise, everything is great. Include the statement and tell the caller
2244 // that we fall through and include the next statement as well.
2245 ResultStmts.push_back(Elt: S);
2246 return CSFC_FallThrough;
2247}
2248
2249/// FindCaseStatementsForValue - Find the case statement being jumped to and
2250/// then invoke CollectStatementsForCase to find the list of statements to emit
2251/// for a switch on constant. See the comment above CollectStatementsForCase
2252/// for more details.
2253static bool FindCaseStatementsForValue(const SwitchStmt &S,
2254 const llvm::APSInt &ConstantCondValue,
2255 SmallVectorImpl<const Stmt*> &ResultStmts,
2256 ASTContext &C,
2257 const SwitchCase *&ResultCase) {
2258 // First step, find the switch case that is being branched to. We can do this
2259 // efficiently by scanning the SwitchCase list.
2260 const SwitchCase *Case = S.getSwitchCaseList();
2261 const DefaultStmt *DefaultCase = nullptr;
2262
2263 for (; Case; Case = Case->getNextSwitchCase()) {
2264 // It's either a default or case. Just remember the default statement in
2265 // case we're not jumping to any numbered cases.
2266 if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Val: Case)) {
2267 DefaultCase = DS;
2268 continue;
2269 }
2270
2271 // Check to see if this case is the one we're looking for.
2272 const CaseStmt *CS = cast<CaseStmt>(Val: Case);
2273 // Don't handle case ranges yet.
2274 if (CS->getRHS()) return false;
2275
2276 // If we found our case, remember it as 'case'.
2277 if (CS->getLHS()->EvaluateKnownConstInt(Ctx: C) == ConstantCondValue)
2278 break;
2279 }
2280
2281 // If we didn't find a matching case, we use a default if it exists, or we
2282 // elide the whole switch body!
2283 if (!Case) {
2284 // It is safe to elide the body of the switch if it doesn't contain labels
2285 // etc. If it is safe, return successfully with an empty ResultStmts list.
2286 if (!DefaultCase)
2287 return !CodeGenFunction::ContainsLabel(S: &S);
2288 Case = DefaultCase;
2289 }
2290
2291 // Ok, we know which case is being jumped to, try to collect all the
2292 // statements that follow it. This can fail for a variety of reasons. Also,
2293 // check to see that the recursive walk actually found our case statement.
2294 // Insane cases like this can fail to find it in the recursive walk since we
2295 // don't handle every stmt kind:
2296 // switch (4) {
2297 // while (1) {
2298 // case 4: ...
2299 bool FoundCase = false;
2300 ResultCase = Case;
2301 return CollectStatementsForCase(S: S.getBody(), Case, FoundCase,
2302 ResultStmts) != CSFC_Failure &&
2303 FoundCase;
2304}
2305
2306static std::optional<SmallVector<uint64_t, 16>>
2307getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
2308 // Are there enough branches to weight them?
2309 if (Likelihoods.size() <= 1)
2310 return std::nullopt;
2311
2312 uint64_t NumUnlikely = 0;
2313 uint64_t NumNone = 0;
2314 uint64_t NumLikely = 0;
2315 for (const auto LH : Likelihoods) {
2316 switch (LH) {
2317 case Stmt::LH_Unlikely:
2318 ++NumUnlikely;
2319 break;
2320 case Stmt::LH_None:
2321 ++NumNone;
2322 break;
2323 case Stmt::LH_Likely:
2324 ++NumLikely;
2325 break;
2326 }
2327 }
2328
2329 // Is there a likelihood attribute used?
2330 if (NumUnlikely == 0 && NumLikely == 0)
2331 return std::nullopt;
2332
2333 // When multiple cases share the same code they can be combined during
2334 // optimization. In that case the weights of the branch will be the sum of
2335 // the individual weights. Make sure the combined sum of all neutral cases
2336 // doesn't exceed the value of a single likely attribute.
2337 // The additions both avoid divisions by 0 and make sure the weights of None
2338 // don't exceed the weight of Likely.
2339 const uint64_t Likely = INT32_MAX / (NumLikely + 2);
2340 const uint64_t None = Likely / (NumNone + 1);
2341 const uint64_t Unlikely = 0;
2342
2343 SmallVector<uint64_t, 16> Result;
2344 Result.reserve(N: Likelihoods.size());
2345 for (const auto LH : Likelihoods) {
2346 switch (LH) {
2347 case Stmt::LH_Unlikely:
2348 Result.push_back(Elt: Unlikely);
2349 break;
2350 case Stmt::LH_None:
2351 Result.push_back(Elt: None);
2352 break;
2353 case Stmt::LH_Likely:
2354 Result.push_back(Elt: Likely);
2355 break;
2356 }
2357 }
2358
2359 return Result;
2360}
2361
2362void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
2363 // Handle nested switch statements.
2364 llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
2365 SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
2366 SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
2367 llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
2368
2369 // See if we can constant fold the condition of the switch and therefore only
2370 // emit the live case statement (if any) of the switch.
2371 llvm::APSInt ConstantCondValue;
2372 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: ConstantCondValue)) {
2373 SmallVector<const Stmt*, 4> CaseStmts;
2374 const SwitchCase *Case = nullptr;
2375 if (FindCaseStatementsForValue(S, ConstantCondValue, ResultStmts&: CaseStmts,
2376 C&: getContext(), ResultCase&: Case)) {
2377 if (Case)
2378 incrementProfileCounter(S: Case);
2379 RunCleanupsScope ExecutedScope(*this);
2380
2381 if (S.getInit())
2382 EmitStmt(S: S.getInit());
2383
2384 // Emit the condition variable if needed inside the entire cleanup scope
2385 // used by this special case for constant folded switches.
2386 if (S.getConditionVariable())
2387 EmitDecl(D: *S.getConditionVariable(), /*EvaluateConditionDecl=*/true);
2388
2389 // At this point, we are no longer "within" a switch instance, so
2390 // we can temporarily enforce this to ensure that any embedded case
2391 // statements are not emitted.
2392 SwitchInsn = nullptr;
2393
2394 // Okay, we can dead code eliminate everything except this case. Emit the
2395 // specified series of statements and we're good.
2396 for (const Stmt *CaseStmt : CaseStmts)
2397 EmitStmt(S: CaseStmt);
2398 incrementProfileCounter(S: &S);
2399 PGO->markStmtMaybeUsed(S: S.getBody());
2400
2401 // Now we want to restore the saved switch instance so that nested
2402 // switches continue to function properly
2403 SwitchInsn = SavedSwitchInsn;
2404
2405 return;
2406 }
2407 }
2408
2409 JumpDest SwitchExit = getJumpDestInCurrentScope(Name: "sw.epilog");
2410
2411 RunCleanupsScope ConditionScope(*this);
2412
2413 if (S.getInit()) {
2414 EmitStmt(S: S.getInit());
2415
2416 // The init statement may have cleared the insertion point (e.g. it ended in
2417 // a 'noreturn' call); the condition emitted below needs a valid one.
2418 EnsureInsertPoint();
2419 }
2420
2421 if (S.getConditionVariable())
2422 EmitDecl(D: *S.getConditionVariable());
2423 llvm::Value *CondV = EmitScalarExpr(E: S.getCond());
2424 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
2425
2426 // Create basic block to hold stuff that comes after switch
2427 // statement. We also need to create a default block now so that
2428 // explicit case ranges tests can have a place to jump to on
2429 // failure.
2430 llvm::BasicBlock *DefaultBlock = createBasicBlock(name: "sw.default");
2431 SwitchInsn = Builder.CreateSwitch(V: CondV, Dest: DefaultBlock);
2432 addInstToNewSourceAtom(KeyInstruction: SwitchInsn, Backup: CondV);
2433
2434 if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {
2435 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2436 llvm::ConstantInt *BranchHintConstant =
2437 HLSLControlFlowAttr ==
2438 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2439 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2440 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2441 llvm::Metadata *Vals[] = {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2442 MDHelper.createConstant(C: BranchHintConstant)};
2443 SwitchInsn->setMetadata(Kind: "hlsl.controlflow.hint",
2444 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2445 }
2446
2447 if (PGO->haveRegionCounts()) {
2448 // Walk the SwitchCase list to find how many there are.
2449 uint64_t DefaultCount = 0;
2450 unsigned NumCases = 0;
2451 for (const SwitchCase *Case = S.getSwitchCaseList();
2452 Case;
2453 Case = Case->getNextSwitchCase()) {
2454 if (isa<DefaultStmt>(Val: Case))
2455 DefaultCount = getProfileCount(S: Case);
2456 NumCases += 1;
2457 }
2458 SwitchWeights = new SmallVector<uint64_t, 16>();
2459 SwitchWeights->reserve(N: NumCases);
2460 // The default needs to be first. We store the edge count, so we already
2461 // know the right weight.
2462 SwitchWeights->push_back(Elt: DefaultCount);
2463 } else if (CGM.getCodeGenOpts().OptimizationLevel) {
2464 SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
2465 // Initialize the default case.
2466 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
2467 }
2468
2469 CaseRangeBlock = DefaultBlock;
2470
2471 // Clear the insertion point to indicate we are in unreachable code.
2472 Builder.ClearInsertionPoint();
2473
2474 // All break statements jump to NextBlock. If BreakContinueStack is non-empty
2475 // then reuse last ContinueBlock.
2476 JumpDest OuterContinue;
2477 if (!BreakContinueStack.empty())
2478 OuterContinue = BreakContinueStack.back().ContinueBlock;
2479
2480 BreakContinueStack.push_back(Elt: BreakContinue(S, SwitchExit, OuterContinue));
2481
2482 // Emit switch body.
2483 EmitStmt(S: S.getBody());
2484
2485 BreakContinueStack.pop_back();
2486
2487 // Update the default block in case explicit case range tests have
2488 // been chained on top.
2489 SwitchInsn->setDefaultDest(CaseRangeBlock);
2490
2491 // If a default was never emitted:
2492 if (!DefaultBlock->getParent()) {
2493 // If we have cleanups, emit the default block so that there's a
2494 // place to jump through the cleanups from.
2495 if (ConditionScope.requiresCleanups()) {
2496 EmitBlock(BB: DefaultBlock);
2497
2498 // Otherwise, just forward the default block to the switch end.
2499 } else {
2500 DefaultBlock->replaceAllUsesWith(V: SwitchExit.getBlock());
2501 delete DefaultBlock;
2502 }
2503 }
2504
2505 ConditionScope.ForceCleanup();
2506
2507 // Close the last case (or DefaultBlock).
2508 EmitBranch(Target: SwitchExit.getBlock());
2509
2510 // Insert a False Counter if SwitchStmt doesn't have DefaultStmt.
2511 if (hasSkipCounter(S: S.getCond())) {
2512 auto *ImplicitDefaultBlock = createBasicBlock(name: "sw.false");
2513 EmitBlock(BB: ImplicitDefaultBlock);
2514 incrementProfileCounter(ExecSkip: UseSkipPath, S: S.getCond());
2515 Builder.CreateBr(Dest: SwitchInsn->getDefaultDest());
2516 SwitchInsn->setDefaultDest(ImplicitDefaultBlock);
2517 }
2518
2519 // Emit continuation.
2520 EmitBlock(BB: SwitchExit.getBlock(), IsFinished: true);
2521 incrementProfileCounter(S: &S);
2522
2523 // If the switch has a condition wrapped by __builtin_unpredictable,
2524 // create metadata that specifies that the switch is unpredictable.
2525 // Don't bother if not optimizing because that metadata would not be used.
2526 auto *Call = dyn_cast<CallExpr>(Val: S.getCond());
2527 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2528 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2529 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2530 llvm::MDBuilder MDHelper(getLLVMContext());
2531 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_unpredictable,
2532 Node: MDHelper.createUnpredictable());
2533 }
2534 }
2535
2536 if (SwitchWeights) {
2537 assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2538 "switch weights do not match switch cases");
2539 // If there's only one jump destination there's no sense weighting it.
2540 if (SwitchWeights->size() > 1)
2541 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2542 Node: createProfileWeights(Weights: *SwitchWeights));
2543 delete SwitchWeights;
2544 } else if (SwitchLikelihood) {
2545 assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2546 "switch likelihoods do not match switch cases");
2547 std::optional<SmallVector<uint64_t, 16>> LHW =
2548 getLikelihoodWeights(Likelihoods: *SwitchLikelihood);
2549 if (LHW) {
2550 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2551 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2552 Node: createProfileWeights(Weights: *LHW));
2553 }
2554 delete SwitchLikelihood;
2555 }
2556 SwitchInsn = SavedSwitchInsn;
2557 SwitchWeights = SavedSwitchWeights;
2558 SwitchLikelihood = SavedSwitchLikelihood;
2559 CaseRangeBlock = SavedCRBlock;
2560}
2561
2562std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2563 const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2564 QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2565 if (Info.allowsRegister() || !Info.allowsMemory()) {
2566 if (CodeGenFunction::hasScalarEvaluationKind(T: InputType))
2567 return {EmitLoadOfLValue(V: InputValue, Loc).getScalarVal(), nullptr};
2568
2569 llvm::Type *Ty = ConvertType(T: InputType);
2570 uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2571 if ((Size <= 64 && llvm::isPowerOf2_64(Value: Size)) ||
2572 getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty)) {
2573 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2574
2575 return {Builder.CreateLoad(Addr: InputValue.getAddress().withElementType(ElemTy: Ty)),
2576 nullptr};
2577 }
2578 }
2579
2580 Address Addr = InputValue.getAddress();
2581 ConstraintStr += '*';
2582 return {InputValue.getPointer(CGF&: *this), Addr.getElementType()};
2583}
2584std::pair<llvm::Value *, llvm::Type *>
2585CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2586 const Expr *InputExpr,
2587 std::string &ConstraintStr) {
2588 // If this can't be a register or memory, i.e., has to be a constant
2589 // (immediate or symbolic), try to emit it as such.
2590 if (!Info.allowsRegister() && !Info.allowsMemory()) {
2591 if (Info.requiresImmediateConstant()) {
2592 Expr::EvalResult EVResult;
2593 InputExpr->EvaluateAsRValue(Result&: EVResult, Ctx: getContext(), InConstantContext: true);
2594
2595 llvm::APSInt IntResult;
2596 if (EVResult.Val.toIntegralConstant(Result&: IntResult, SrcTy: InputExpr->getType(),
2597 Ctx: getContext()))
2598 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: IntResult), nullptr};
2599 }
2600
2601 Expr::EvalResult Result;
2602 if (InputExpr->EvaluateAsInt(Result, Ctx: getContext()))
2603 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: Result.Val.getInt()),
2604 nullptr};
2605 }
2606
2607 if (Info.allowsRegister() || !Info.allowsMemory())
2608 if (CodeGenFunction::hasScalarEvaluationKind(T: InputExpr->getType()))
2609 return {EmitScalarExpr(E: InputExpr), nullptr};
2610 if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2611 return {EmitScalarExpr(E: InputExpr), nullptr};
2612 InputExpr = InputExpr->IgnoreParenNoopCasts(Ctx: getContext());
2613 LValue Dest = EmitLValue(E: InputExpr);
2614 return EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr,
2615 Loc: InputExpr->getExprLoc());
2616}
2617
2618/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2619/// asm call instruction. The !srcloc MDNode contains a list of constant
2620/// integers which are the source locations of the start of each line in the
2621/// asm.
2622static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2623 CodeGenFunction &CGF) {
2624 SmallVector<llvm::Metadata *, 8> Locs;
2625 // Add the location of the first line to the MDNode.
2626 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
2627 Ty: CGF.Int64Ty, V: Str->getBeginLoc().getRawEncoding())));
2628 StringRef StrVal = Str->getString();
2629 if (!StrVal.empty()) {
2630 const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2631 const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2632 unsigned StartToken = 0;
2633 unsigned ByteOffset = 0;
2634
2635 // Add the location of the start of each subsequent line of the asm to the
2636 // MDNode.
2637 for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2638 if (StrVal[i] != '\n') continue;
2639 SourceLocation LineLoc = Str->getLocationOfByte(
2640 ByteNo: i + 1, SM, Features: LangOpts, Target: CGF.getTarget(), StartToken: &StartToken, StartTokenByteOffset: &ByteOffset);
2641 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(
2642 C: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: LineLoc.getRawEncoding())));
2643 }
2644 }
2645
2646 return llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: Locs);
2647}
2648
2649namespace clang {
2650
2651/// This structure holds the information gathered about the constraints for an
2652/// inline assembly statement. It helps in separating the constraint processing
2653/// from the code generation.
2654class AsmConstraintsInfo {
2655 CodeGenFunction &CGF;
2656 CodeGenModule &CGM; // Per-module state.
2657 const AsmStmt &S;
2658 CGBuilderTy &Builder;
2659
2660 // The final asm string.
2661 std::string AsmString;
2662
2663 // The output and input constraints.
2664 SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2665 SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2666
2667 // Constraint strings.
2668 std::string Constraints;
2669 std::string InOutConstraints;
2670
2671 // Keep track of out constraints for tied input operand.
2672 std::vector<std::string> OutputConstraints;
2673
2674 // Keep track of argument types.
2675 std::vector<llvm::Value *> Args;
2676 std::vector<llvm::Type *> ArgTypes;
2677 std::vector<llvm::Type *> ArgElemTypes;
2678
2679 // Keep track of result register constraints.
2680 std::vector<LValue> ResultRegDests;
2681 std::vector<QualType> ResultRegQualTys;
2682 std::vector<llvm::Type *> ResultRegTypes;
2683 std::vector<llvm::Type *> ResultTruncRegTypes;
2684
2685 llvm::BitVector ResultTypeRequiresCast;
2686
2687 // Keep track of in/out constraints.
2688 std::vector<llvm::Value *> InOutArgs;
2689 std::vector<llvm::Type *> InOutArgTypes;
2690 std::vector<llvm::Type *> InOutArgElemTypes;
2691
2692 // Destination blocks for 'asm gotos'.
2693 llvm::BasicBlock *DefaultDest = nullptr;
2694 SmallVector<llvm::BasicBlock *, 3> IndirectDests;
2695
2696 std::vector<std::optional<std::pair<unsigned, unsigned>>> ResultBounds;
2697
2698 // An inline asm can be marked readonly if it meets the following
2699 // conditions:
2700 //
2701 // - it doesn't have any sideeffects
2702 // - it doesn't clobber memory
2703 // - it doesn't return a value by-reference
2704 //
2705 // It can be marked readnone if it doesn't have any input memory
2706 // constraints in addition to meeting the conditions listed above.
2707 bool ReadOnly = true;
2708 bool ReadNone = true;
2709
2710 bool GetOutputAndInputConstraints();
2711 void HandleOutputConstraints();
2712 void HandleMSStyleAsmBlob();
2713 void HandleInputConstraints();
2714 bool HandleLabels();
2715 bool HandleClobbers();
2716 void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2717 bool HasUnwindClobber, bool NoMerge, bool NoConvergent,
2718 std::vector<llvm::Value *> &RegResults);
2719 void EmitAsmStores(const llvm::ArrayRef<llvm::Value *> RegResults);
2720
2721 void EmitHipStdParUnsupportedAsm() {
2722 constexpr auto Name = "__ASM__hipstdpar_unsupported";
2723
2724 std::string Asm;
2725 if (auto GCCAsm = dyn_cast<GCCAsmStmt>(Val: &S))
2726 Asm = GCCAsm->getAsmString();
2727
2728 auto &Ctx = getLLVMContext();
2729 auto StrTy = llvm::ConstantDataArray::getString(Context&: Ctx, Initializer: Asm);
2730 auto FnTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx),
2731 Params: {StrTy->getType()}, isVarArg: false);
2732 auto UBF = CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2733
2734 Builder.CreateCall(Callee: UBF, Args: {StrTy});
2735 }
2736
2737 ASTContext &getContext() { return CGF.getContext(); }
2738 llvm::LLVMContext &getLLVMContext() { return CGF.getLLVMContext(); }
2739 const TargetInfo &getTarget() const { return CGF.getTarget(); }
2740 const LangOptions &getLangOpts() const { return CGF.getLangOpts(); }
2741 const TargetCodeGenInfo &getTargetHooks() const {
2742 return CGM.getTargetCodeGenInfo();
2743 }
2744
2745public:
2746 AsmConstraintsInfo(CodeGenFunction &CGF, const AsmStmt &S)
2747 : CGF(CGF), CGM(CGF.CGM), S(S), Builder(CGF.Builder),
2748 AsmString(S.generateAsmString(C: CGF.getContext())) {}
2749
2750 void EmitAsmStmt();
2751};
2752
2753} // namespace clang
2754
2755void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2756 // Pop all cleanup blocks at the end of the asm statement.
2757 CodeGenFunction::RunCleanupsScope Cleanups(*this);
2758
2759 // Get all the output and input constraints together.
2760 AsmConstraintsInfo AsmInfo(*this, S);
2761 AsmInfo.EmitAsmStmt();
2762}
2763
2764void AsmConstraintsInfo::EmitAsmStmt() {
2765 if (!GetOutputAndInputConstraints())
2766 return EmitHipStdParUnsupportedAsm();
2767
2768 // Handle output constraints.
2769 HandleOutputConstraints();
2770
2771 // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2772 // to the return value slot. Only do this when returning in registers.
2773 HandleMSStyleAsmBlob();
2774
2775 // Handle input constraints.
2776 HandleInputConstraints();
2777
2778 // Handle 'asm goto' labels.
2779 bool IsGCCAsmGoto = HandleLabels();
2780
2781 // Handle any clobbers.
2782 bool HasUnwindClobber = HandleClobbers();
2783 assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
2784 "unwind clobber can't be used with asm goto");
2785
2786 // Add machine specific clobbers
2787 std::string_view MachineClobbers = getTarget().getClobbers();
2788 if (!MachineClobbers.empty()) {
2789 if (!Constraints.empty())
2790 Constraints += ',';
2791 Constraints += MachineClobbers;
2792 }
2793
2794 llvm::Type *ResultType;
2795 if (ResultRegTypes.empty())
2796 ResultType = CGF.VoidTy;
2797 else if (ResultRegTypes.size() == 1)
2798 ResultType = ResultRegTypes[0];
2799 else
2800 ResultType = llvm::StructType::get(Context&: getLLVMContext(), Elements: ResultRegTypes);
2801
2802 llvm::FunctionType *FTy =
2803 llvm::FunctionType::get(Result: ResultType, Params: ArgTypes, isVarArg: false);
2804
2805 bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
2806
2807 llvm::InlineAsm::AsmDialect GnuAsmDialect =
2808 CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
2809 ? llvm::InlineAsm::AD_ATT
2810 : llvm::InlineAsm::AD_Intel;
2811 llvm::InlineAsm::AsmDialect AsmDialect =
2812 isa<MSAsmStmt>(Val: &S) ? llvm::InlineAsm::AD_Intel : GnuAsmDialect;
2813
2814 llvm::InlineAsm *IA = llvm::InlineAsm::get(
2815 Ty: FTy, AsmString, Constraints, hasSideEffects: HasSideEffect,
2816 /* IsAlignStack */ isAlignStack: false, asmDialect: AsmDialect, canThrow: HasUnwindClobber);
2817 std::vector<llvm::Value *> RegResults;
2818 llvm::CallBrInst *CBR;
2819 llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>
2820 CBRRegResults;
2821
2822 if (IsGCCAsmGoto) {
2823 CBR = Builder.CreateCallBr(Callee: IA, DefaultDest, IndirectDests, Args);
2824 CGF.EmitBlock(BB: DefaultDest);
2825 UpdateAsmCallInst(Result&: *CBR, HasSideEffect,
2826 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2827 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2828
2829 // Because we are emitting code top to bottom, we don't have enough
2830 // information at this point to know precisely whether we have a critical
2831 // edge. If we have outputs, split all indirect destinations.
2832 if (!RegResults.empty()) {
2833 unsigned I = 0;
2834 for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {
2835 llvm::Twine SynthName = Dest->getName() + ".split";
2836 llvm::BasicBlock *SynthBB = CGF.createBasicBlock(name: SynthName);
2837 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2838 Builder.SetInsertPoint(SynthBB);
2839
2840 if (ResultRegTypes.size() == 1) {
2841 CBRRegResults[SynthBB].push_back(Elt: CBR);
2842 } else {
2843 for (unsigned J = 0, E = ResultRegTypes.size(); J != E; ++J) {
2844 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: CBR, Idxs: J, Name: "asmresult");
2845 CBRRegResults[SynthBB].push_back(Elt: Tmp);
2846 }
2847 }
2848
2849 CGF.EmitBranch(Target: Dest);
2850 CGF.EmitBlock(BB: SynthBB);
2851 CBR->setIndirectDest(i: I++, B: SynthBB);
2852 }
2853 }
2854 } else if (HasUnwindClobber) {
2855 llvm::CallBase *Result = CGF.EmitCallOrInvoke(Callee: IA, Args, Name: "");
2856 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2857 /*HasUnwindClobber=*/true, NoMerge: CGF.InNoMergeAttributedStmt,
2858 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2859 } else {
2860 llvm::CallInst *Result =
2861 Builder.CreateCall(Callee: IA, Args, OpBundles: CGF.getBundlesForFunclet(Callee: IA));
2862 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2863 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2864 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2865 }
2866
2867 EmitAsmStores(RegResults);
2868
2869 // If this is an asm goto with outputs, repeat EmitAsmStores, but with a
2870 // different insertion point; one for each indirect destination and with
2871 // CBRRegResults rather than RegResults.
2872 if (IsGCCAsmGoto && !CBRRegResults.empty()) {
2873 for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {
2874 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2875 Builder.SetInsertPoint(TheBB: Succ, IP: --(Succ->end()));
2876 EmitAsmStores(RegResults: CBRRegResults[Succ]);
2877 }
2878 }
2879}
2880
2881/// Gather and validate the output and input constraints for the given inline
2882/// assembly statement. This ensures that the constraints are valid for the
2883/// target and prepares them for further processing.
2884bool AsmConstraintsInfo::GetOutputAndInputConstraints() {
2885 bool IsValidTargetAsm = true;
2886 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2887 for (unsigned I = 0, E = S.getNumOutputs(); I != E && IsValidTargetAsm; I++) {
2888 StringRef Name;
2889 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2890 Name = GAS->getOutputName(i: I);
2891
2892 TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i: I), Name);
2893
2894 bool IsValid = getTarget().validateOutputConstraint(Info);
2895 if (IsHipStdPar && !IsValid)
2896 IsValidTargetAsm = false;
2897 else
2898 assert(IsValid && "Failed to parse output constraint");
2899
2900 OutputConstraintInfos.push_back(Elt: Info);
2901 }
2902
2903 for (unsigned I = 0, E = S.getNumInputs(); I != E && IsValidTargetAsm; I++) {
2904 StringRef Name;
2905 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2906 Name = GAS->getInputName(i: I);
2907
2908 TargetInfo::ConstraintInfo Info(S.getInputConstraint(i: I), Name);
2909
2910 bool IsValid =
2911 getTarget().validateInputConstraint(OutputConstraints: OutputConstraintInfos, info&: Info);
2912 if (IsHipStdPar && !IsValid)
2913 IsValidTargetAsm = false;
2914 else
2915 assert(IsValid && "Failed to parse input constraint");
2916
2917 InputConstraintInfos.push_back(Elt: Info);
2918 }
2919
2920 return IsValidTargetAsm;
2921}
2922
2923/// Process the output constraints of an inline assembly statement. This method
2924/// handles the complexity of determining whether an output should be a
2925/// register or memory operand, manages tied operands, and prepares the
2926/// necessary arguments for the LLVM inline asm call.
2927void AsmConstraintsInfo::HandleOutputConstraints() {
2928 // Keep track of defined physregs.
2929 llvm::SmallSet<std::string, 8> PhysRegOutputs;
2930
2931 for (unsigned I = 0, E = S.getNumOutputs(); I != E; I++) {
2932 TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[I];
2933
2934 // Simplify the output constraint.
2935 std::string OutputConstraint(S.getOutputConstraint(i: I));
2936 OutputConstraint = getTarget().simplifyConstraint(
2937 Constraint: StringRef(OutputConstraint).substr(Start: 1), OutCons: &OutputConstraintInfos);
2938
2939 const Expr *OutExpr = S.getOutputExpr(i: I);
2940 OutExpr = OutExpr->IgnoreParenNoopCasts(Ctx: getContext());
2941
2942 std::string GCCReg;
2943 OutputConstraint = S.addVariableConstraints(
2944 Constraint: OutputConstraint, AsmExpr: *OutExpr, Target: getTarget(), EarlyClobber: Info.earlyClobber(),
2945 UnsupportedCB: [&](const Stmt *UnspStmt, StringRef Msg) {
2946 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
2947 },
2948 GCCReg: &GCCReg);
2949
2950 // Give an error on multiple outputs to same physreg.
2951 if (!GCCReg.empty() && !PhysRegOutputs.insert(V: GCCReg).second)
2952 CGM.Error(loc: S.getAsmLoc(), error: "multiple outputs to hard register: " + GCCReg);
2953
2954 OutputConstraints.push_back(x: OutputConstraint);
2955 LValue Dest = CGF.EmitLValue(E: OutExpr);
2956 if (!Constraints.empty())
2957 Constraints += ',';
2958
2959 // If this is a register output, then make the inline asm return it
2960 // by-value. If this is a memory result, return the value by-reference.
2961 QualType QTy = OutExpr->getType();
2962 const bool IsScalarOrAggregate =
2963 CodeGenFunction::hasScalarEvaluationKind(T: QTy) ||
2964 CodeGenFunction::hasAggregateEvaluationKind(T: QTy);
2965
2966 if (!Info.allowsMemory() && IsScalarOrAggregate) {
2967 Constraints += "=" + OutputConstraint;
2968 ResultRegQualTys.push_back(x: QTy);
2969 ResultRegDests.push_back(x: Dest);
2970
2971 ResultBounds.emplace_back(args: Info.getOutputOperandBounds());
2972
2973 llvm::Type *Ty = CGF.ConvertTypeForMem(T: QTy);
2974 const bool RequiresCast =
2975 Info.allowsRegister() &&
2976 (getTargetHooks().isScalarizableAsmOperand(CGF, Ty) ||
2977 Ty->isAggregateType());
2978
2979 ResultTruncRegTypes.push_back(x: Ty);
2980 ResultTypeRequiresCast.push_back(Val: RequiresCast);
2981
2982 if (RequiresCast) {
2983 if (unsigned Size = getContext().getTypeSize(T: QTy))
2984 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2985 else
2986 CGM.Error(loc: OutExpr->getExprLoc(), error: "output size should not be zero");
2987 }
2988
2989 ResultRegTypes.push_back(x: Ty);
2990
2991 // If this output is tied to an input, and if the input is larger, then
2992 // we need to set the actual result type of the inline asm node to be the
2993 // same as the input type.
2994 if (Info.hasMatchingInput()) {
2995 unsigned InputNo;
2996 for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2997 TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2998 if (Input.hasTiedOperand() && Input.getTiedOperand() == I)
2999 break;
3000 }
3001 assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
3002
3003 QualType InputTy = S.getInputExpr(i: InputNo)->getType();
3004 QualType OutputType = OutExpr->getType();
3005
3006 uint64_t InputSize = getContext().getTypeSize(T: InputTy);
3007 if (getContext().getTypeSize(T: OutputType) < InputSize)
3008 // Form the asm to return the value as a larger integer or fp type.
3009 ResultRegTypes.back() = CGF.ConvertType(T: InputTy);
3010 }
3011
3012 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3013 CGF, Constraint: OutputConstraint, Ty: ResultRegTypes.back()))
3014 ResultRegTypes.back() = AdjTy;
3015 else
3016 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3017 DiagID: diag::err_asm_invalid_type_in_input)
3018 << OutExpr->getType() << OutputConstraint;
3019
3020 // Update largest vector width for any vector types.
3021 if (auto *VT = dyn_cast<llvm::VectorType>(Val: ResultRegTypes.back()))
3022 CGF.LargestVectorWidth =
3023 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3024 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3025 } else {
3026 Address DestAddr = Dest.getAddress();
3027
3028 // Matrix types in memory are represented by arrays, but accessed through
3029 // vector pointers, with the alignment specified on the access operation.
3030 // For inline assembly, update pointer arguments to use vector pointers.
3031 // Otherwise there will be a mis-match if the matrix is also an
3032 // input-argument which is represented as vector.
3033 if (isa<MatrixType>(Val: OutExpr->getType().getCanonicalType()))
3034 DestAddr =
3035 DestAddr.withElementType(ElemTy: CGF.ConvertType(T: OutExpr->getType()));
3036
3037 ArgTypes.push_back(x: DestAddr.getType());
3038 ArgElemTypes.push_back(x: DestAddr.getElementType());
3039 Args.push_back(x: DestAddr.emitRawPointer(CGF));
3040
3041 Constraints += "=*" + OutputConstraint;
3042 ReadOnly = false;
3043 ReadNone = false;
3044 }
3045
3046 if (!Info.isReadWrite())
3047 continue;
3048
3049 InOutConstraints += ',';
3050
3051 const Expr *InputExpr = S.getOutputExpr(i: I);
3052 llvm::Value *Arg;
3053 llvm::Type *ArgElemType;
3054 std::tie(args&: Arg, args&: ArgElemType) =
3055 CGF.EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(),
3056 ConstraintStr&: InOutConstraints, Loc: InputExpr->getExprLoc());
3057
3058 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3059 CGF, Constraint: OutputConstraint, Ty: Arg->getType()))
3060 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3061
3062 // Update largest vector width for any vector types.
3063 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3064 CGF.LargestVectorWidth =
3065 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3066 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3067
3068 // Only tie earlyclobber physregs.
3069 if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
3070 InOutConstraints += llvm::utostr(X: I);
3071 else
3072 InOutConstraints += OutputConstraint;
3073
3074 InOutArgTypes.push_back(x: Arg->getType());
3075 InOutArgElemTypes.push_back(x: ArgElemType);
3076 InOutArgs.push_back(x: Arg);
3077 }
3078}
3079
3080/// Special handling for Microsoft-style inline assembly blocks. This ensures
3081/// that return registers (like EAX:EDX) are correctly mapped to the function's
3082/// return value slot when necessary.
3083void AsmConstraintsInfo::HandleMSStyleAsmBlob() {
3084 if (!isa<MSAsmStmt>(Val: &S))
3085 return;
3086
3087 const ABIArgInfo &RetAI = CGF.CurFnInfo->getReturnInfo();
3088 if (!RetAI.isDirect() && !RetAI.isExtend())
3089 return;
3090
3091 // Make a fake lvalue for the return value slot.
3092 LValue ReturnSlot =
3093 CGF.MakeAddrLValueWithoutTBAA(Addr: CGF.ReturnValue, T: CGF.FnRetTy);
3094 CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
3095 CGF, ReturnValue: ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
3096 ResultRegDests, AsmString, NumOutputs: S.getNumOutputs());
3097 CGF.SawAsmBlock = true;
3098}
3099
3100/// Process the input constraints of an inline assembly statement. It handles
3101/// type conversions, extensions for tied operands, and collects the necessary
3102/// LLVM values to be passed to the inline assembly call.
3103void AsmConstraintsInfo::HandleInputConstraints() {
3104 ASTContext &Ctx = getContext();
3105
3106 for (unsigned I = 0, E = S.getNumInputs(); I != E; I++) {
3107 TargetInfo::ConstraintInfo &Info = InputConstraintInfos[I];
3108 const Expr *InputExpr = S.getInputExpr(i: I);
3109
3110 if (Info.allowsMemory())
3111 ReadNone = false;
3112
3113 if (!Constraints.empty())
3114 Constraints += ',';
3115
3116 // Simplify the input constraint.
3117 std::string InputConstraint(S.getInputConstraint(i: I));
3118 InputConstraint =
3119 getTarget().simplifyConstraint(Constraint: InputConstraint, OutCons: &OutputConstraintInfos);
3120
3121 InputConstraint = S.addVariableConstraints(
3122 Constraint: InputConstraint, AsmExpr: *InputExpr->IgnoreParenNoopCasts(Ctx), Target: getTarget(),
3123 EarlyClobber: false /* No EarlyClobber */,
3124 UnsupportedCB: [&](const Stmt *UnspStmt, std::string_view Msg) {
3125 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
3126 });
3127
3128 std::string ReplaceConstraint(InputConstraint);
3129 llvm::Value *Arg;
3130 llvm::Type *ArgElemType;
3131 std::tie(args&: Arg, args&: ArgElemType) = CGF.EmitAsmInput(Info, InputExpr, ConstraintStr&: Constraints);
3132
3133 // If this input argument is tied to a larger output result, extend the
3134 // input to be the same size as the output. The LLVM backend wants to see
3135 // the input and output of a matching constraint be the same size. Note
3136 // that GCC does not define what the top bits are here. We use zext because
3137 // that is usually cheaper, but LLVM IR should really get an anyext someday.
3138 if (Info.hasTiedOperand()) {
3139 unsigned Output = Info.getTiedOperand();
3140 QualType OutputType = S.getOutputExpr(i: Output)->getType();
3141 QualType InputTy = InputExpr->getType();
3142
3143 if (Ctx.getTypeSize(T: OutputType) > Ctx.getTypeSize(T: InputTy)) {
3144 // Use ptrtoint as appropriate so that we can do our extension.
3145 if (isa<llvm::PointerType>(Val: Arg->getType()))
3146 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: CGF.IntPtrTy);
3147
3148 llvm::Type *OutputTy = CGF.ConvertType(T: OutputType);
3149 if (isa<llvm::IntegerType>(Val: OutputTy))
3150 Arg = Builder.CreateZExt(V: Arg, DestTy: OutputTy);
3151 else if (isa<llvm::PointerType>(Val: OutputTy))
3152 Arg = Builder.CreateZExt(V: Arg, DestTy: CGF.IntPtrTy);
3153 else if (OutputTy->isFloatingPointTy())
3154 Arg = Builder.CreateFPExt(V: Arg, DestTy: OutputTy);
3155 }
3156
3157 // Deal with the tied operands' constraint code in adjustInlineAsmType.
3158 ReplaceConstraint = OutputConstraints[Output];
3159 }
3160
3161 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3162 CGF, Constraint: ReplaceConstraint, Ty: Arg->getType()))
3163 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3164 else
3165 CGM.getDiags().Report(Loc: S.getAsmLoc(), DiagID: diag::err_asm_invalid_type_in_input)
3166 << InputExpr->getType() << InputConstraint;
3167
3168 // Update largest vector width for any vector types.
3169 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3170 CGF.LargestVectorWidth =
3171 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3172 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3173
3174 ArgTypes.push_back(x: Arg->getType());
3175 ArgElemTypes.push_back(x: ArgElemType);
3176 Args.push_back(x: Arg);
3177
3178 Constraints += InputConstraint;
3179 }
3180
3181 // Append the "input" part of in/out constraints.
3182 for (unsigned I = 0, E = InOutArgs.size(); I != E; I++) {
3183 ArgTypes.push_back(x: InOutArgTypes[I]);
3184 ArgElemTypes.push_back(x: InOutArgElemTypes[I]);
3185 Args.push_back(x: InOutArgs[I]);
3186 }
3187
3188 Constraints += InOutConstraints;
3189}
3190
3191/// Handle labels in an 'asm goto' statement. This method resolves the symbolic
3192/// labels to LLVM basic blocks and updates the constraint string to reflect
3193/// the indirect jump targets.
3194bool AsmConstraintsInfo::HandleLabels() {
3195 if (const auto *GS = dyn_cast<GCCAsmStmt>(Val: &S); GS && GS->isAsmGoto()) {
3196 for (const auto *E : GS->labels()) {
3197 CodeGenFunction::JumpDest Dest = CGF.getJumpDestForLabel(D: E->getLabel());
3198 IndirectDests.push_back(Elt: Dest.getBlock());
3199
3200 if (!Constraints.empty())
3201 Constraints += ',';
3202
3203 Constraints += "!i";
3204 }
3205
3206 DefaultDest = CGF.createBasicBlock(name: "asm.fallthrough");
3207 return true;
3208 }
3209
3210 return false;
3211}
3212
3213/// Process clobber constraints for an inline assembly statement. This
3214/// identifies which registers or system state (like "memory" or "cc") are
3215/// modified by the assembly block, which is crucial for correct optimization
3216/// and side-effect modeling.
3217bool AsmConstraintsInfo::HandleClobbers() {
3218 bool HasUnwindClobber = false;
3219 for (unsigned I = 0, E = S.getNumClobbers(); I != E; I++) {
3220 std::string Clobber = S.getClobber(i: I);
3221
3222 if (Clobber == "unwind") {
3223 HasUnwindClobber = true;
3224 continue;
3225 }
3226
3227 if (Clobber == "memory") {
3228 ReadOnly = false;
3229 ReadNone = false;
3230 } else if (Clobber != "cc") {
3231 Clobber = getTarget().getNormalizedGCCRegisterName(Name: Clobber);
3232 if (CGM.getCodeGenOpts().StackClashProtector &&
3233 getTarget().isSPRegName(Clobber)) {
3234 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3235 DiagID: diag::warn_stack_clash_protection_inline_asm);
3236 }
3237 }
3238
3239 if (isa<MSAsmStmt>(Val: &S)) {
3240 if (Clobber == "eax" || Clobber == "edx") {
3241 if (Constraints.find(s: "=&A") != std::string::npos)
3242 continue;
3243
3244 std::string::size_type position1 =
3245 Constraints.find(str: "={" + Clobber + "}");
3246 if (position1 != std::string::npos) {
3247 Constraints.insert(pos: position1 + 1, s: "&");
3248 continue;
3249 }
3250
3251 std::string::size_type position2 = Constraints.find(s: "=A");
3252 if (position2 != std::string::npos) {
3253 Constraints.insert(pos: position2 + 1, s: "&");
3254 continue;
3255 }
3256 }
3257 }
3258
3259 if (!Constraints.empty())
3260 Constraints += ',';
3261
3262 Constraints += "~{" + Clobber + '}';
3263 }
3264
3265 return HasUnwindClobber;
3266}
3267
3268void AsmConstraintsInfo::UpdateAsmCallInst(
3269 llvm::CallBase &Result, bool HasSideEffect, bool HasUnwindClobber,
3270 bool NoMerge, bool NoConvergent, std::vector<llvm::Value *> &RegResults) {
3271 if (!HasUnwindClobber)
3272 Result.addFnAttr(Kind: llvm::Attribute::NoUnwind);
3273
3274 if (NoMerge)
3275 Result.addFnAttr(Kind: llvm::Attribute::NoMerge);
3276
3277 // Attach readnone and readonly attributes.
3278 if (!HasSideEffect) {
3279 if (ReadNone)
3280 Result.setDoesNotAccessMemory();
3281 else if (ReadOnly)
3282 Result.setOnlyReadsMemory();
3283 }
3284
3285 // Add elementtype attribute for indirect constraints.
3286 for (auto Pair : llvm::enumerate(First&: ArgElemTypes)) {
3287 if (Pair.value()) {
3288 auto Attr = llvm::Attribute::get(
3289 Context&: getLLVMContext(), Kind: llvm::Attribute::ElementType, Ty: Pair.value());
3290 Result.addParamAttr(ArgNo: Pair.index(), Attr);
3291 }
3292 }
3293
3294 // Slap the source location of the inline asm into a !srcloc metadata on the
3295 // call.
3296 const StringLiteral *SL;
3297 if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(Val: &S);
3298 gccAsmStmt &&
3299 (SL = dyn_cast<StringLiteral>(Val: gccAsmStmt->getAsmStringExpr()))) {
3300 Result.setMetadata(Kind: "srcloc", Node: getAsmSrcLocInfo(Str: SL, CGF));
3301 } else {
3302 // At least put the line number on MS inline asm blobs and GCC asm constexpr
3303 // strings.
3304 llvm::Constant *Loc =
3305 llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: S.getAsmLoc().getRawEncoding());
3306 Result.setMetadata(Kind: "srcloc",
3307 Node: llvm::MDNode::get(Context&: getLLVMContext(),
3308 MDs: llvm::ConstantAsMetadata::get(C: Loc)));
3309 }
3310
3311 // Make inline-asm calls Key for the debug info feature Key Instructions.
3312 CGF.addInstToNewSourceAtom(KeyInstruction: &Result, Backup: nullptr);
3313
3314 if (!NoConvergent && getLangOpts().assumeFunctionsAreConvergent())
3315 // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
3316 // convergent (meaning, they may call an intrinsically convergent op, such
3317 // as bar.sync, and so can't have certain optimizations applied around
3318 // them) unless it's explicitly marked 'noconvergent'.
3319 Result.addFnAttr(Kind: llvm::Attribute::Convergent);
3320
3321 // Extract all of the register value results from the asm.
3322 if (ResultRegTypes.size() == 1) {
3323 RegResults.push_back(x: &Result);
3324 } else {
3325 for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
3326 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: &Result, Idxs: i, Name: "asmresult");
3327 RegResults.push_back(x: Tmp);
3328 }
3329 }
3330}
3331
3332void AsmConstraintsInfo::EmitAsmStores(
3333 const llvm::ArrayRef<llvm::Value *> RegResults) {
3334 llvm::LLVMContext &CTX = getLLVMContext();
3335
3336 assert(RegResults.size() == ResultRegTypes.size());
3337 assert(RegResults.size() == ResultTruncRegTypes.size());
3338 assert(RegResults.size() == ResultRegDests.size());
3339
3340 // ResultRegDests can also be populated by addReturnRegisterOutputs() above,
3341 // in which case its size may grow.
3342 assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
3343 assert(ResultBounds.size() <= ResultRegDests.size());
3344
3345 for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
3346 llvm::Value *Tmp = RegResults[i];
3347 llvm::Type *TruncTy = ResultTruncRegTypes[i];
3348
3349 if (i < ResultBounds.size() && ResultBounds[i].has_value()) {
3350 const auto [LowerBound, UpperBound] = ResultBounds[i].value();
3351
3352 // FIXME: Support for nonzero lower bounds not yet implemented.
3353 assert(LowerBound == 0 && "Output operand lower bound is not zero.");
3354
3355 llvm::Constant *UpperBoundConst =
3356 llvm::ConstantInt::get(Ty: Tmp->getType(), V: UpperBound);
3357 llvm::Value *IsBooleanValue =
3358 Builder.CreateCmp(Pred: llvm::CmpInst::ICMP_ULT, LHS: Tmp, RHS: UpperBoundConst);
3359 llvm::Function *FnAssume = CGM.getIntrinsic(IID: llvm::Intrinsic::assume);
3360
3361 Builder.CreateCall(Callee: FnAssume, Args: IsBooleanValue);
3362 }
3363
3364 // If the result type of the LLVM IR asm doesn't match the result type of
3365 // the expression, do the conversion.
3366 if (ResultRegTypes[i] != TruncTy) {
3367 // Truncate the integer result to the right size, note that TruncTy can be
3368 // a pointer.
3369 if (TruncTy->isFloatingPointTy())
3370 Tmp = Builder.CreateFPTrunc(V: Tmp, DestTy: TruncTy);
3371 else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
3372 uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(Ty: TruncTy);
3373 Tmp = Builder.CreateTrunc(
3374 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)ResSize));
3375 Tmp = Builder.CreateIntToPtr(V: Tmp, DestTy: TruncTy);
3376 } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
3377 uint64_t TmpSize =
3378 CGM.getDataLayout().getTypeSizeInBits(Ty: Tmp->getType());
3379 Tmp = Builder.CreatePtrToInt(
3380 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)TmpSize));
3381 Tmp = Builder.CreateTrunc(V: Tmp, DestTy: TruncTy);
3382 } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
3383 Tmp = Builder.CreateZExtOrTrunc(V: Tmp, DestTy: TruncTy);
3384 } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
3385 Tmp = Builder.CreateBitCast(V: Tmp, DestTy: TruncTy);
3386 }
3387 }
3388
3389 ApplyAtomGroup Grp(CGF.getDebugInfo());
3390 LValue Dest = ResultRegDests[i];
3391
3392 // ResultTypeRequiresCast elements correspond to the first
3393 // ResultTypeRequiresCast.size() elements of RegResults.
3394 if (i < ResultTypeRequiresCast.size() && ResultTypeRequiresCast[i]) {
3395 unsigned Size = getContext().getTypeSize(T: ResultRegQualTys[i]);
3396 Address A = Dest.getAddress().withElementType(ElemTy: ResultRegTypes[i]);
3397
3398 if (getTargetHooks().isScalarizableAsmOperand(CGF, Ty: TruncTy)) {
3399 llvm::StoreInst *S = Builder.CreateStore(Val: Tmp, Addr: A);
3400 CGF.addInstToCurrentSourceAtom(KeyInstruction: S, Backup: S->getValueOperand());
3401 continue;
3402 }
3403
3404 QualType Ty = getContext().getIntTypeForBitwidth(DestWidth: Size, /*Signed=*/false);
3405 if (Ty.isNull()) {
3406 const Expr *OutExpr = S.getOutputExpr(i);
3407 CGM.getDiags().Report(Loc: OutExpr->getExprLoc(),
3408 DiagID: diag::err_store_value_to_reg);
3409 return;
3410 }
3411
3412 Dest = CGF.MakeAddrLValue(Addr: A, T: Ty);
3413 }
3414
3415 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Tmp), Dst: Dest);
3416 }
3417}
3418
3419LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
3420 const RecordDecl *RD = S.getCapturedRecordDecl();
3421 CanQualType RecordTy = getContext().getCanonicalTagType(TD: RD);
3422
3423 // Initialize the captured struct.
3424 LValue SlotLV =
3425 MakeAddrLValue(Addr: CreateMemTemp(T: RecordTy, Name: "agg.captured"), T: RecordTy);
3426
3427 RecordDecl::field_iterator CurField = RD->field_begin();
3428 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
3429 E = S.capture_init_end();
3430 I != E; ++I, ++CurField) {
3431 LValue LV = EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
3432 if (CurField->hasCapturedVLAType()) {
3433 EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
3434 } else {
3435 EmitInitializerForField(Field: *CurField, LHS: LV, Init: *I);
3436 }
3437 }
3438
3439 return SlotLV;
3440}
3441
3442/// Generate an outlined function for the body of a CapturedStmt, store any
3443/// captured variables into the captured struct, and call the outlined function.
3444llvm::Function *
3445CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
3446 LValue CapStruct = InitCapturedStruct(S);
3447
3448 // Emit the CapturedDecl
3449 CodeGenFunction CGF(CGM, true);
3450 CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
3451 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
3452 delete CGF.CapturedStmtInfo;
3453
3454 // Emit call to the helper function.
3455 EmitCallOrInvoke(Callee: F, Args: CapStruct.getPointer(CGF&: *this));
3456
3457 return F;
3458}
3459
3460Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
3461 LValue CapStruct = InitCapturedStruct(S);
3462 return CapStruct.getAddress();
3463}
3464
3465/// Creates the outlined function for a CapturedStmt.
3466llvm::Function *
3467CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
3468 assert(CapturedStmtInfo &&
3469 "CapturedStmtInfo should be set when generating the captured function");
3470 const CapturedDecl *CD = S.getCapturedDecl();
3471 const RecordDecl *RD = S.getCapturedRecordDecl();
3472 SourceLocation Loc = S.getBeginLoc();
3473 assert(CD->hasBody() && "missing CapturedDecl body");
3474
3475 // Build the argument list.
3476 ASTContext &Ctx = CGM.getContext();
3477 FunctionArgList Args;
3478 Args.append(in_start: CD->param_begin(), in_end: CD->param_end());
3479
3480 // Create the function declaration.
3481 const CGFunctionInfo &FuncInfo =
3482 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
3483 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
3484
3485 llvm::Function *F =
3486 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
3487 N: CapturedStmtInfo->getHelperName(), M: &CGM.getModule());
3488 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
3489 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3490 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3491 if (CD->isNothrow())
3492 F->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3493
3494 // Generate the function.
3495 StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc: CD->getLocation(),
3496 StartLoc: CD->getBody()->getBeginLoc());
3497 // Set the context parameter in CapturedStmtInfo.
3498 Address DeclPtr = GetAddrOfLocalVar(VD: CD->getContextParam());
3499 CapturedStmtInfo->setContextValue(Builder.CreateLoad(Addr: DeclPtr));
3500
3501 // Initialize variable-length arrays.
3502 LValue Base = MakeNaturalAlignRawAddrLValue(
3503 V: CapturedStmtInfo->getContextValue(), T: Ctx.getCanonicalTagType(TD: RD));
3504 for (auto *FD : RD->fields()) {
3505 if (FD->hasCapturedVLAType()) {
3506 auto *ExprArg =
3507 EmitLoadOfLValue(V: EmitLValueForField(Base, Field: FD), Loc: S.getBeginLoc())
3508 .getScalarVal();
3509 auto VAT = FD->getCapturedVLAType();
3510 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
3511 }
3512 }
3513
3514 // If 'this' is captured, load it into CXXThisValue.
3515 if (CapturedStmtInfo->isCXXThisExprCaptured()) {
3516 FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
3517 LValue ThisLValue = EmitLValueForField(Base, Field: FD);
3518 CXXThisValue = EmitLoadOfLValue(V: ThisLValue, Loc).getScalarVal();
3519 }
3520
3521 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
3522 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
3523 FinishFunction(EndLoc: CD->getBodyRBrace());
3524
3525 return F;
3526}
3527
3528// Returns the first convergence entry/loop/anchor instruction found in |BB|.
3529// std::nullptr otherwise.
3530static llvm::ConvergenceControlInst *getConvergenceToken(llvm::BasicBlock *BB) {
3531 for (auto &I : *BB) {
3532 if (auto *CI = dyn_cast<llvm::ConvergenceControlInst>(Val: &I))
3533 return CI;
3534 }
3535 return nullptr;
3536}
3537
3538llvm::CallBase *
3539CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input) {
3540 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3541 assert(ParentToken);
3542
3543 llvm::Value *bundleArgs[] = {ParentToken};
3544 llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
3545 auto *Output = llvm::CallBase::addOperandBundle(
3546 CB: Input, ID: llvm::LLVMContext::OB_convergencectrl, OB, InsertPt: Input->getIterator());
3547 Input->replaceAllUsesWith(V: Output);
3548 Input->eraseFromParent();
3549 return Output;
3550}
3551
3552llvm::ConvergenceControlInst *
3553CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {
3554 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3555 assert(ParentToken);
3556 return llvm::ConvergenceControlInst::CreateLoop(BB&: *BB, Parent: ParentToken);
3557}
3558
3559llvm::ConvergenceControlInst *
3560CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
3561 llvm::BasicBlock *BB = &F->getEntryBlock();
3562 llvm::ConvergenceControlInst *Token = getConvergenceToken(BB);
3563 if (Token)
3564 return Token;
3565
3566 // Adding a convergence token requires the function to be marked as
3567 // convergent.
3568 F->setConvergent();
3569 return llvm::ConvergenceControlInst::CreateEntry(BB&: *BB);
3570}
3571