1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Stmt nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGDebugInfo.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "CodeGenPGO.h"
18#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Expr.h"
21#include "clang/AST/Stmt.h"
22#include "clang/AST/StmtSYCL.h"
23#include "clang/AST/StmtVisitor.h"
24#include "clang/Basic/Builtins.h"
25#include "clang/Basic/DiagnosticSema.h"
26#include "clang/Basic/PrettyStackTrace.h"
27#include "clang/Basic/SourceManager.h"
28#include "clang/Basic/TargetInfo.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/IR/Assumptions.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/InlineAsm.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/MDBuilder.h"
38#include "llvm/Support/SaveAndRestore.h"
39#include <optional>
40
41using namespace clang;
42using namespace CodeGen;
43
44//===----------------------------------------------------------------------===//
45// Statement Emission
46//===----------------------------------------------------------------------===//
47
48void CodeGenFunction::EmitStopPoint(const Stmt *S) {
49 if (CGDebugInfo *DI = getDebugInfo()) {
50 SourceLocation Loc;
51 Loc = S->getBeginLoc();
52 DI->EmitLocation(Builder, Loc);
53
54 LastStopPoint = Loc;
55 }
56}
57
58void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
59 assert(S && "Null statement?");
60 PGO->setCurrentStmt(S);
61
62 // These statements have their own debug info handling.
63 if (EmitSimpleStmt(S, Attrs))
64 return;
65
66 // Check if we are generating unreachable code.
67 if (!HaveInsertPoint()) {
68 // If so, and the statement doesn't contain a label, then we do not need to
69 // generate actual code. This is safe because (1) the current point is
70 // unreachable, so we don't need to execute the code, and (2) we've already
71 // handled the statements which update internal data structures (like the
72 // local variable map) which could be used by subsequent statements.
73 if (!ContainsLabel(S)) {
74 // Verify that any decl statements were handled as simple, they may be in
75 // scope of subsequent reachable statements.
76 assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
77 PGO->markStmtMaybeUsed(S);
78 return;
79 }
80
81 // Otherwise, make a new block to hold the code.
82 EnsureInsertPoint();
83 }
84
85 // Generate a stoppoint if we are emitting debug info.
86 EmitStopPoint(S);
87
88 // Ignore all OpenMP directives except for simd if OpenMP with Simd is
89 // enabled.
90 if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
91 if (const auto *D = dyn_cast<OMPExecutableDirective>(Val: S)) {
92 EmitSimpleOMPExecutableDirective(D: *D);
93 return;
94 }
95 }
96
97 switch (S->getStmtClass()) {
98 case Stmt::NoStmtClass:
99 case Stmt::CXXCatchStmtClass:
100 case Stmt::SEHExceptStmtClass:
101 case Stmt::SEHFinallyStmtClass:
102 case Stmt::MSDependentExistsStmtClass:
103 case Stmt::UnresolvedSYCLKernelCallStmtClass:
104 llvm_unreachable("invalid statement class to emit generically");
105 case Stmt::NullStmtClass:
106 case Stmt::CompoundStmtClass:
107 case Stmt::DeclStmtClass:
108 case Stmt::LabelStmtClass:
109 case Stmt::AttributedStmtClass:
110 case Stmt::GotoStmtClass:
111 case Stmt::BreakStmtClass:
112 case Stmt::ContinueStmtClass:
113 case Stmt::DefaultStmtClass:
114 case Stmt::CaseStmtClass:
115 case Stmt::DeferStmtClass:
116 case Stmt::SEHLeaveStmtClass:
117 case Stmt::SYCLKernelCallStmtClass:
118 llvm_unreachable("should have emitted these statements as simple");
119
120#define STMT(Type, Base)
121#define ABSTRACT_STMT(Op)
122#define EXPR(Type, Base) \
123 case Stmt::Type##Class:
124#include "clang/AST/StmtNodes.inc"
125 {
126 // Remember the block we came in on.
127 llvm::BasicBlock *incoming = Builder.GetInsertBlock();
128 assert(incoming && "expression emission must have an insertion point");
129
130 EmitIgnoredExpr(E: cast<Expr>(Val: S));
131
132 llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
133 assert(outgoing && "expression emission cleared block!");
134
135 // The expression emitters assume (reasonably!) that the insertion
136 // point is always set. To maintain that, the call-emission code
137 // for noreturn functions has to enter a new block with no
138 // predecessors. We want to kill that block and mark the current
139 // insertion point unreachable in the common case of a call like
140 // "exit();". Since expression emission doesn't otherwise create
141 // blocks with no predecessors, we can just test for that.
142 // However, we must be careful not to do this to our incoming
143 // block, because *statement* emission does sometimes create
144 // reachable blocks which will have no predecessors until later in
145 // the function. This occurs with, e.g., labels that are not
146 // reachable by fallthrough.
147 if (incoming != outgoing && outgoing->use_empty()) {
148 outgoing->eraseFromParent();
149 Builder.ClearInsertionPoint();
150 }
151 break;
152 }
153
154 case Stmt::IndirectGotoStmtClass:
155 EmitIndirectGotoStmt(S: cast<IndirectGotoStmt>(Val: *S)); break;
156
157 case Stmt::IfStmtClass: EmitIfStmt(S: cast<IfStmt>(Val: *S)); break;
158 case Stmt::WhileStmtClass: EmitWhileStmt(S: cast<WhileStmt>(Val: *S), Attrs); break;
159 case Stmt::DoStmtClass: EmitDoStmt(S: cast<DoStmt>(Val: *S), Attrs); break;
160 case Stmt::ForStmtClass: EmitForStmt(S: cast<ForStmt>(Val: *S), Attrs); break;
161
162 case Stmt::ReturnStmtClass: EmitReturnStmt(S: cast<ReturnStmt>(Val: *S)); break;
163
164 case Stmt::SwitchStmtClass: EmitSwitchStmt(S: cast<SwitchStmt>(Val: *S)); break;
165 case Stmt::GCCAsmStmtClass: // Intentional fall-through.
166 case Stmt::MSAsmStmtClass: EmitAsmStmt(S: cast<AsmStmt>(Val: *S)); break;
167 case Stmt::CoroutineBodyStmtClass:
168 EmitCoroutineBody(S: cast<CoroutineBodyStmt>(Val: *S));
169 break;
170 case Stmt::CoreturnStmtClass:
171 EmitCoreturnStmt(S: cast<CoreturnStmt>(Val: *S));
172 break;
173 case Stmt::CapturedStmtClass: {
174 const CapturedStmt *CS = cast<CapturedStmt>(Val: S);
175 EmitCapturedStmt(S: *CS, K: CS->getCapturedRegionKind());
176 }
177 break;
178 case Stmt::ObjCAtTryStmtClass:
179 EmitObjCAtTryStmt(S: cast<ObjCAtTryStmt>(Val: *S));
180 break;
181 case Stmt::ObjCAtCatchStmtClass:
182 llvm_unreachable(
183 "@catch statements should be handled by EmitObjCAtTryStmt");
184 case Stmt::ObjCAtFinallyStmtClass:
185 llvm_unreachable(
186 "@finally statements should be handled by EmitObjCAtTryStmt");
187 case Stmt::ObjCAtThrowStmtClass:
188 EmitObjCAtThrowStmt(S: cast<ObjCAtThrowStmt>(Val: *S));
189 break;
190 case Stmt::ObjCAtSynchronizedStmtClass:
191 EmitObjCAtSynchronizedStmt(S: cast<ObjCAtSynchronizedStmt>(Val: *S));
192 break;
193 case Stmt::ObjCForCollectionStmtClass:
194 EmitObjCForCollectionStmt(S: cast<ObjCForCollectionStmt>(Val: *S));
195 break;
196 case Stmt::ObjCAutoreleasePoolStmtClass:
197 EmitObjCAutoreleasePoolStmt(S: cast<ObjCAutoreleasePoolStmt>(Val: *S));
198 break;
199
200 case Stmt::CXXTryStmtClass:
201 EmitCXXTryStmt(S: cast<CXXTryStmt>(Val: *S));
202 break;
203 case Stmt::CXXForRangeStmtClass:
204 EmitCXXForRangeStmt(S: cast<CXXForRangeStmt>(Val: *S), Attrs);
205 break;
206 case Stmt::SEHTryStmtClass:
207 EmitSEHTryStmt(S: cast<SEHTryStmt>(Val: *S));
208 break;
209 case Stmt::OMPMetaDirectiveClass:
210 EmitOMPMetaDirective(S: cast<OMPMetaDirective>(Val: *S));
211 break;
212 case Stmt::OMPCanonicalLoopClass:
213 EmitOMPCanonicalLoop(S: cast<OMPCanonicalLoop>(Val: S));
214 break;
215 case Stmt::OMPParallelDirectiveClass:
216 EmitOMPParallelDirective(S: cast<OMPParallelDirective>(Val: *S));
217 break;
218 case Stmt::OMPSimdDirectiveClass:
219 EmitOMPSimdDirective(S: cast<OMPSimdDirective>(Val: *S));
220 break;
221 case Stmt::OMPTileDirectiveClass:
222 EmitOMPTileDirective(S: cast<OMPTileDirective>(Val: *S));
223 break;
224 case Stmt::OMPStripeDirectiveClass:
225 EmitOMPStripeDirective(S: cast<OMPStripeDirective>(Val: *S));
226 break;
227 case Stmt::OMPUnrollDirectiveClass:
228 EmitOMPUnrollDirective(S: cast<OMPUnrollDirective>(Val: *S));
229 break;
230 case Stmt::OMPReverseDirectiveClass:
231 EmitOMPReverseDirective(S: cast<OMPReverseDirective>(Val: *S));
232 break;
233 case Stmt::OMPSplitDirectiveClass:
234 EmitOMPSplitDirective(S: cast<OMPSplitDirective>(Val: *S));
235 break;
236 case Stmt::OMPInterchangeDirectiveClass:
237 EmitOMPInterchangeDirective(S: cast<OMPInterchangeDirective>(Val: *S));
238 break;
239 case Stmt::OMPFuseDirectiveClass:
240 EmitOMPFuseDirective(S: cast<OMPFuseDirective>(Val: *S));
241 break;
242 case Stmt::OMPForDirectiveClass:
243 EmitOMPForDirective(S: cast<OMPForDirective>(Val: *S));
244 break;
245 case Stmt::OMPForSimdDirectiveClass:
246 EmitOMPForSimdDirective(S: cast<OMPForSimdDirective>(Val: *S));
247 break;
248 case Stmt::OMPSectionsDirectiveClass:
249 EmitOMPSectionsDirective(S: cast<OMPSectionsDirective>(Val: *S));
250 break;
251 case Stmt::OMPSectionDirectiveClass:
252 EmitOMPSectionDirective(S: cast<OMPSectionDirective>(Val: *S));
253 break;
254 case Stmt::OMPSingleDirectiveClass:
255 EmitOMPSingleDirective(S: cast<OMPSingleDirective>(Val: *S));
256 break;
257 case Stmt::OMPMasterDirectiveClass:
258 EmitOMPMasterDirective(S: cast<OMPMasterDirective>(Val: *S));
259 break;
260 case Stmt::OMPCriticalDirectiveClass:
261 EmitOMPCriticalDirective(S: cast<OMPCriticalDirective>(Val: *S));
262 break;
263 case Stmt::OMPParallelForDirectiveClass:
264 EmitOMPParallelForDirective(S: cast<OMPParallelForDirective>(Val: *S));
265 break;
266 case Stmt::OMPParallelForSimdDirectiveClass:
267 EmitOMPParallelForSimdDirective(S: cast<OMPParallelForSimdDirective>(Val: *S));
268 break;
269 case Stmt::OMPParallelMasterDirectiveClass:
270 EmitOMPParallelMasterDirective(S: cast<OMPParallelMasterDirective>(Val: *S));
271 break;
272 case Stmt::OMPParallelSectionsDirectiveClass:
273 EmitOMPParallelSectionsDirective(S: cast<OMPParallelSectionsDirective>(Val: *S));
274 break;
275 case Stmt::OMPTaskDirectiveClass:
276 EmitOMPTaskDirective(S: cast<OMPTaskDirective>(Val: *S));
277 break;
278 case Stmt::OMPTaskyieldDirectiveClass:
279 EmitOMPTaskyieldDirective(S: cast<OMPTaskyieldDirective>(Val: *S));
280 break;
281 case Stmt::OMPErrorDirectiveClass:
282 EmitOMPErrorDirective(S: cast<OMPErrorDirective>(Val: *S));
283 break;
284 case Stmt::OMPBarrierDirectiveClass:
285 EmitOMPBarrierDirective(S: cast<OMPBarrierDirective>(Val: *S));
286 break;
287 case Stmt::OMPTaskwaitDirectiveClass:
288 EmitOMPTaskwaitDirective(S: cast<OMPTaskwaitDirective>(Val: *S));
289 break;
290 case Stmt::OMPTaskgroupDirectiveClass:
291 EmitOMPTaskgroupDirective(S: cast<OMPTaskgroupDirective>(Val: *S));
292 break;
293 case Stmt::OMPFlushDirectiveClass:
294 EmitOMPFlushDirective(S: cast<OMPFlushDirective>(Val: *S));
295 break;
296 case Stmt::OMPDepobjDirectiveClass:
297 EmitOMPDepobjDirective(S: cast<OMPDepobjDirective>(Val: *S));
298 break;
299 case Stmt::OMPScanDirectiveClass:
300 EmitOMPScanDirective(S: cast<OMPScanDirective>(Val: *S));
301 break;
302 case Stmt::OMPOrderedDirectiveClass:
303 EmitOMPOrderedDirective(S: cast<OMPOrderedDirective>(Val: *S));
304 break;
305 case Stmt::OMPAtomicDirectiveClass:
306 EmitOMPAtomicDirective(S: cast<OMPAtomicDirective>(Val: *S));
307 break;
308 case Stmt::OMPTargetDirectiveClass:
309 EmitOMPTargetDirective(S: cast<OMPTargetDirective>(Val: *S));
310 break;
311 case Stmt::OMPTeamsDirectiveClass:
312 EmitOMPTeamsDirective(S: cast<OMPTeamsDirective>(Val: *S));
313 break;
314 case Stmt::OMPCancellationPointDirectiveClass:
315 EmitOMPCancellationPointDirective(S: cast<OMPCancellationPointDirective>(Val: *S));
316 break;
317 case Stmt::OMPCancelDirectiveClass:
318 EmitOMPCancelDirective(S: cast<OMPCancelDirective>(Val: *S));
319 break;
320 case Stmt::OMPTargetDataDirectiveClass:
321 EmitOMPTargetDataDirective(S: cast<OMPTargetDataDirective>(Val: *S));
322 break;
323 case Stmt::OMPTargetEnterDataDirectiveClass:
324 EmitOMPTargetEnterDataDirective(S: cast<OMPTargetEnterDataDirective>(Val: *S));
325 break;
326 case Stmt::OMPTargetExitDataDirectiveClass:
327 EmitOMPTargetExitDataDirective(S: cast<OMPTargetExitDataDirective>(Val: *S));
328 break;
329 case Stmt::OMPTargetParallelDirectiveClass:
330 EmitOMPTargetParallelDirective(S: cast<OMPTargetParallelDirective>(Val: *S));
331 break;
332 case Stmt::OMPTargetParallelForDirectiveClass:
333 EmitOMPTargetParallelForDirective(S: cast<OMPTargetParallelForDirective>(Val: *S));
334 break;
335 case Stmt::OMPTaskLoopDirectiveClass:
336 EmitOMPTaskLoopDirective(S: cast<OMPTaskLoopDirective>(Val: *S));
337 break;
338 case Stmt::OMPTaskLoopSimdDirectiveClass:
339 EmitOMPTaskLoopSimdDirective(S: cast<OMPTaskLoopSimdDirective>(Val: *S));
340 break;
341 case Stmt::OMPMasterTaskLoopDirectiveClass:
342 EmitOMPMasterTaskLoopDirective(S: cast<OMPMasterTaskLoopDirective>(Val: *S));
343 break;
344 case Stmt::OMPMaskedTaskLoopDirectiveClass:
345 EmitOMPMaskedTaskLoopDirective(S: cast<OMPMaskedTaskLoopDirective>(Val: *S));
346 break;
347 case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
348 EmitOMPMasterTaskLoopSimdDirective(
349 S: cast<OMPMasterTaskLoopSimdDirective>(Val: *S));
350 break;
351 case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
352 EmitOMPMaskedTaskLoopSimdDirective(
353 S: cast<OMPMaskedTaskLoopSimdDirective>(Val: *S));
354 break;
355 case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
356 EmitOMPParallelMasterTaskLoopDirective(
357 S: cast<OMPParallelMasterTaskLoopDirective>(Val: *S));
358 break;
359 case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
360 EmitOMPParallelMaskedTaskLoopDirective(
361 S: cast<OMPParallelMaskedTaskLoopDirective>(Val: *S));
362 break;
363 case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
364 EmitOMPParallelMasterTaskLoopSimdDirective(
365 S: cast<OMPParallelMasterTaskLoopSimdDirective>(Val: *S));
366 break;
367 case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
368 EmitOMPParallelMaskedTaskLoopSimdDirective(
369 S: cast<OMPParallelMaskedTaskLoopSimdDirective>(Val: *S));
370 break;
371 case Stmt::OMPDistributeDirectiveClass:
372 EmitOMPDistributeDirective(S: cast<OMPDistributeDirective>(Val: *S));
373 break;
374 case Stmt::OMPTargetUpdateDirectiveClass:
375 EmitOMPTargetUpdateDirective(S: cast<OMPTargetUpdateDirective>(Val: *S));
376 break;
377 case Stmt::OMPDistributeParallelForDirectiveClass:
378 EmitOMPDistributeParallelForDirective(
379 S: cast<OMPDistributeParallelForDirective>(Val: *S));
380 break;
381 case Stmt::OMPDistributeParallelForSimdDirectiveClass:
382 EmitOMPDistributeParallelForSimdDirective(
383 S: cast<OMPDistributeParallelForSimdDirective>(Val: *S));
384 break;
385 case Stmt::OMPDistributeSimdDirectiveClass:
386 EmitOMPDistributeSimdDirective(S: cast<OMPDistributeSimdDirective>(Val: *S));
387 break;
388 case Stmt::OMPTargetParallelForSimdDirectiveClass:
389 EmitOMPTargetParallelForSimdDirective(
390 S: cast<OMPTargetParallelForSimdDirective>(Val: *S));
391 break;
392 case Stmt::OMPTargetSimdDirectiveClass:
393 EmitOMPTargetSimdDirective(S: cast<OMPTargetSimdDirective>(Val: *S));
394 break;
395 case Stmt::OMPTeamsDistributeDirectiveClass:
396 EmitOMPTeamsDistributeDirective(S: cast<OMPTeamsDistributeDirective>(Val: *S));
397 break;
398 case Stmt::OMPTeamsDistributeSimdDirectiveClass:
399 EmitOMPTeamsDistributeSimdDirective(
400 S: cast<OMPTeamsDistributeSimdDirective>(Val: *S));
401 break;
402 case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
403 EmitOMPTeamsDistributeParallelForSimdDirective(
404 S: cast<OMPTeamsDistributeParallelForSimdDirective>(Val: *S));
405 break;
406 case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
407 EmitOMPTeamsDistributeParallelForDirective(
408 S: cast<OMPTeamsDistributeParallelForDirective>(Val: *S));
409 break;
410 case Stmt::OMPTargetTeamsDirectiveClass:
411 EmitOMPTargetTeamsDirective(S: cast<OMPTargetTeamsDirective>(Val: *S));
412 break;
413 case Stmt::OMPTargetTeamsDistributeDirectiveClass:
414 EmitOMPTargetTeamsDistributeDirective(
415 S: cast<OMPTargetTeamsDistributeDirective>(Val: *S));
416 break;
417 case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
418 EmitOMPTargetTeamsDistributeParallelForDirective(
419 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: *S));
420 break;
421 case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
422 EmitOMPTargetTeamsDistributeParallelForSimdDirective(
423 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: *S));
424 break;
425 case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
426 EmitOMPTargetTeamsDistributeSimdDirective(
427 S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: *S));
428 break;
429 case Stmt::OMPInteropDirectiveClass:
430 EmitOMPInteropDirective(S: cast<OMPInteropDirective>(Val: *S));
431 break;
432 case Stmt::OMPDispatchDirectiveClass:
433 CGM.ErrorUnsupported(S, Type: "OpenMP dispatch directive");
434 break;
435 case Stmt::OMPScopeDirectiveClass:
436 EmitOMPScopeDirective(S: cast<OMPScopeDirective>(Val: *S));
437 break;
438 case Stmt::OMPMaskedDirectiveClass:
439 EmitOMPMaskedDirective(S: cast<OMPMaskedDirective>(Val: *S));
440 break;
441 case Stmt::OMPGenericLoopDirectiveClass:
442 EmitOMPGenericLoopDirective(S: cast<OMPGenericLoopDirective>(Val: *S));
443 break;
444 case Stmt::OMPTeamsGenericLoopDirectiveClass:
445 EmitOMPTeamsGenericLoopDirective(S: cast<OMPTeamsGenericLoopDirective>(Val: *S));
446 break;
447 case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
448 EmitOMPTargetTeamsGenericLoopDirective(
449 S: cast<OMPTargetTeamsGenericLoopDirective>(Val: *S));
450 break;
451 case Stmt::OMPParallelGenericLoopDirectiveClass:
452 EmitOMPParallelGenericLoopDirective(
453 S: cast<OMPParallelGenericLoopDirective>(Val: *S));
454 break;
455 case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
456 EmitOMPTargetParallelGenericLoopDirective(
457 S: cast<OMPTargetParallelGenericLoopDirective>(Val: *S));
458 break;
459 case Stmt::OMPParallelMaskedDirectiveClass:
460 EmitOMPParallelMaskedDirective(S: cast<OMPParallelMaskedDirective>(Val: *S));
461 break;
462 case Stmt::OMPAssumeDirectiveClass:
463 EmitOMPAssumeDirective(S: cast<OMPAssumeDirective>(Val: *S));
464 break;
465 case Stmt::OpenACCComputeConstructClass:
466 EmitOpenACCComputeConstruct(S: cast<OpenACCComputeConstruct>(Val: *S));
467 break;
468 case Stmt::OpenACCLoopConstructClass:
469 EmitOpenACCLoopConstruct(S: cast<OpenACCLoopConstruct>(Val: *S));
470 break;
471 case Stmt::OpenACCCombinedConstructClass:
472 EmitOpenACCCombinedConstruct(S: cast<OpenACCCombinedConstruct>(Val: *S));
473 break;
474 case Stmt::OpenACCDataConstructClass:
475 EmitOpenACCDataConstruct(S: cast<OpenACCDataConstruct>(Val: *S));
476 break;
477 case Stmt::OpenACCEnterDataConstructClass:
478 EmitOpenACCEnterDataConstruct(S: cast<OpenACCEnterDataConstruct>(Val: *S));
479 break;
480 case Stmt::OpenACCExitDataConstructClass:
481 EmitOpenACCExitDataConstruct(S: cast<OpenACCExitDataConstruct>(Val: *S));
482 break;
483 case Stmt::OpenACCHostDataConstructClass:
484 EmitOpenACCHostDataConstruct(S: cast<OpenACCHostDataConstruct>(Val: *S));
485 break;
486 case Stmt::OpenACCWaitConstructClass:
487 EmitOpenACCWaitConstruct(S: cast<OpenACCWaitConstruct>(Val: *S));
488 break;
489 case Stmt::OpenACCInitConstructClass:
490 EmitOpenACCInitConstruct(S: cast<OpenACCInitConstruct>(Val: *S));
491 break;
492 case Stmt::OpenACCShutdownConstructClass:
493 EmitOpenACCShutdownConstruct(S: cast<OpenACCShutdownConstruct>(Val: *S));
494 break;
495 case Stmt::OpenACCSetConstructClass:
496 EmitOpenACCSetConstruct(S: cast<OpenACCSetConstruct>(Val: *S));
497 break;
498 case Stmt::OpenACCUpdateConstructClass:
499 EmitOpenACCUpdateConstruct(S: cast<OpenACCUpdateConstruct>(Val: *S));
500 break;
501 case Stmt::OpenACCAtomicConstructClass:
502 EmitOpenACCAtomicConstruct(S: cast<OpenACCAtomicConstruct>(Val: *S));
503 break;
504 case Stmt::OpenACCCacheConstructClass:
505 EmitOpenACCCacheConstruct(S: cast<OpenACCCacheConstruct>(Val: *S));
506 break;
507 }
508}
509
510bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
511 ArrayRef<const Attr *> Attrs) {
512 switch (S->getStmtClass()) {
513 default:
514 return false;
515 case Stmt::NullStmtClass:
516 break;
517 case Stmt::CompoundStmtClass:
518 EmitCompoundStmt(S: cast<CompoundStmt>(Val: *S));
519 break;
520 case Stmt::DeclStmtClass:
521 EmitDeclStmt(S: cast<DeclStmt>(Val: *S));
522 break;
523 case Stmt::LabelStmtClass:
524 EmitLabelStmt(S: cast<LabelStmt>(Val: *S));
525 break;
526 case Stmt::AttributedStmtClass:
527 EmitAttributedStmt(S: cast<AttributedStmt>(Val: *S));
528 break;
529 case Stmt::GotoStmtClass:
530 EmitGotoStmt(S: cast<GotoStmt>(Val: *S));
531 break;
532 case Stmt::BreakStmtClass:
533 EmitBreakStmt(S: cast<BreakStmt>(Val: *S));
534 break;
535 case Stmt::ContinueStmtClass:
536 EmitContinueStmt(S: cast<ContinueStmt>(Val: *S));
537 break;
538 case Stmt::DefaultStmtClass:
539 EmitDefaultStmt(S: cast<DefaultStmt>(Val: *S), Attrs);
540 break;
541 case Stmt::CaseStmtClass:
542 EmitCaseStmt(S: cast<CaseStmt>(Val: *S), Attrs);
543 break;
544 case Stmt::DeferStmtClass:
545 EmitDeferStmt(S: cast<DeferStmt>(Val: *S));
546 break;
547 case Stmt::SEHLeaveStmtClass:
548 EmitSEHLeaveStmt(S: cast<SEHLeaveStmt>(Val: *S));
549 break;
550 case Stmt::SYCLKernelCallStmtClass:
551 EmitSYCLKernelCallStmt(S: cast<SYCLKernelCallStmt>(Val: *S));
552 break;
553 }
554 return true;
555}
556
557/// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true,
558/// this captures the expression result of the last sub-statement and returns it
559/// (for use by the statement expression extension).
560Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
561 AggValueSlot AggSlot) {
562 PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
563 "LLVM IR generation of compound statement ('{}')");
564
565 // Keep track of the current cleanup stack depth, including debug scopes.
566 LexicalScope Scope(*this, S.getSourceRange());
567
568 return EmitCompoundStmtWithoutScope(S, GetLast, AVS: AggSlot);
569}
570
571Address
572CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
573 bool GetLast,
574 AggValueSlot AggSlot) {
575
576 for (CompoundStmt::const_body_iterator I = S.body_begin(),
577 E = S.body_end() - GetLast;
578 I != E; ++I)
579 EmitStmt(S: *I);
580
581 Address RetAlloca = Address::invalid();
582 if (GetLast) {
583 // We have to special case labels here. They are statements, but when put
584 // at the end of a statement expression, they yield the value of their
585 // subexpression. Handle this by walking through all labels we encounter,
586 // emitting them before we evaluate the subexpr.
587 // Similar issues arise for attributed statements.
588 const Stmt *LastStmt = S.body_back();
589 while (!isa<Expr>(Val: LastStmt)) {
590 if (const auto *LS = dyn_cast<LabelStmt>(Val: LastStmt)) {
591 EmitLabel(D: LS->getDecl());
592 LastStmt = LS->getSubStmt();
593 } else if (const auto *AS = dyn_cast<AttributedStmt>(Val: LastStmt)) {
594 // FIXME: Update this if we ever have attributes that affect the
595 // semantics of an expression.
596 LastStmt = AS->getSubStmt();
597 } else {
598 llvm_unreachable("unknown value statement");
599 }
600 }
601
602 EnsureInsertPoint();
603
604 const Expr *E = cast<Expr>(Val: LastStmt);
605 QualType ExprTy = E->getType();
606 if (hasAggregateEvaluationKind(T: ExprTy)) {
607 EmitAggExpr(E, AS: AggSlot);
608 } else {
609 // We can't return an RValue here because there might be cleanups at
610 // the end of the StmtExpr. Because of that, we have to emit the result
611 // here into a temporary alloca.
612 RetAlloca = CreateMemTempWithoutCast(T: ExprTy);
613 EmitAnyExprToMem(E, Location: RetAlloca, Quals: Qualifiers(),
614 /*IsInit*/ IsInitializer: false);
615 }
616 }
617
618 return RetAlloca;
619}
620
621void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
622 llvm::UncondBrInst *BI = dyn_cast<llvm::UncondBrInst>(Val: BB->getTerminator());
623
624 // If there is a cleanup stack, then we it isn't worth trying to
625 // simplify this block (we would need to remove it from the scope map
626 // and cleanup entry).
627 if (!EHStack.empty())
628 return;
629
630 // Can only simplify direct branches.
631 if (!BI)
632 return;
633
634 // Can only simplify empty blocks.
635 if (BI->getIterator() != BB->begin())
636 return;
637
638 BB->replaceAllUsesWith(V: BI->getSuccessor());
639 BI->eraseFromParent();
640 BB->eraseFromParent();
641}
642
643void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
644 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
645
646 // Fall out of the current block (if necessary).
647 EmitBranch(Block: BB);
648
649 if (IsFinished && BB->use_empty()) {
650 delete BB;
651 return;
652 }
653
654 // Place the block after the current block, if possible, or else at
655 // the end of the function.
656 if (CurBB && CurBB->getParent())
657 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
658 else
659 CurFn->insert(Position: CurFn->end(), BB);
660 Builder.SetInsertPoint(BB);
661}
662
663void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
664 // Emit a branch from the current block to the target one if this
665 // was a real block. If this was just a fall-through block after a
666 // terminator, don't emit it.
667 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
668
669 if (!CurBB || CurBB->hasTerminator()) {
670 // If there is no insert point or the previous block is already
671 // terminated, don't touch it.
672 } else {
673 // Otherwise, create a fall-through branch.
674 Builder.CreateBr(Dest: Target);
675 }
676
677 Builder.ClearInsertionPoint();
678}
679
680void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
681 bool inserted = false;
682 for (llvm::User *u : block->users()) {
683 if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(Val: u)) {
684 CurFn->insert(Position: std::next(x: insn->getParent()->getIterator()), BB: block);
685 inserted = true;
686 break;
687 }
688 }
689
690 if (!inserted)
691 CurFn->insert(Position: CurFn->end(), BB: block);
692
693 Builder.SetInsertPoint(block);
694}
695
696CodeGenFunction::JumpDest
697CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
698 JumpDest &Dest = LabelMap[D];
699 if (Dest.isValid()) return Dest;
700
701 // Create, but don't insert, the new block.
702 Dest = JumpDest(createBasicBlock(name: D->getName()),
703 EHScopeStack::stable_iterator::invalid(),
704 NextCleanupDestIndex++);
705 return Dest;
706}
707
708void CodeGenFunction::EmitLabel(const LabelDecl *D) {
709 // Add this label to the current lexical scope if we're within any
710 // normal cleanups. Jumps "in" to this label --- when permitted by
711 // the language --- may need to be routed around such cleanups.
712 if (EHStack.hasNormalCleanups() && CurLexicalScope)
713 CurLexicalScope->addLabel(label: D);
714
715 JumpDest &Dest = LabelMap[D];
716
717 // If we didn't need a forward reference to this label, just go
718 // ahead and create a destination at the current scope.
719 if (!Dest.isValid()) {
720 Dest = getJumpDestInCurrentScope(Name: D->getName());
721
722 // Otherwise, we need to give this label a target depth and remove
723 // it from the branch-fixups list.
724 } else {
725 assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
726 Dest.setScopeDepth(EHStack.stable_begin());
727 ResolveBranchFixups(Target: Dest.getBlock());
728 }
729
730 EmitBlock(BB: Dest.getBlock());
731
732 // Emit debug info for labels.
733 if (CGDebugInfo *DI = getDebugInfo()) {
734 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
735 DI->setLocation(D->getLocation());
736 DI->EmitLabel(D, Builder);
737 }
738 }
739
740 incrementProfileCounter(S: D->getStmt());
741}
742
743/// Change the cleanup scope of the labels in this lexical scope to
744/// match the scope of the enclosing context.
745void CodeGenFunction::LexicalScope::rescopeLabels() {
746 assert(!Labels.empty());
747 EHScopeStack::stable_iterator innermostScope
748 = CGF.EHStack.getInnermostNormalCleanup();
749
750 // Change the scope depth of all the labels.
751 for (const LabelDecl *Label : Labels) {
752 assert(CGF.LabelMap.count(Label));
753 JumpDest &dest = CGF.LabelMap.find(Val: Label)->second;
754 assert(dest.getScopeDepth().isValid());
755 assert(innermostScope.encloses(dest.getScopeDepth()));
756 dest.setScopeDepth(innermostScope);
757 }
758
759 // Reparent the labels if the new scope also has cleanups.
760 if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
761 ParentScope->Labels.append(in_start: Labels.begin(), in_end: Labels.end());
762 }
763}
764
765
766void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
767 EmitLabel(D: S.getDecl());
768
769 // IsEHa - emit eha.scope.begin if it's a side entry of a scope
770 if (getLangOpts().EHAsynch && S.isSideEntry())
771 EmitSehCppScopeBegin();
772
773 EmitStmt(S: S.getSubStmt());
774}
775
776void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
777 bool nomerge = false;
778 bool noinline = false;
779 bool alwaysinline = false;
780 bool noconvergent = false;
781 HLSLControlFlowHintAttr::Spelling flattenOrBranch =
782 HLSLControlFlowHintAttr::SpellingNotCalculated;
783 const CallExpr *musttail = nullptr;
784 const AtomicAttr *AA = nullptr;
785
786 for (const auto *A : S.getAttrs()) {
787 switch (A->getKind()) {
788 default:
789 break;
790 case attr::NoMerge:
791 nomerge = true;
792 break;
793 case attr::NoInline:
794 noinline = true;
795 break;
796 case attr::AlwaysInline:
797 alwaysinline = true;
798 break;
799 case attr::NoConvergent:
800 noconvergent = true;
801 break;
802 case attr::MustTail: {
803 const Stmt *Sub = S.getSubStmt();
804 const ReturnStmt *R = cast<ReturnStmt>(Val: Sub);
805 musttail = cast<CallExpr>(Val: R->getRetValue()->IgnoreParens());
806 } break;
807 case attr::CXXAssume: {
808 const Expr *Assumption = cast<CXXAssumeAttr>(Val: A)->getAssumption();
809 if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() &&
810 !Assumption->HasSideEffects(Ctx: getContext())) {
811 llvm::Value *AssumptionVal = EmitCheckedArgForAssume(E: Assumption);
812 Builder.CreateAssumption(Cond: AssumptionVal);
813 }
814 } break;
815 case attr::Atomic:
816 AA = cast<AtomicAttr>(Val: A);
817 break;
818 case attr::HLSLControlFlowHint: {
819 flattenOrBranch = cast<HLSLControlFlowHintAttr>(Val: A)->getSemanticSpelling();
820 } break;
821 }
822 }
823 SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);
824 SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);
825 SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);
826 SaveAndRestore save_noconvergent(InNoConvergentAttributedStmt, noconvergent);
827 SaveAndRestore save_musttail(MustTailCall, musttail);
828 SaveAndRestore save_flattenOrBranch(HLSLControlFlowAttr, flattenOrBranch);
829 CGAtomicOptionsRAII AORAII(CGM, AA);
830 EmitStmt(S: S.getSubStmt(), Attrs: S.getAttrs());
831}
832
833void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
834 // If this code is reachable then emit a stop point (if generating
835 // debug info). We have to do this ourselves because we are on the
836 // "simple" statement path.
837 if (HaveInsertPoint())
838 EmitStopPoint(S: &S);
839
840 ApplyAtomGroup Grp(getDebugInfo());
841 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: S.getLabel()));
842}
843
844
845void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
846 ApplyAtomGroup Grp(getDebugInfo());
847 if (const LabelDecl *Target = S.getConstantTarget()) {
848 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: Target));
849 return;
850 }
851
852 // Ensure that we have an i8* for our PHI node.
853 llvm::Value *V = Builder.CreateBitCast(V: EmitScalarExpr(E: S.getTarget()),
854 DestTy: Int8PtrTy, Name: "addr");
855 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
856
857 // Get the basic block for the indirect goto.
858 llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
859
860 // The first instruction in the block has to be the PHI for the switch dest,
861 // add an entry for this branch.
862 cast<llvm::PHINode>(Val: IndGotoBB->begin())->addIncoming(V, BB: CurBB);
863
864 EmitBranch(Target: IndGotoBB);
865 if (CurBB && CurBB->hasTerminator())
866 addInstToCurrentSourceAtom(KeyInstruction: CurBB->getTerminator(), Backup: nullptr);
867}
868
869void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
870 const Stmt *Else = S.getElse();
871
872 // The else branch of a consteval if statement is always the only branch that
873 // can be runtime evaluated.
874 if (S.isConsteval()) {
875 const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;
876 if (Executed) {
877 RunCleanupsScope ExecutedScope(*this);
878 EmitStmt(S: Executed);
879 }
880 return;
881 }
882
883 // C99 6.8.4.1: The first substatement is executed if the expression compares
884 // unequal to 0. The condition must be a scalar type.
885 LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
886 ApplyDebugLocation DL(*this, S.getCond());
887
888 if (S.getInit()) {
889 EmitStmt(S: S.getInit());
890
891 // The init statement may have cleared the insertion point (e.g. it ended in
892 // a 'noreturn' call); the condition emitted below needs a valid one.
893 EnsureInsertPoint();
894 }
895
896 if (S.getConditionVariable())
897 EmitDecl(D: *S.getConditionVariable());
898
899 // If the condition constant folds and can be elided, try to avoid emitting
900 // the condition and the dead arm of the if/else.
901 bool CondConstant;
902 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: CondConstant,
903 AllowLabels: S.isConstexpr())) {
904 // Figure out which block (then or else) is executed.
905 const Stmt *Executed = S.getThen();
906 const Stmt *Skipped = Else;
907 if (!CondConstant) // Condition false?
908 std::swap(a&: Executed, b&: Skipped);
909
910 // If the skipped block has no labels in it, just emit the executed block.
911 // This avoids emitting dead code and simplifies the CFG substantially.
912 if (S.isConstexpr() || !ContainsLabel(S: Skipped)) {
913 incrementProfileCounter(ExecSkip: CondConstant ? UseExecPath : UseSkipPath, S: &S,
914 /*UseBoth=*/true);
915 if (Executed) {
916 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
917 RunCleanupsScope ExecutedScope(*this);
918 EmitStmt(S: Executed);
919 }
920 PGO->markStmtMaybeUsed(S: Skipped);
921 return;
922 }
923 }
924
925 auto HasSkip = hasSkipCounter(S: &S);
926
927 // Otherwise, the condition did not fold, or we couldn't elide it. Just emit
928 // the conditional branch.
929 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.then");
930 llvm::BasicBlock *ContBlock = createBasicBlock(name: "if.end");
931 llvm::BasicBlock *ElseBlock =
932 (Else || HasSkip ? createBasicBlock(name: "if.else") : ContBlock);
933 // Prefer the PGO based weights over the likelihood attribute.
934 // When the build isn't optimized the metadata isn't used, so don't generate
935 // it.
936 // Also, differentiate between disabled PGO and a never executed branch with
937 // PGO. Assuming PGO is in use:
938 // - we want to ignore the [[likely]] attribute if the branch is never
939 // executed,
940 // - assuming the profile is poor, preserving the attribute may still be
941 // beneficial.
942 // As an approximation, preserve the attribute only if both the branch and the
943 // parent context were not executed.
944 Stmt::Likelihood LH = Stmt::LH_None;
945 uint64_t ThenCount = getProfileCount(S: S.getThen());
946 if (!ThenCount && !getCurrentProfileCount() &&
947 CGM.getCodeGenOpts().OptimizationLevel)
948 LH = Stmt::getLikelihood(Then: S.getThen(), Else);
949
950 // When measuring MC/DC, always fully evaluate the condition up front using
951 // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
952 // executing the body of the if.then or if.else. This is useful for when
953 // there is a 'return' within the body, but this is particularly beneficial
954 // when one if-stmt is nested within another if-stmt so that all of the MC/DC
955 // updates are kept linear and consistent.
956 if (!CGM.getCodeGenOpts().MCDCCoverage) {
957 EmitBranchOnBoolExpr(Cond: S.getCond(), TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount: ThenCount, LH,
958 /*ConditionalOp=*/nullptr,
959 /*ConditionalDecl=*/S.getConditionVariable());
960 } else {
961 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
962 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
963 Builder.CreateCondBr(Cond: BoolCondVal, True: ThenBlock, False: ElseBlock);
964 }
965
966 // Emit the 'then' code.
967 EmitBlock(BB: ThenBlock);
968 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
969 {
970 RunCleanupsScope ThenScope(*this);
971 EmitStmt(S: S.getThen());
972 }
973 EmitBranch(Target: ContBlock);
974
975 // Emit the 'else' code if present.
976 if (Else) {
977 {
978 // There is no need to emit line number for an unconditional branch.
979 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
980 EmitBlock(BB: ElseBlock);
981 }
982 // Add a counter to else block unless it has CounterExpr.
983 if (HasSkip)
984 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
985 {
986 RunCleanupsScope ElseScope(*this);
987 EmitStmt(S: Else);
988 }
989 {
990 // There is no need to emit line number for an unconditional branch.
991 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
992 EmitBranch(Target: ContBlock);
993 }
994 } else if (HasSkip) {
995 EmitBlock(BB: ElseBlock);
996 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
997 EmitBranch(Target: ContBlock);
998 }
999
1000 // Emit the continuation block for code after the if.
1001 EmitBlock(BB: ContBlock, IsFinished: true);
1002}
1003
1004bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression,
1005 bool HasEmptyBody) {
1006 if (CGM.getCodeGenOpts().getFiniteLoops() ==
1007 CodeGenOptions::FiniteLoopsKind::Never)
1008 return false;
1009
1010 // Now apply rules for plain C (see 6.8.5.6 in C11).
1011 // Loops with constant conditions do not have to make progress in any C
1012 // version.
1013 // As an extension, we consisider loops whose constant expression
1014 // can be constant-folded.
1015 Expr::EvalResult Result;
1016 bool CondIsConstInt =
1017 !ControllingExpression ||
1018 (ControllingExpression->EvaluateAsInt(Result, Ctx: getContext()) &&
1019 Result.Val.isInt());
1020
1021 bool CondIsTrue = CondIsConstInt && (!ControllingExpression ||
1022 Result.Val.getInt().getBoolValue());
1023
1024 // Loops with non-constant conditions must make progress in C11 and later.
1025 if (getLangOpts().C11 && !CondIsConstInt)
1026 return true;
1027
1028 // [C++26][intro.progress] (DR)
1029 // The implementation may assume that any thread will eventually do one of the
1030 // following:
1031 // [...]
1032 // - continue execution of a trivial infinite loop ([stmt.iter.general]).
1033 if (CGM.getCodeGenOpts().getFiniteLoops() ==
1034 CodeGenOptions::FiniteLoopsKind::Always ||
1035 getLangOpts().CPlusPlus11) {
1036 if (HasEmptyBody && CondIsTrue) {
1037 CurFn->removeFnAttr(Kind: llvm::Attribute::MustProgress);
1038 return false;
1039 }
1040 return true;
1041 }
1042 return false;
1043}
1044
1045// [C++26][stmt.iter.general] (DR)
1046// A trivially empty iteration statement is an iteration statement matching one
1047// of the following forms:
1048// - while ( expression ) ;
1049// - while ( expression ) { }
1050// - do ; while ( expression ) ;
1051// - do { } while ( expression ) ;
1052// - for ( init-statement expression(opt); ) ;
1053// - for ( init-statement expression(opt); ) { }
1054template <typename LoopStmt> static bool hasEmptyLoopBody(const LoopStmt &S) {
1055 if constexpr (std::is_same_v<LoopStmt, ForStmt>) {
1056 if (S.getInc())
1057 return false;
1058 }
1059 const Stmt *Body = S.getBody();
1060 if (!Body || isa<NullStmt>(Val: Body))
1061 return true;
1062 if (const CompoundStmt *Compound = dyn_cast<CompoundStmt>(Val: Body))
1063 return Compound->body_empty();
1064 return false;
1065}
1066
1067void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
1068 ArrayRef<const Attr *> WhileAttrs) {
1069 // Emit the header for the loop, which will also become
1070 // the continue target.
1071 JumpDest LoopHeader = getJumpDestInCurrentScope(Name: "while.cond");
1072 EmitBlock(BB: LoopHeader.getBlock());
1073
1074 if (CGM.shouldEmitConvergenceTokens())
1075 ConvergenceTokenStack.push_back(
1076 Elt: emitConvergenceLoopToken(BB: LoopHeader.getBlock()));
1077
1078 // Create an exit block for when the condition fails, which will
1079 // also become the break target.
1080 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "while.end");
1081
1082 // Store the blocks to use for break and continue.
1083 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopHeader));
1084
1085 // C++ [stmt.while]p2:
1086 // When the condition of a while statement is a declaration, the
1087 // scope of the variable that is declared extends from its point
1088 // of declaration (3.3.2) to the end of the while statement.
1089 // [...]
1090 // The object created in a condition is destroyed and created
1091 // with each iteration of the loop.
1092 RunCleanupsScope ConditionScope(*this);
1093
1094 if (S.getConditionVariable())
1095 EmitDecl(D: *S.getConditionVariable());
1096
1097 // Evaluate the conditional in the while header. C99 6.8.5.1: The
1098 // evaluation of the controlling expression takes place before each
1099 // execution of the loop body.
1100 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1101
1102 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1103
1104 // while(1) is common, avoid extra exit blocks. Be sure
1105 // to correctly handle break/continue though.
1106 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1107 bool EmitBoolCondBranch = !C || !C->isOne();
1108 const SourceRange &R = S.getSourceRange();
1109 LoopStack.push(Header: LoopHeader.getBlock(), Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
1110 Attrs: WhileAttrs, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1111 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1112 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1113
1114 // As long as the condition is true, go to the loop body.
1115 llvm::BasicBlock *LoopBody = createBasicBlock(name: "while.body");
1116 if (EmitBoolCondBranch) {
1117 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1118 if (hasSkipCounter(S: &S) || ConditionScope.requiresCleanups())
1119 ExitBlock = createBasicBlock(name: "while.exit");
1120 llvm::MDNode *Weights =
1121 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1122 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1123 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1124 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1125 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock, BranchWeights: Weights);
1126 // Key Instructions: Emit the condition and branch as separate source
1127 // location atoms otherwise we may omit a step onto the loop condition in
1128 // favour of the `while` keyword.
1129 // FIXME: We could have the branch as the backup location for the condition,
1130 // which would probably be a better experience. Explore this later.
1131 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1132 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1133 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1134
1135 if (ExitBlock != LoopExit.getBlock()) {
1136 EmitBlock(BB: ExitBlock);
1137 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1138 EmitBranchThroughCleanup(Dest: LoopExit);
1139 }
1140 } else if (const Attr *A = Stmt::getLikelihoodAttr(S: S.getBody())) {
1141 CGM.getDiags().Report(Loc: A->getLocation(),
1142 DiagID: diag::warn_attribute_has_no_effect_on_infinite_loop)
1143 << A << A->getRange();
1144 CGM.getDiags().Report(
1145 Loc: S.getWhileLoc(),
1146 DiagID: diag::note_attribute_has_no_effect_on_infinite_loop_here)
1147 << SourceRange(S.getWhileLoc(), S.getRParenLoc());
1148 }
1149
1150 // Emit the loop body. We have to emit this in a cleanup scope
1151 // because it might be a singleton DeclStmt.
1152 {
1153 RunCleanupsScope BodyScope(*this);
1154 EmitBlock(BB: LoopBody);
1155 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1156 EmitStmt(S: S.getBody());
1157 }
1158
1159 BreakContinueStack.pop_back();
1160
1161 // Immediately force cleanup.
1162 ConditionScope.ForceCleanup();
1163
1164 EmitStopPoint(S: &S);
1165 // Branch to the loop header again.
1166 EmitBranch(Target: LoopHeader.getBlock());
1167
1168 LoopStack.pop();
1169
1170 // Emit the exit block.
1171 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1172
1173 // The LoopHeader typically is just a branch if we skipped emitting
1174 // a branch, try to erase it.
1175 if (!EmitBoolCondBranch) {
1176 SimplifyForwardingBlocks(BB: LoopHeader.getBlock());
1177 PGO->markStmtAsUsed(Skipped: true, S: &S);
1178 }
1179
1180 if (CGM.shouldEmitConvergenceTokens())
1181 ConvergenceTokenStack.pop_back();
1182}
1183
1184void CodeGenFunction::EmitDoStmt(const DoStmt &S,
1185 ArrayRef<const Attr *> DoAttrs) {
1186 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "do.end");
1187 JumpDest LoopCond = getJumpDestInCurrentScope(Name: "do.cond");
1188
1189 uint64_t ParentCount = getCurrentProfileCount();
1190
1191 // Store the blocks to use for break and continue.
1192 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopCond));
1193
1194 // Emit the body of the loop.
1195 llvm::BasicBlock *LoopBody = createBasicBlock(name: "do.body");
1196
1197 EmitBlockWithFallThrough(BB: LoopBody, S: &S);
1198
1199 if (CGM.shouldEmitConvergenceTokens())
1200 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: LoopBody));
1201
1202 {
1203 RunCleanupsScope BodyScope(*this);
1204 EmitStmt(S: S.getBody());
1205 }
1206
1207 EmitBlock(BB: LoopCond.getBlock());
1208
1209 // C99 6.8.5.2: "The evaluation of the controlling expression takes place
1210 // after each execution of the loop body."
1211
1212 // Evaluate the conditional in the while header.
1213 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1214 // compares unequal to 0. The condition must be a scalar type.
1215 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1216
1217 BreakContinueStack.pop_back();
1218
1219 // "do {} while (0)" is common in macros, avoid extra blocks. Be sure
1220 // to correctly handle break/continue though.
1221 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1222 bool EmitBoolCondBranch = !C || !C->isZero();
1223
1224 const SourceRange &R = S.getSourceRange();
1225 LoopStack.push(Header: LoopBody, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: DoAttrs,
1226 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1227 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1228 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1229
1230 auto *LoopFalse = (hasSkipCounter(S: &S) ? createBasicBlock(name: "do.loopfalse")
1231 : LoopExit.getBlock());
1232
1233 // As long as the condition is true, iterate the loop.
1234 if (EmitBoolCondBranch) {
1235 uint64_t BackedgeCount = getProfileCount(S: S.getBody()) - ParentCount;
1236 auto *I = Builder.CreateCondBr(
1237 Cond: BoolCondVal, True: LoopBody, False: LoopFalse,
1238 BranchWeights: createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: BackedgeCount));
1239
1240 // Key Instructions: Emit the condition and branch as separate source
1241 // location atoms otherwise we may omit a step onto the loop condition in
1242 // favour of the closing brace.
1243 // FIXME: We could have the branch as the backup location for the condition,
1244 // which would probably be a better experience (no jumping to the brace).
1245 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1246 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1247 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1248 }
1249
1250 LoopStack.pop();
1251
1252 if (LoopFalse != LoopExit.getBlock()) {
1253 EmitBlock(BB: LoopFalse);
1254 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S, /*UseBoth=*/true);
1255 }
1256
1257 // Emit the exit block.
1258 EmitBlock(BB: LoopExit.getBlock());
1259
1260 // The DoCond block typically is just a branch if we skipped
1261 // emitting a branch, try to erase it.
1262 if (!EmitBoolCondBranch)
1263 SimplifyForwardingBlocks(BB: LoopCond.getBlock());
1264
1265 if (CGM.shouldEmitConvergenceTokens())
1266 ConvergenceTokenStack.pop_back();
1267}
1268
1269void CodeGenFunction::EmitForStmt(const ForStmt &S,
1270 ArrayRef<const Attr *> ForAttrs) {
1271 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1272
1273 std::optional<LexicalScope> ForScope;
1274 if (getLangOpts().C99 || getLangOpts().CPlusPlus)
1275 ForScope.emplace(args&: *this, args: S.getSourceRange());
1276
1277 // Evaluate the first part before the loop.
1278 if (S.getInit())
1279 EmitStmt(S: S.getInit());
1280
1281 // Start the loop with a block that tests the condition.
1282 // If there's an increment, the continue scope will be overwritten
1283 // later.
1284 JumpDest CondDest = getJumpDestInCurrentScope(Name: "for.cond");
1285 llvm::BasicBlock *CondBlock = CondDest.getBlock();
1286 EmitBlock(BB: CondBlock);
1287
1288 if (CGM.shouldEmitConvergenceTokens())
1289 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1290
1291 const SourceRange &R = S.getSourceRange();
1292 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1293 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1294 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1295 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1296
1297 // Create a cleanup scope for the condition variable cleanups.
1298 LexicalScope ConditionScope(*this, S.getSourceRange());
1299
1300 // If the for loop doesn't have an increment we can just use the condition as
1301 // the continue block. Otherwise, if there is no condition variable, we can
1302 // form the continue block now. If there is a condition variable, we can't
1303 // form the continue block until after we've emitted the condition, because
1304 // the condition is in scope in the increment, but Sema's jump diagnostics
1305 // ensure that there are no continues from the condition variable that jump
1306 // to the loop increment.
1307 JumpDest Continue;
1308 if (!S.getInc())
1309 Continue = CondDest;
1310 else if (!S.getConditionVariable())
1311 Continue = getJumpDestInCurrentScope(Name: "for.inc");
1312 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1313
1314 if (S.getCond()) {
1315 // If the for statement has a condition scope, emit the local variable
1316 // declaration.
1317 if (S.getConditionVariable()) {
1318 EmitDecl(D: *S.getConditionVariable());
1319
1320 // We have entered the condition variable's scope, so we're now able to
1321 // jump to the continue block.
1322 Continue = S.getInc() ? getJumpDestInCurrentScope(Name: "for.inc") : CondDest;
1323 BreakContinueStack.back().ContinueBlock = Continue;
1324 }
1325
1326 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1327 // If there are any cleanups between here and the loop-exit scope,
1328 // create a block to stage a loop exit along.
1329 if (hasSkipCounter(S: &S) || (ForScope && ForScope->requiresCleanups()))
1330 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1331
1332 // As long as the condition is true, iterate the loop.
1333 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1334
1335 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1336 // compares unequal to 0. The condition must be a scalar type.
1337 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1338
1339 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1340
1341 llvm::MDNode *Weights =
1342 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1343 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1344 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1345 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1346
1347 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1348 // Key Instructions: Emit the condition and branch as separate atoms to
1349 // match existing loop stepping behaviour. FIXME: We could have the branch
1350 // as the backup location for the condition, which would probably be a
1351 // better experience (no jumping to the brace).
1352 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1353 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1354 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1355
1356 if (ExitBlock != LoopExit.getBlock()) {
1357 EmitBlock(BB: ExitBlock);
1358 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1359 EmitBranchThroughCleanup(Dest: LoopExit);
1360 }
1361
1362 EmitBlock(BB: ForBody);
1363 } else {
1364 // Treat it as a non-zero constant. Don't even create a new block for the
1365 // body, just fall into it.
1366 PGO->markStmtAsUsed(Skipped: true, S: &S);
1367 }
1368
1369 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1370
1371 {
1372 // Create a separate cleanup scope for the body, in case it is not
1373 // a compound statement.
1374 RunCleanupsScope BodyScope(*this);
1375 EmitStmt(S: S.getBody());
1376 }
1377
1378 // The last block in the loop's body (which unconditionally branches to the
1379 // `inc` block if there is one).
1380 auto *FinalBodyBB = Builder.GetInsertBlock();
1381
1382 // If there is an increment, emit it next.
1383 if (S.getInc()) {
1384 EmitBlock(BB: Continue.getBlock());
1385 EmitStmt(S: S.getInc());
1386 }
1387
1388 BreakContinueStack.pop_back();
1389
1390 ConditionScope.ForceCleanup();
1391
1392 EmitStopPoint(S: &S);
1393 EmitBranch(Target: CondBlock);
1394
1395 if (ForScope)
1396 ForScope->ForceCleanup();
1397
1398 LoopStack.pop();
1399
1400 // Emit the fall-through block.
1401 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1402
1403 if (CGM.shouldEmitConvergenceTokens())
1404 ConvergenceTokenStack.pop_back();
1405
1406 if (FinalBodyBB) {
1407 // Key Instructions: We want the for closing brace to be step-able on to
1408 // match existing behaviour.
1409 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1410 }
1411}
1412
1413void
1414CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1415 ArrayRef<const Attr *> ForAttrs) {
1416 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1417
1418 LexicalScope ForScope(*this, S.getSourceRange());
1419
1420 // Evaluate the first pieces before the loop.
1421 if (S.getInit())
1422 EmitStmt(S: S.getInit());
1423 EmitStmt(S: S.getRangeStmt());
1424 EmitStmt(S: S.getBeginStmt());
1425 EmitStmt(S: S.getEndStmt());
1426
1427 // Start the loop with a block that tests the condition.
1428 // If there's an increment, the continue scope will be overwritten
1429 // later.
1430 llvm::BasicBlock *CondBlock = createBasicBlock(name: "for.cond");
1431 EmitBlock(BB: CondBlock);
1432
1433 if (CGM.shouldEmitConvergenceTokens())
1434 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1435
1436 const SourceRange &R = S.getSourceRange();
1437 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1438 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1439 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
1440
1441 // If there are any cleanups between here and the loop-exit scope,
1442 // create a block to stage a loop exit along.
1443 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1444 if (hasSkipCounter(S: &S) || ForScope.requiresCleanups())
1445 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1446
1447 // The loop body, consisting of the specified body and the loop variable.
1448 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1449
1450 // The body is executed if the expression, contextually converted
1451 // to bool, is true.
1452 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1453 llvm::MDNode *Weights =
1454 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1455 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1456 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1457 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1458 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1459 // Key Instructions: Emit the condition and branch as separate atoms to
1460 // match existing loop stepping behaviour. FIXME: We could have the branch as
1461 // the backup location for the condition, which would probably be a better
1462 // experience.
1463 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1464 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1465 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1466
1467 if (ExitBlock != LoopExit.getBlock()) {
1468 EmitBlock(BB: ExitBlock);
1469 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1470 EmitBranchThroughCleanup(Dest: LoopExit);
1471 }
1472
1473 EmitBlock(BB: ForBody);
1474 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1475
1476 // Create a block for the increment. In case of a 'continue', we jump there.
1477 JumpDest Continue = getJumpDestInCurrentScope(Name: "for.inc");
1478
1479 // Store the blocks to use for break and continue.
1480 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1481
1482 {
1483 // Create a separate cleanup scope for the loop variable and body.
1484 LexicalScope BodyScope(*this, S.getSourceRange());
1485 EmitStmt(S: S.getLoopVarStmt());
1486 EmitStmt(S: S.getBody());
1487 }
1488 // The last block in the loop's body (which unconditionally branches to the
1489 // `inc` block if there is one).
1490 auto *FinalBodyBB = Builder.GetInsertBlock();
1491
1492 EmitStopPoint(S: &S);
1493 // If there is an increment, emit it next.
1494 EmitBlock(BB: Continue.getBlock());
1495 EmitStmt(S: S.getInc());
1496
1497 BreakContinueStack.pop_back();
1498
1499 EmitBranch(Target: CondBlock);
1500
1501 ForScope.ForceCleanup();
1502
1503 LoopStack.pop();
1504
1505 // Emit the fall-through block.
1506 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1507
1508 if (CGM.shouldEmitConvergenceTokens())
1509 ConvergenceTokenStack.pop_back();
1510
1511 if (FinalBodyBB) {
1512 // We want the for closing brace to be step-able on to match existing
1513 // behaviour.
1514 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1515 }
1516}
1517
1518void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1519 if (RV.isScalar()) {
1520 Builder.CreateStore(Val: RV.getScalarVal(), Addr: ReturnValue);
1521 } else if (RV.isAggregate()) {
1522 LValue Dest = MakeAddrLValue(Addr: ReturnValue, T: Ty);
1523 LValue Src = MakeAddrLValue(Addr: RV.getAggregateAddress(), T: Ty);
1524 EmitAggregateCopy(Dest, Src, EltTy: Ty, MayOverlap: getOverlapForReturnValue());
1525 } else {
1526 EmitStoreOfComplex(V: RV.getComplexVal(), dest: MakeAddrLValue(Addr: ReturnValue, T: Ty),
1527 /*init*/ isInit: true);
1528 }
1529 EmitBranchThroughCleanup(Dest: ReturnBlock);
1530}
1531
1532namespace {
1533// RAII struct used to save and restore a return statment's result expression.
1534struct SaveRetExprRAII {
1535 SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1536 : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1537 CGF.RetExpr = RetExpr;
1538 }
1539 ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1540 const Expr *OldRetExpr;
1541 CodeGenFunction &CGF;
1542};
1543} // namespace
1544
1545/// Determine if the given call uses the swiftasync calling convention.
1546static bool isSwiftAsyncCallee(const CallExpr *CE) {
1547 auto calleeQualType = CE->getCallee()->getType();
1548 const FunctionType *calleeType = nullptr;
1549 if (calleeQualType->isFunctionPointerType() ||
1550 calleeQualType->isFunctionReferenceType() ||
1551 calleeQualType->isBlockPointerType() ||
1552 calleeQualType->isMemberFunctionPointerType()) {
1553 calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1554 } else if (auto *ty = dyn_cast<FunctionType>(Val&: calleeQualType)) {
1555 calleeType = ty;
1556 } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(Val: CE)) {
1557 if (auto methodDecl = CMCE->getMethodDecl()) {
1558 // getMethodDecl() doesn't handle member pointers at the moment.
1559 calleeType = methodDecl->getType()->castAs<FunctionType>();
1560 } else {
1561 return false;
1562 }
1563 } else {
1564 return false;
1565 }
1566 return calleeType->getCallConv() == CallingConv::CC_SwiftAsync;
1567}
1568
1569/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1570/// if the function returns void, or may be missing one if the function returns
1571/// non-void. Fun stuff :).
1572void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1573 ApplyAtomGroup Grp(getDebugInfo());
1574 if (requiresReturnValueCheck()) {
1575 llvm::Constant *SLoc = EmitCheckSourceLocation(Loc: S.getBeginLoc());
1576 auto *SLocPtr =
1577 new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1578 llvm::GlobalVariable::PrivateLinkage, SLoc);
1579 SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1580 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: SLocPtr);
1581 assert(ReturnLocation.isValid() && "No valid return location");
1582 Builder.CreateStore(Val: SLocPtr, Addr: ReturnLocation);
1583 }
1584
1585 // Returning from an outlined SEH helper is UB, and we already warn on it.
1586 if (IsOutlinedSEHHelper) {
1587 Builder.CreateUnreachable();
1588 Builder.ClearInsertionPoint();
1589 }
1590
1591 // Emit the result value, even if unused, to evaluate the side effects.
1592 const Expr *RV = S.getRetValue();
1593
1594 // Record the result expression of the return statement. The recorded
1595 // expression is used to determine whether a block capture's lifetime should
1596 // end at the end of the full expression as opposed to the end of the scope
1597 // enclosing the block expression.
1598 //
1599 // This permits a small, easily-implemented exception to our over-conservative
1600 // rules about not jumping to statements following block literals with
1601 // non-trivial cleanups.
1602 SaveRetExprRAII SaveRetExpr(RV, *this);
1603
1604 RunCleanupsScope cleanupScope(*this);
1605 if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(Val: RV))
1606 RV = EWC->getSubExpr();
1607
1608 // If we're in a swiftasynccall function, and the return expression is a
1609 // call to a swiftasynccall function, mark the call as the musttail call.
1610 std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail;
1611 if (RV && CurFnInfo &&
1612 CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) {
1613 if (auto CE = dyn_cast<CallExpr>(Val: RV)) {
1614 if (isSwiftAsyncCallee(CE)) {
1615 SaveMustTail.emplace(args&: MustTailCall, args&: CE);
1616 }
1617 }
1618 }
1619
1620 // FIXME: Clean this up by using an LValue for ReturnTemp,
1621 // EmitStoreThroughLValue, and EmitAnyExpr.
1622 // Check if the NRVO candidate was not globalized in OpenMP mode.
1623 if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1624 S.getNRVOCandidate()->isNRVOVariable() &&
1625 (!getLangOpts().OpenMP ||
1626 !CGM.getOpenMPRuntime()
1627 .getAddressOfLocalVariable(CGF&: *this, VD: S.getNRVOCandidate())
1628 .isValid())) {
1629 // Apply the named return value optimization for this return statement,
1630 // which means doing nothing: the appropriate result has already been
1631 // constructed into the NRVO variable.
1632
1633 // If there is an NRVO flag for this variable, set it to 1 into indicate
1634 // that the cleanup code should not destroy the variable.
1635 if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1636 Builder.CreateFlagStore(Value: Builder.getTrue(), Addr: NRVOFlag);
1637 } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1638 // Make sure not to return anything, but evaluate the expression
1639 // for side effects.
1640 if (RV) {
1641 EmitAnyExpr(E: RV);
1642 }
1643 } else if (!RV) {
1644 // Do nothing (return value is left uninitialized)
1645 } else if (FnRetTy->isReferenceType()) {
1646 // If this function returns a reference, take the address of the expression
1647 // rather than the value.
1648 RValue Result = EmitReferenceBindingToExpr(E: RV);
1649 auto *I = Builder.CreateStore(Val: Result.getScalarVal(), Addr: ReturnValue);
1650 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1651 } else {
1652 switch (getEvaluationKind(T: RV->getType())) {
1653 case TEK_Scalar: {
1654 llvm::Value *Ret = EmitScalarExpr(E: RV);
1655 if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1656 EmitStoreOfScalar(value: Ret, lvalue: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1657 /*isInit*/ true);
1658 } else {
1659 auto *I = Builder.CreateStore(Val: Ret, Addr: ReturnValue);
1660 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1661 }
1662 break;
1663 }
1664 case TEK_Complex:
1665 EmitComplexExprIntoLValue(E: RV, dest: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1666 /*isInit*/ true);
1667 break;
1668 case TEK_Aggregate:
1669 EmitAggExpr(E: RV, AS: AggValueSlot::forAddr(
1670 addr: ReturnValue, quals: Qualifiers(),
1671 isDestructed: AggValueSlot::IsDestructed,
1672 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1673 isAliased: AggValueSlot::IsNotAliased,
1674 mayOverlap: getOverlapForReturnValue()));
1675 break;
1676 }
1677 }
1678
1679 ++NumReturnExprs;
1680 if (!RV || RV->isEvaluatable(Ctx: getContext()))
1681 ++NumSimpleReturnExprs;
1682
1683 cleanupScope.ForceCleanup();
1684 EmitBranchThroughCleanup(Dest: ReturnBlock);
1685}
1686
1687void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1688 // As long as debug info is modeled with instructions, we have to ensure we
1689 // have a place to insert here and write the stop point here.
1690 if (HaveInsertPoint())
1691 EmitStopPoint(S: &S);
1692
1693 for (const auto *I : S.decls())
1694 EmitDecl(D: *I, /*EvaluateConditionDecl=*/true);
1695}
1696
1697auto CodeGenFunction::GetDestForLoopControlStmt(const LoopControlStmt &S)
1698 -> const BreakContinue * {
1699 if (!S.hasLabelTarget())
1700 return &BreakContinueStack.back();
1701
1702 const Stmt *LoopOrSwitch = S.getNamedLoopOrSwitch();
1703 assert(LoopOrSwitch && "break/continue target not set?");
1704 for (const BreakContinue &BC : llvm::reverse(C&: BreakContinueStack))
1705 if (BC.LoopOrSwitch == LoopOrSwitch)
1706 return &BC;
1707
1708 llvm_unreachable("break/continue target not found");
1709}
1710
1711void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1712 assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1713
1714 // If this code is reachable then emit a stop point (if generating
1715 // debug info). We have to do this ourselves because we are on the
1716 // "simple" statement path.
1717 if (HaveInsertPoint())
1718 EmitStopPoint(S: &S);
1719
1720 ApplyAtomGroup Grp(getDebugInfo());
1721 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->BreakBlock);
1722}
1723
1724void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1725 assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1726
1727 // If this code is reachable then emit a stop point (if generating
1728 // debug info). We have to do this ourselves because we are on the
1729 // "simple" statement path.
1730 if (HaveInsertPoint())
1731 EmitStopPoint(S: &S);
1732
1733 ApplyAtomGroup Grp(getDebugInfo());
1734 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->ContinueBlock);
1735}
1736
1737/// EmitCaseStmtRange - If case statement range is not too big then
1738/// add multiple cases to switch instruction, one for each value within
1739/// the range. If range is too big then emit "if" condition check.
1740void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1741 ArrayRef<const Attr *> Attrs) {
1742 assert(S.getRHS() && "Expected RHS value in CaseStmt");
1743
1744 llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(Ctx: getContext());
1745 llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(Ctx: getContext());
1746
1747 // Emit the code for this case. We do this first to make sure it is
1748 // properly chained from our predecessor before generating the
1749 // switch machinery to enter this block.
1750 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1751 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1752 EmitStmt(S: S.getSubStmt());
1753
1754 // If range is empty, do nothing.
1755 if (LHS.isSigned() ? RHS.slt(RHS: LHS) : RHS.ult(RHS: LHS))
1756 return;
1757
1758 Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1759 llvm::APInt Range = RHS - LHS;
1760 // FIXME: parameters such as this should not be hardcoded.
1761 if (Range.ult(RHS: llvm::APInt(Range.getBitWidth(), 64))) {
1762 // Range is small enough to add multiple switch instruction cases.
1763 uint64_t Total = getProfileCount(S: &S);
1764 unsigned NCases = Range.getZExtValue() + 1;
1765 // We only have one region counter for the entire set of cases here, so we
1766 // need to divide the weights evenly between the generated cases, ensuring
1767 // that the total weight is preserved. E.g., a weight of 5 over three cases
1768 // will be distributed as weights of 2, 2, and 1.
1769 uint64_t Weight = Total / NCases, Rem = Total % NCases;
1770 for (unsigned I = 0; I != NCases; ++I) {
1771 if (SwitchWeights)
1772 SwitchWeights->push_back(Elt: Weight + (Rem ? 1 : 0));
1773 else if (SwitchLikelihood)
1774 SwitchLikelihood->push_back(Elt: LH);
1775
1776 if (Rem)
1777 Rem--;
1778 SwitchInsn->addCase(OnVal: Builder.getInt(AI: LHS), Dest: CaseDest);
1779 ++LHS;
1780 }
1781 return;
1782 }
1783
1784 // The range is too big. Emit "if" condition into a new block,
1785 // making sure to save and restore the current insertion point.
1786 llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1787
1788 // Push this test onto the chain of range checks (which terminates
1789 // in the default basic block). The switch's default will be changed
1790 // to the top of this chain after switch emission is complete.
1791 llvm::BasicBlock *FalseDest = CaseRangeBlock;
1792 CaseRangeBlock = createBasicBlock(name: "sw.caserange");
1793
1794 CurFn->insert(Position: CurFn->end(), BB: CaseRangeBlock);
1795 Builder.SetInsertPoint(CaseRangeBlock);
1796
1797 // Emit range check.
1798 llvm::Value *Diff =
1799 Builder.CreateSub(LHS: SwitchInsn->getCondition(), RHS: Builder.getInt(AI: LHS));
1800 llvm::Value *Cond =
1801 Builder.CreateICmpULE(LHS: Diff, RHS: Builder.getInt(AI: Range), Name: "inbounds");
1802
1803 llvm::MDNode *Weights = nullptr;
1804 if (SwitchWeights) {
1805 uint64_t ThisCount = getProfileCount(S: &S);
1806 uint64_t DefaultCount = (*SwitchWeights)[0];
1807 Weights = createProfileWeights(TrueCount: ThisCount, FalseCount: DefaultCount);
1808
1809 // Since we're chaining the switch default through each large case range, we
1810 // need to update the weight for the default, ie, the first case, to include
1811 // this case.
1812 (*SwitchWeights)[0] += ThisCount;
1813 } else if (SwitchLikelihood)
1814 Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1815
1816 Builder.CreateCondBr(Cond, True: CaseDest, False: FalseDest, BranchWeights: Weights);
1817
1818 // Restore the appropriate insertion point.
1819 if (RestoreBB)
1820 Builder.SetInsertPoint(RestoreBB);
1821 else
1822 Builder.ClearInsertionPoint();
1823}
1824
1825void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1826 ArrayRef<const Attr *> Attrs) {
1827 // If there is no enclosing switch instance that we're aware of, then this
1828 // case statement and its block can be elided. This situation only happens
1829 // when we've constant-folded the switch, are emitting the constant case,
1830 // and part of the constant case includes another case statement. For
1831 // instance: switch (4) { case 4: do { case 5: } while (1); }
1832 if (!SwitchInsn) {
1833 EmitStmt(S: S.getSubStmt());
1834 return;
1835 }
1836
1837 // Handle case ranges.
1838 if (S.getRHS()) {
1839 EmitCaseStmtRange(S, Attrs);
1840 return;
1841 }
1842
1843 llvm::ConstantInt *CaseVal =
1844 Builder.getInt(AI: S.getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1845
1846 // Emit debuginfo for the case value if it is an enum value.
1847 const ConstantExpr *CE;
1848 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: S.getLHS()))
1849 CE = dyn_cast<ConstantExpr>(Val: ICE->getSubExpr());
1850 else
1851 CE = dyn_cast<ConstantExpr>(Val: S.getLHS());
1852 if (CE) {
1853 if (auto DE = dyn_cast<DeclRefExpr>(Val: CE->getSubExpr()))
1854 if (CGDebugInfo *Dbg = getDebugInfo())
1855 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
1856 Dbg->EmitGlobalVariable(VD: DE->getDecl(),
1857 Init: APValue(llvm::APSInt(CaseVal->getValue())));
1858 }
1859
1860 if (SwitchLikelihood)
1861 SwitchLikelihood->push_back(Elt: Stmt::getLikelihood(Attrs));
1862
1863 // If the body of the case is just a 'break', try to not emit an empty block.
1864 // If we're profiling or we're not optimizing, leave the block in for better
1865 // debug and coverage analysis.
1866 if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1867 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1868 isa<BreakStmt>(Val: S.getSubStmt())) {
1869 JumpDest Block = BreakContinueStack.back().BreakBlock;
1870
1871 // Only do this optimization if there are no cleanups that need emitting.
1872 if (isObviouslyBranchWithoutCleanups(Dest: Block)) {
1873 if (SwitchWeights)
1874 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1875 SwitchInsn->addCase(OnVal: CaseVal, Dest: Block.getBlock());
1876
1877 // If there was a fallthrough into this case, make sure to redirect it to
1878 // the end of the switch as well.
1879 if (Builder.GetInsertBlock()) {
1880 Builder.CreateBr(Dest: Block.getBlock());
1881 Builder.ClearInsertionPoint();
1882 }
1883 return;
1884 }
1885 }
1886
1887 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1888 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1889 if (SwitchWeights)
1890 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1891 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1892
1893 // Recursively emitting the statement is acceptable, but is not wonderful for
1894 // code where we have many case statements nested together, i.e.:
1895 // case 1:
1896 // case 2:
1897 // case 3: etc.
1898 // Handling this recursively will create a new block for each case statement
1899 // that falls through to the next case which is IR intensive. It also causes
1900 // deep recursion which can run into stack depth limitations. Handle
1901 // sequential non-range case statements specially.
1902 //
1903 // TODO When the next case has a likelihood attribute the code returns to the
1904 // recursive algorithm. Maybe improve this case if it becomes common practice
1905 // to use a lot of attributes.
1906 const CaseStmt *CurCase = &S;
1907 const CaseStmt *NextCase = dyn_cast<CaseStmt>(Val: S.getSubStmt());
1908
1909 // Otherwise, iteratively add consecutive cases to this switch stmt.
1910 while (NextCase && NextCase->getRHS() == nullptr) {
1911 CurCase = NextCase;
1912 llvm::ConstantInt *CaseVal =
1913 Builder.getInt(AI: CurCase->getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1914
1915 if (SwitchWeights)
1916 SwitchWeights->push_back(Elt: getProfileCount(S: NextCase));
1917 if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1918 CaseDest = createBasicBlock(name: "sw.bb");
1919 EmitBlockWithFallThrough(BB: CaseDest, S: CurCase);
1920 }
1921 // Since this loop is only executed when the CaseStmt has no attributes
1922 // use a hard-coded value.
1923 if (SwitchLikelihood)
1924 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
1925
1926 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1927 NextCase = dyn_cast<CaseStmt>(Val: CurCase->getSubStmt());
1928 }
1929
1930 // Generate a stop point for debug info if the case statement is
1931 // followed by a default statement. A fallthrough case before a
1932 // default case gets its own branch target.
1933 if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1934 EmitStopPoint(S: CurCase);
1935
1936 // Normal default recursion for non-cases.
1937 EmitStmt(S: CurCase->getSubStmt());
1938}
1939
1940void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1941 ArrayRef<const Attr *> Attrs) {
1942 // If there is no enclosing switch instance that we're aware of, then this
1943 // default statement can be elided. This situation only happens when we've
1944 // constant-folded the switch.
1945 if (!SwitchInsn) {
1946 EmitStmt(S: S.getSubStmt());
1947 return;
1948 }
1949
1950 llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1951 assert(DefaultBlock->empty() &&
1952 "EmitDefaultStmt: Default block already defined?");
1953
1954 if (SwitchLikelihood)
1955 SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1956
1957 EmitBlockWithFallThrough(BB: DefaultBlock, S: &S);
1958
1959 EmitStmt(S: S.getSubStmt());
1960}
1961
1962namespace {
1963struct EmitDeferredStatement final : EHScopeStack::Cleanup {
1964 const DeferStmt &Stmt;
1965 EmitDeferredStatement(const DeferStmt *Stmt) : Stmt(*Stmt) {}
1966
1967 void Emit(CodeGenFunction &CGF, Flags) override {
1968 // Take care that any cleanups pushed by the body of a '_Defer' statement
1969 // don't clobber the current cleanup slot value.
1970 //
1971 // Assume we have a scope that pushes a cleanup; when that scope is exited,
1972 // we need to run that cleanup; this is accomplished by emitting the cleanup
1973 // into a separate block and then branching to that block at scope exit.
1974 //
1975 // Where this gets complicated is if we exit the scope in multiple different
1976 // ways; e.g. in a 'for' loop, we may exit the scope of its body by falling
1977 // off the end (in which case we need to run the cleanup and then branch to
1978 // the increment), or by 'break'ing out of the loop (in which case we need
1979 // to run the cleanup and then branch to the loop exit block); in both cases
1980 // we first branch to the cleanup block to run the cleanup, but the block we
1981 // need to jump to *after* running the cleanup is different.
1982 //
1983 // This is accomplished using a local integer variable called the 'cleanup
1984 // slot': before branching to the cleanup block, we store a value into that
1985 // slot. Then, in the cleanup block, after running the cleanup, we load the
1986 // value of that variable and 'switch' on it to branch to the appropriate
1987 // continuation block.
1988 //
1989 // The problem that arises once '_Defer' statements are involved is that the
1990 // body of a '_Defer' is an arbitrary statement which itself can create more
1991 // cleanups. This means we may end up overwriting the cleanup slot before we
1992 // ever have a chance to 'switch' on it, which means that once we *do* get
1993 // to the 'switch', we end up in whatever block the cleanup code happened to
1994 // pick as the default 'switch' exit label!
1995 //
1996 // That is, what is normally supposed to happen is something like:
1997 //
1998 // 1. Store 'X' to cleanup slot.
1999 // 2. Branch to cleanup block.
2000 // 3. Execute cleanup.
2001 // 4. Read value from cleanup slot.
2002 // 5. Branch to the block associated with 'X'.
2003 //
2004 // But if we encounter a _Defer' statement that contains a cleanup, then
2005 // what might instead happen is:
2006 //
2007 // 1. Store 'X' to cleanup slot.
2008 // 2. Branch to cleanup block.
2009 // 3. Execute cleanup; this ends up pushing another cleanup, so:
2010 // 3a. Store 'Y' to cleanup slot.
2011 // 3b. Run steps 2–5 recursively.
2012 // 4. Read value from cleanup slot, which is now 'Y' instead of 'X'.
2013 // 5. Branch to the block associated with 'Y'... which doesn't even
2014 // exist because the value 'Y' is only meaningful for the inner
2015 // cleanup. The result is we just branch 'somewhere random'.
2016 //
2017 // The rest of the cleanup code simply isn't prepared to handle this case
2018 // because most other cleanups can't push more cleanups, and thus, emitting
2019 // other cleanups generally cannot clobber the cleanup slot.
2020 //
2021 // To prevent this from happening, save the current cleanup slot value and
2022 // restore it after emitting the '_Defer' statement.
2023 llvm::Value *SavedCleanupDest = nullptr;
2024 if (CGF.NormalCleanupDest.isValid())
2025 SavedCleanupDest =
2026 CGF.Builder.CreateLoad(Addr: CGF.NormalCleanupDest, Name: "cleanup.dest.saved");
2027
2028 CGF.EmitStmt(S: Stmt.getBody());
2029
2030 if (SavedCleanupDest && CGF.HaveInsertPoint())
2031 CGF.Builder.CreateStore(Val: SavedCleanupDest, Addr: CGF.NormalCleanupDest);
2032
2033 // Cleanups must end with an insert point.
2034 CGF.EnsureInsertPoint();
2035 }
2036};
2037} // namespace
2038
2039void CodeGenFunction::EmitDeferStmt(const DeferStmt &S) {
2040 EHStack.pushCleanup<EmitDeferredStatement>(Kind: NormalAndEHCleanup, A: &S);
2041}
2042
2043/// CollectStatementsForCase - Given the body of a 'switch' statement and a
2044/// constant value that is being switched on, see if we can dead code eliminate
2045/// the body of the switch to a simple series of statements to emit. Basically,
2046/// on a switch (5) we want to find these statements:
2047/// case 5:
2048/// printf(...); <--
2049/// ++i; <--
2050/// break;
2051///
2052/// and add them to the ResultStmts vector. If it is unsafe to do this
2053/// transformation (for example, one of the elided statements contains a label
2054/// that might be jumped to), return CSFC_Failure. If we handled it and 'S'
2055/// should include statements after it (e.g. the printf() line is a substmt of
2056/// the case) then return CSFC_FallThrough. If we handled it and found a break
2057/// statement, then return CSFC_Success.
2058///
2059/// If Case is non-null, then we are looking for the specified case, checking
2060/// that nothing we jump over contains labels. If Case is null, then we found
2061/// the case and are looking for the break.
2062///
2063/// If the recursive walk actually finds our Case, then we set FoundCase to
2064/// true.
2065///
2066enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
2067static CSFC_Result CollectStatementsForCase(const Stmt *S,
2068 const SwitchCase *Case,
2069 bool &FoundCase,
2070 SmallVectorImpl<const Stmt*> &ResultStmts) {
2071 // If this is a null statement, just succeed.
2072 if (!S)
2073 return Case ? CSFC_Success : CSFC_FallThrough;
2074
2075 // If this is the switchcase (case 4: or default) that we're looking for, then
2076 // we're in business. Just add the substatement.
2077 if (const SwitchCase *SC = dyn_cast<SwitchCase>(Val: S)) {
2078 if (S == Case) {
2079 FoundCase = true;
2080 return CollectStatementsForCase(S: SC->getSubStmt(), Case: nullptr, FoundCase,
2081 ResultStmts);
2082 }
2083
2084 // Otherwise, this is some other case or default statement, just ignore it.
2085 return CollectStatementsForCase(S: SC->getSubStmt(), Case, FoundCase,
2086 ResultStmts);
2087 }
2088
2089 // If we are in the live part of the code and we found our break statement,
2090 // return a success!
2091 if (!Case && isa<BreakStmt>(Val: S))
2092 return CSFC_Success;
2093
2094 // If this is a switch statement, then it might contain the SwitchCase, the
2095 // break, or neither.
2096 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: S)) {
2097 // Handle this as two cases: we might be looking for the SwitchCase (if so
2098 // the skipped statements must be skippable) or we might already have it.
2099 CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
2100 bool StartedInLiveCode = FoundCase;
2101 unsigned StartSize = ResultStmts.size();
2102
2103 // If we've not found the case yet, scan through looking for it.
2104 if (Case) {
2105 // Keep track of whether we see a skipped declaration. The code could be
2106 // using the declaration even if it is skipped, so we can't optimize out
2107 // the decl if the kept statements might refer to it.
2108 bool HadSkippedDecl = false;
2109
2110 // If we're looking for the case, just see if we can skip each of the
2111 // substatements.
2112 for (; Case && I != E; ++I) {
2113 HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(S: *I);
2114
2115 switch (CollectStatementsForCase(S: *I, Case, FoundCase, ResultStmts)) {
2116 case CSFC_Failure: return CSFC_Failure;
2117 case CSFC_Success:
2118 // A successful result means that either 1) that the statement doesn't
2119 // have the case and is skippable, or 2) does contain the case value
2120 // and also contains the break to exit the switch. In the later case,
2121 // we just verify the rest of the statements are elidable.
2122 if (FoundCase) {
2123 // If we found the case and skipped declarations, we can't do the
2124 // optimization.
2125 if (HadSkippedDecl)
2126 return CSFC_Failure;
2127
2128 for (++I; I != E; ++I)
2129 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2130 return CSFC_Failure;
2131 return CSFC_Success;
2132 }
2133 break;
2134 case CSFC_FallThrough:
2135 // If we have a fallthrough condition, then we must have found the
2136 // case started to include statements. Consider the rest of the
2137 // statements in the compound statement as candidates for inclusion.
2138 assert(FoundCase && "Didn't find case but returned fallthrough?");
2139 // We recursively found Case, so we're not looking for it anymore.
2140 Case = nullptr;
2141
2142 // If we found the case and skipped declarations, we can't do the
2143 // optimization.
2144 if (HadSkippedDecl)
2145 return CSFC_Failure;
2146 break;
2147 }
2148 }
2149
2150 if (!FoundCase)
2151 return CSFC_Success;
2152
2153 assert(!HadSkippedDecl && "fallthrough after skipping decl");
2154 }
2155
2156 // If we have statements in our range, then we know that the statements are
2157 // live and need to be added to the set of statements we're tracking.
2158 bool AnyDecls = false;
2159 for (; I != E; ++I) {
2160 AnyDecls |= CodeGenFunction::mightAddDeclToScope(S: *I);
2161
2162 switch (CollectStatementsForCase(S: *I, Case: nullptr, FoundCase, ResultStmts)) {
2163 case CSFC_Failure: return CSFC_Failure;
2164 case CSFC_FallThrough:
2165 // A fallthrough result means that the statement was simple and just
2166 // included in ResultStmt, keep adding them afterwards.
2167 break;
2168 case CSFC_Success:
2169 // A successful result means that we found the break statement and
2170 // stopped statement inclusion. We just ensure that any leftover stmts
2171 // are skippable and return success ourselves.
2172 for (++I; I != E; ++I)
2173 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2174 return CSFC_Failure;
2175 return CSFC_Success;
2176 }
2177 }
2178
2179 // If we're about to fall out of a scope without hitting a 'break;', we
2180 // can't perform the optimization if there were any decls in that scope
2181 // (we'd lose their end-of-lifetime).
2182 if (AnyDecls) {
2183 // If the entire compound statement was live, there's one more thing we
2184 // can try before giving up: emit the whole thing as a single statement.
2185 // We can do that unless the statement contains a 'break;'.
2186 // FIXME: Such a break must be at the end of a construct within this one.
2187 // We could emit this by just ignoring the BreakStmts entirely.
2188 if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
2189 ResultStmts.resize(N: StartSize);
2190 ResultStmts.push_back(Elt: S);
2191 } else {
2192 return CSFC_Failure;
2193 }
2194 }
2195
2196 return CSFC_FallThrough;
2197 }
2198
2199 // Okay, this is some other statement that we don't handle explicitly, like a
2200 // for statement or increment etc. If we are skipping over this statement,
2201 // just verify it doesn't have labels, which would make it invalid to elide.
2202 if (Case) {
2203 if (CodeGenFunction::ContainsLabel(S, IgnoreCaseStmts: true))
2204 return CSFC_Failure;
2205 return CSFC_Success;
2206 }
2207
2208 // Otherwise, we want to include this statement. Everything is cool with that
2209 // so long as it doesn't contain a break out of the switch we're in.
2210 if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
2211
2212 // Otherwise, everything is great. Include the statement and tell the caller
2213 // that we fall through and include the next statement as well.
2214 ResultStmts.push_back(Elt: S);
2215 return CSFC_FallThrough;
2216}
2217
2218/// FindCaseStatementsForValue - Find the case statement being jumped to and
2219/// then invoke CollectStatementsForCase to find the list of statements to emit
2220/// for a switch on constant. See the comment above CollectStatementsForCase
2221/// for more details.
2222static bool FindCaseStatementsForValue(const SwitchStmt &S,
2223 const llvm::APSInt &ConstantCondValue,
2224 SmallVectorImpl<const Stmt*> &ResultStmts,
2225 ASTContext &C,
2226 const SwitchCase *&ResultCase) {
2227 // First step, find the switch case that is being branched to. We can do this
2228 // efficiently by scanning the SwitchCase list.
2229 const SwitchCase *Case = S.getSwitchCaseList();
2230 const DefaultStmt *DefaultCase = nullptr;
2231
2232 for (; Case; Case = Case->getNextSwitchCase()) {
2233 // It's either a default or case. Just remember the default statement in
2234 // case we're not jumping to any numbered cases.
2235 if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Val: Case)) {
2236 DefaultCase = DS;
2237 continue;
2238 }
2239
2240 // Check to see if this case is the one we're looking for.
2241 const CaseStmt *CS = cast<CaseStmt>(Val: Case);
2242 // Don't handle case ranges yet.
2243 if (CS->getRHS()) return false;
2244
2245 // If we found our case, remember it as 'case'.
2246 if (CS->getLHS()->EvaluateKnownConstInt(Ctx: C) == ConstantCondValue)
2247 break;
2248 }
2249
2250 // If we didn't find a matching case, we use a default if it exists, or we
2251 // elide the whole switch body!
2252 if (!Case) {
2253 // It is safe to elide the body of the switch if it doesn't contain labels
2254 // etc. If it is safe, return successfully with an empty ResultStmts list.
2255 if (!DefaultCase)
2256 return !CodeGenFunction::ContainsLabel(S: &S);
2257 Case = DefaultCase;
2258 }
2259
2260 // Ok, we know which case is being jumped to, try to collect all the
2261 // statements that follow it. This can fail for a variety of reasons. Also,
2262 // check to see that the recursive walk actually found our case statement.
2263 // Insane cases like this can fail to find it in the recursive walk since we
2264 // don't handle every stmt kind:
2265 // switch (4) {
2266 // while (1) {
2267 // case 4: ...
2268 bool FoundCase = false;
2269 ResultCase = Case;
2270 return CollectStatementsForCase(S: S.getBody(), Case, FoundCase,
2271 ResultStmts) != CSFC_Failure &&
2272 FoundCase;
2273}
2274
2275static std::optional<SmallVector<uint64_t, 16>>
2276getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
2277 // Are there enough branches to weight them?
2278 if (Likelihoods.size() <= 1)
2279 return std::nullopt;
2280
2281 uint64_t NumUnlikely = 0;
2282 uint64_t NumNone = 0;
2283 uint64_t NumLikely = 0;
2284 for (const auto LH : Likelihoods) {
2285 switch (LH) {
2286 case Stmt::LH_Unlikely:
2287 ++NumUnlikely;
2288 break;
2289 case Stmt::LH_None:
2290 ++NumNone;
2291 break;
2292 case Stmt::LH_Likely:
2293 ++NumLikely;
2294 break;
2295 }
2296 }
2297
2298 // Is there a likelihood attribute used?
2299 if (NumUnlikely == 0 && NumLikely == 0)
2300 return std::nullopt;
2301
2302 // When multiple cases share the same code they can be combined during
2303 // optimization. In that case the weights of the branch will be the sum of
2304 // the individual weights. Make sure the combined sum of all neutral cases
2305 // doesn't exceed the value of a single likely attribute.
2306 // The additions both avoid divisions by 0 and make sure the weights of None
2307 // don't exceed the weight of Likely.
2308 const uint64_t Likely = INT32_MAX / (NumLikely + 2);
2309 const uint64_t None = Likely / (NumNone + 1);
2310 const uint64_t Unlikely = 0;
2311
2312 SmallVector<uint64_t, 16> Result;
2313 Result.reserve(N: Likelihoods.size());
2314 for (const auto LH : Likelihoods) {
2315 switch (LH) {
2316 case Stmt::LH_Unlikely:
2317 Result.push_back(Elt: Unlikely);
2318 break;
2319 case Stmt::LH_None:
2320 Result.push_back(Elt: None);
2321 break;
2322 case Stmt::LH_Likely:
2323 Result.push_back(Elt: Likely);
2324 break;
2325 }
2326 }
2327
2328 return Result;
2329}
2330
2331void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
2332 // Handle nested switch statements.
2333 llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
2334 SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
2335 SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
2336 llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
2337
2338 // See if we can constant fold the condition of the switch and therefore only
2339 // emit the live case statement (if any) of the switch.
2340 llvm::APSInt ConstantCondValue;
2341 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: ConstantCondValue)) {
2342 SmallVector<const Stmt*, 4> CaseStmts;
2343 const SwitchCase *Case = nullptr;
2344 if (FindCaseStatementsForValue(S, ConstantCondValue, ResultStmts&: CaseStmts,
2345 C&: getContext(), ResultCase&: Case)) {
2346 if (Case)
2347 incrementProfileCounter(S: Case);
2348 RunCleanupsScope ExecutedScope(*this);
2349
2350 if (S.getInit())
2351 EmitStmt(S: S.getInit());
2352
2353 // Emit the condition variable if needed inside the entire cleanup scope
2354 // used by this special case for constant folded switches.
2355 if (S.getConditionVariable())
2356 EmitDecl(D: *S.getConditionVariable(), /*EvaluateConditionDecl=*/true);
2357
2358 // At this point, we are no longer "within" a switch instance, so
2359 // we can temporarily enforce this to ensure that any embedded case
2360 // statements are not emitted.
2361 SwitchInsn = nullptr;
2362
2363 // Okay, we can dead code eliminate everything except this case. Emit the
2364 // specified series of statements and we're good.
2365 for (const Stmt *CaseStmt : CaseStmts)
2366 EmitStmt(S: CaseStmt);
2367 incrementProfileCounter(S: &S);
2368 PGO->markStmtMaybeUsed(S: S.getBody());
2369
2370 // Now we want to restore the saved switch instance so that nested
2371 // switches continue to function properly
2372 SwitchInsn = SavedSwitchInsn;
2373
2374 return;
2375 }
2376 }
2377
2378 JumpDest SwitchExit = getJumpDestInCurrentScope(Name: "sw.epilog");
2379
2380 RunCleanupsScope ConditionScope(*this);
2381
2382 if (S.getInit()) {
2383 EmitStmt(S: S.getInit());
2384
2385 // The init statement may have cleared the insertion point (e.g. it ended in
2386 // a 'noreturn' call); the condition emitted below needs a valid one.
2387 EnsureInsertPoint();
2388 }
2389
2390 if (S.getConditionVariable())
2391 EmitDecl(D: *S.getConditionVariable());
2392 llvm::Value *CondV = EmitScalarExpr(E: S.getCond());
2393 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
2394
2395 // Create basic block to hold stuff that comes after switch
2396 // statement. We also need to create a default block now so that
2397 // explicit case ranges tests can have a place to jump to on
2398 // failure.
2399 llvm::BasicBlock *DefaultBlock = createBasicBlock(name: "sw.default");
2400 SwitchInsn = Builder.CreateSwitch(V: CondV, Dest: DefaultBlock);
2401 addInstToNewSourceAtom(KeyInstruction: SwitchInsn, Backup: CondV);
2402
2403 if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {
2404 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2405 llvm::ConstantInt *BranchHintConstant =
2406 HLSLControlFlowAttr ==
2407 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2408 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2409 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2410 llvm::Metadata *Vals[] = {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2411 MDHelper.createConstant(C: BranchHintConstant)};
2412 SwitchInsn->setMetadata(Kind: "hlsl.controlflow.hint",
2413 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2414 }
2415
2416 if (PGO->haveRegionCounts()) {
2417 // Walk the SwitchCase list to find how many there are.
2418 uint64_t DefaultCount = 0;
2419 unsigned NumCases = 0;
2420 for (const SwitchCase *Case = S.getSwitchCaseList();
2421 Case;
2422 Case = Case->getNextSwitchCase()) {
2423 if (isa<DefaultStmt>(Val: Case))
2424 DefaultCount = getProfileCount(S: Case);
2425 NumCases += 1;
2426 }
2427 SwitchWeights = new SmallVector<uint64_t, 16>();
2428 SwitchWeights->reserve(N: NumCases);
2429 // The default needs to be first. We store the edge count, so we already
2430 // know the right weight.
2431 SwitchWeights->push_back(Elt: DefaultCount);
2432 } else if (CGM.getCodeGenOpts().OptimizationLevel) {
2433 SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
2434 // Initialize the default case.
2435 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
2436 }
2437
2438 CaseRangeBlock = DefaultBlock;
2439
2440 // Clear the insertion point to indicate we are in unreachable code.
2441 Builder.ClearInsertionPoint();
2442
2443 // All break statements jump to NextBlock. If BreakContinueStack is non-empty
2444 // then reuse last ContinueBlock.
2445 JumpDest OuterContinue;
2446 if (!BreakContinueStack.empty())
2447 OuterContinue = BreakContinueStack.back().ContinueBlock;
2448
2449 BreakContinueStack.push_back(Elt: BreakContinue(S, SwitchExit, OuterContinue));
2450
2451 // Emit switch body.
2452 EmitStmt(S: S.getBody());
2453
2454 BreakContinueStack.pop_back();
2455
2456 // Update the default block in case explicit case range tests have
2457 // been chained on top.
2458 SwitchInsn->setDefaultDest(CaseRangeBlock);
2459
2460 // If a default was never emitted:
2461 if (!DefaultBlock->getParent()) {
2462 // If we have cleanups, emit the default block so that there's a
2463 // place to jump through the cleanups from.
2464 if (ConditionScope.requiresCleanups()) {
2465 EmitBlock(BB: DefaultBlock);
2466
2467 // Otherwise, just forward the default block to the switch end.
2468 } else {
2469 DefaultBlock->replaceAllUsesWith(V: SwitchExit.getBlock());
2470 delete DefaultBlock;
2471 }
2472 }
2473
2474 ConditionScope.ForceCleanup();
2475
2476 // Close the last case (or DefaultBlock).
2477 EmitBranch(Target: SwitchExit.getBlock());
2478
2479 // Insert a False Counter if SwitchStmt doesn't have DefaultStmt.
2480 if (hasSkipCounter(S: S.getCond())) {
2481 auto *ImplicitDefaultBlock = createBasicBlock(name: "sw.false");
2482 EmitBlock(BB: ImplicitDefaultBlock);
2483 incrementProfileCounter(ExecSkip: UseSkipPath, S: S.getCond());
2484 Builder.CreateBr(Dest: SwitchInsn->getDefaultDest());
2485 SwitchInsn->setDefaultDest(ImplicitDefaultBlock);
2486 }
2487
2488 // Emit continuation.
2489 EmitBlock(BB: SwitchExit.getBlock(), IsFinished: true);
2490 incrementProfileCounter(S: &S);
2491
2492 // If the switch has a condition wrapped by __builtin_unpredictable,
2493 // create metadata that specifies that the switch is unpredictable.
2494 // Don't bother if not optimizing because that metadata would not be used.
2495 auto *Call = dyn_cast<CallExpr>(Val: S.getCond());
2496 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2497 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2498 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2499 llvm::MDBuilder MDHelper(getLLVMContext());
2500 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_unpredictable,
2501 Node: MDHelper.createUnpredictable());
2502 }
2503 }
2504
2505 if (SwitchWeights) {
2506 assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2507 "switch weights do not match switch cases");
2508 // If there's only one jump destination there's no sense weighting it.
2509 if (SwitchWeights->size() > 1)
2510 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2511 Node: createProfileWeights(Weights: *SwitchWeights));
2512 delete SwitchWeights;
2513 } else if (SwitchLikelihood) {
2514 assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2515 "switch likelihoods do not match switch cases");
2516 std::optional<SmallVector<uint64_t, 16>> LHW =
2517 getLikelihoodWeights(Likelihoods: *SwitchLikelihood);
2518 if (LHW) {
2519 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2520 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2521 Node: createProfileWeights(Weights: *LHW));
2522 }
2523 delete SwitchLikelihood;
2524 }
2525 SwitchInsn = SavedSwitchInsn;
2526 SwitchWeights = SavedSwitchWeights;
2527 SwitchLikelihood = SavedSwitchLikelihood;
2528 CaseRangeBlock = SavedCRBlock;
2529}
2530
2531std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2532 const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2533 QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2534 if (Info.allowsRegister() || !Info.allowsMemory()) {
2535 if (CodeGenFunction::hasScalarEvaluationKind(T: InputType))
2536 return {EmitLoadOfLValue(V: InputValue, Loc).getScalarVal(), nullptr};
2537
2538 llvm::Type *Ty = ConvertType(T: InputType);
2539 uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2540 if ((Size <= 64 && llvm::isPowerOf2_64(Value: Size)) ||
2541 getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty)) {
2542 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2543
2544 return {Builder.CreateLoad(Addr: InputValue.getAddress().withElementType(ElemTy: Ty)),
2545 nullptr};
2546 }
2547 }
2548
2549 Address Addr = InputValue.getAddress();
2550 ConstraintStr += '*';
2551 return {InputValue.getPointer(CGF&: *this), Addr.getElementType()};
2552}
2553std::pair<llvm::Value *, llvm::Type *>
2554CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2555 const Expr *InputExpr,
2556 std::string &ConstraintStr) {
2557 // If this can't be a register or memory, i.e., has to be a constant
2558 // (immediate or symbolic), try to emit it as such.
2559 if (!Info.allowsRegister() && !Info.allowsMemory()) {
2560 if (Info.requiresImmediateConstant()) {
2561 Expr::EvalResult EVResult;
2562 InputExpr->EvaluateAsRValue(Result&: EVResult, Ctx: getContext(), InConstantContext: true);
2563
2564 llvm::APSInt IntResult;
2565 if (EVResult.Val.toIntegralConstant(Result&: IntResult, SrcTy: InputExpr->getType(),
2566 Ctx: getContext()))
2567 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: IntResult), nullptr};
2568 }
2569
2570 Expr::EvalResult Result;
2571 if (InputExpr->EvaluateAsInt(Result, Ctx: getContext()))
2572 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: Result.Val.getInt()),
2573 nullptr};
2574 }
2575
2576 if (Info.allowsRegister() || !Info.allowsMemory())
2577 if (CodeGenFunction::hasScalarEvaluationKind(T: InputExpr->getType()))
2578 return {EmitScalarExpr(E: InputExpr), nullptr};
2579 if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2580 return {EmitScalarExpr(E: InputExpr), nullptr};
2581 InputExpr = InputExpr->IgnoreParenNoopCasts(Ctx: getContext());
2582 LValue Dest = EmitLValue(E: InputExpr);
2583 return EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr,
2584 Loc: InputExpr->getExprLoc());
2585}
2586
2587/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2588/// asm call instruction. The !srcloc MDNode contains a list of constant
2589/// integers which are the source locations of the start of each line in the
2590/// asm.
2591static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2592 CodeGenFunction &CGF) {
2593 SmallVector<llvm::Metadata *, 8> Locs;
2594 // Add the location of the first line to the MDNode.
2595 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
2596 Ty: CGF.Int64Ty, V: Str->getBeginLoc().getRawEncoding())));
2597 StringRef StrVal = Str->getString();
2598 if (!StrVal.empty()) {
2599 const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2600 const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2601 unsigned StartToken = 0;
2602 unsigned ByteOffset = 0;
2603
2604 // Add the location of the start of each subsequent line of the asm to the
2605 // MDNode.
2606 for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2607 if (StrVal[i] != '\n') continue;
2608 SourceLocation LineLoc = Str->getLocationOfByte(
2609 ByteNo: i + 1, SM, Features: LangOpts, Target: CGF.getTarget(), StartToken: &StartToken, StartTokenByteOffset: &ByteOffset);
2610 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(
2611 C: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: LineLoc.getRawEncoding())));
2612 }
2613 }
2614
2615 return llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: Locs);
2616}
2617
2618namespace clang {
2619
2620/// This structure holds the information gathered about the constraints for an
2621/// inline assembly statement. It helps in separating the constraint processing
2622/// from the code generation.
2623class AsmConstraintsInfo {
2624 CodeGenFunction &CGF;
2625 CodeGenModule &CGM; // Per-module state.
2626 const AsmStmt &S;
2627 CGBuilderTy &Builder;
2628
2629 // The final asm string.
2630 std::string AsmString;
2631
2632 // The output and input constraints.
2633 SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2634 SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2635
2636 // Constraint strings.
2637 std::string Constraints;
2638 std::string InOutConstraints;
2639
2640 // Keep track of out constraints for tied input operand.
2641 std::vector<std::string> OutputConstraints;
2642
2643 // Keep track of argument types.
2644 std::vector<llvm::Value *> Args;
2645 std::vector<llvm::Type *> ArgTypes;
2646 std::vector<llvm::Type *> ArgElemTypes;
2647
2648 // Keep track of result register constraints.
2649 std::vector<LValue> ResultRegDests;
2650 std::vector<QualType> ResultRegQualTys;
2651 std::vector<llvm::Type *> ResultRegTypes;
2652 std::vector<llvm::Type *> ResultTruncRegTypes;
2653
2654 llvm::BitVector ResultTypeRequiresCast;
2655
2656 // Keep track of in/out constraints.
2657 std::vector<llvm::Value *> InOutArgs;
2658 std::vector<llvm::Type *> InOutArgTypes;
2659 std::vector<llvm::Type *> InOutArgElemTypes;
2660
2661 // Destination blocks for 'asm gotos'.
2662 llvm::BasicBlock *DefaultDest = nullptr;
2663 SmallVector<llvm::BasicBlock *, 3> IndirectDests;
2664
2665 std::vector<std::optional<std::pair<unsigned, unsigned>>> ResultBounds;
2666
2667 // An inline asm can be marked readonly if it meets the following
2668 // conditions:
2669 //
2670 // - it doesn't have any sideeffects
2671 // - it doesn't clobber memory
2672 // - it doesn't return a value by-reference
2673 //
2674 // It can be marked readnone if it doesn't have any input memory
2675 // constraints in addition to meeting the conditions listed above.
2676 bool ReadOnly = true;
2677 bool ReadNone = true;
2678
2679 bool GetOutputAndInputConstraints();
2680 void HandleOutputConstraints();
2681 void HandleMSStyleAsmBlob();
2682 void HandleInputConstraints();
2683 bool HandleLabels();
2684 bool HandleClobbers();
2685 void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2686 bool HasUnwindClobber, bool NoMerge, bool NoConvergent,
2687 std::vector<llvm::Value *> &RegResults);
2688 void EmitAsmStores(const llvm::ArrayRef<llvm::Value *> RegResults);
2689
2690 void EmitHipStdParUnsupportedAsm() {
2691 constexpr auto Name = "__ASM__hipstdpar_unsupported";
2692
2693 std::string Asm;
2694 if (auto GCCAsm = dyn_cast<GCCAsmStmt>(Val: &S))
2695 Asm = GCCAsm->getAsmString();
2696
2697 auto &Ctx = getLLVMContext();
2698 auto StrTy = llvm::ConstantDataArray::getString(Context&: Ctx, Initializer: Asm);
2699 auto FnTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx),
2700 Params: {StrTy->getType()}, isVarArg: false);
2701 auto UBF = CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2702
2703 Builder.CreateCall(Callee: UBF, Args: {StrTy});
2704 }
2705
2706 ASTContext &getContext() { return CGF.getContext(); }
2707 llvm::LLVMContext &getLLVMContext() { return CGF.getLLVMContext(); }
2708 const TargetInfo &getTarget() const { return CGF.getTarget(); }
2709 const LangOptions &getLangOpts() const { return CGF.getLangOpts(); }
2710 const TargetCodeGenInfo &getTargetHooks() const {
2711 return CGM.getTargetCodeGenInfo();
2712 }
2713
2714public:
2715 AsmConstraintsInfo(CodeGenFunction &CGF, const AsmStmt &S)
2716 : CGF(CGF), CGM(CGF.CGM), S(S), Builder(CGF.Builder),
2717 AsmString(S.generateAsmString(C: CGF.getContext())) {}
2718
2719 void EmitAsmStmt();
2720};
2721
2722} // namespace clang
2723
2724void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2725 // Pop all cleanup blocks at the end of the asm statement.
2726 CodeGenFunction::RunCleanupsScope Cleanups(*this);
2727
2728 // Get all the output and input constraints together.
2729 AsmConstraintsInfo AsmInfo(*this, S);
2730 AsmInfo.EmitAsmStmt();
2731}
2732
2733void AsmConstraintsInfo::EmitAsmStmt() {
2734 if (!GetOutputAndInputConstraints())
2735 return EmitHipStdParUnsupportedAsm();
2736
2737 // Handle output constraints.
2738 HandleOutputConstraints();
2739
2740 // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2741 // to the return value slot. Only do this when returning in registers.
2742 HandleMSStyleAsmBlob();
2743
2744 // Handle input constraints.
2745 HandleInputConstraints();
2746
2747 // Handle 'asm goto' labels.
2748 bool IsGCCAsmGoto = HandleLabels();
2749
2750 // Handle any clobbers.
2751 bool HasUnwindClobber = HandleClobbers();
2752 assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
2753 "unwind clobber can't be used with asm goto");
2754
2755 // Add machine specific clobbers
2756 std::string_view MachineClobbers = getTarget().getClobbers();
2757 if (!MachineClobbers.empty()) {
2758 if (!Constraints.empty())
2759 Constraints += ',';
2760 Constraints += MachineClobbers;
2761 }
2762
2763 llvm::Type *ResultType;
2764 if (ResultRegTypes.empty())
2765 ResultType = CGF.VoidTy;
2766 else if (ResultRegTypes.size() == 1)
2767 ResultType = ResultRegTypes[0];
2768 else
2769 ResultType = llvm::StructType::get(Context&: getLLVMContext(), Elements: ResultRegTypes);
2770
2771 llvm::FunctionType *FTy =
2772 llvm::FunctionType::get(Result: ResultType, Params: ArgTypes, isVarArg: false);
2773
2774 bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
2775
2776 llvm::InlineAsm::AsmDialect GnuAsmDialect =
2777 CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
2778 ? llvm::InlineAsm::AD_ATT
2779 : llvm::InlineAsm::AD_Intel;
2780 llvm::InlineAsm::AsmDialect AsmDialect =
2781 isa<MSAsmStmt>(Val: &S) ? llvm::InlineAsm::AD_Intel : GnuAsmDialect;
2782
2783 llvm::InlineAsm *IA = llvm::InlineAsm::get(
2784 Ty: FTy, AsmString, Constraints, hasSideEffects: HasSideEffect,
2785 /* IsAlignStack */ isAlignStack: false, asmDialect: AsmDialect, canThrow: HasUnwindClobber);
2786 std::vector<llvm::Value *> RegResults;
2787 llvm::CallBrInst *CBR;
2788 llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>
2789 CBRRegResults;
2790
2791 if (IsGCCAsmGoto) {
2792 CBR = Builder.CreateCallBr(Callee: IA, DefaultDest, IndirectDests, Args);
2793 CGF.EmitBlock(BB: DefaultDest);
2794 UpdateAsmCallInst(Result&: *CBR, HasSideEffect,
2795 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2796 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2797
2798 // Because we are emitting code top to bottom, we don't have enough
2799 // information at this point to know precisely whether we have a critical
2800 // edge. If we have outputs, split all indirect destinations.
2801 if (!RegResults.empty()) {
2802 unsigned I = 0;
2803 for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {
2804 llvm::Twine SynthName = Dest->getName() + ".split";
2805 llvm::BasicBlock *SynthBB = CGF.createBasicBlock(name: SynthName);
2806 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2807 Builder.SetInsertPoint(SynthBB);
2808
2809 if (ResultRegTypes.size() == 1) {
2810 CBRRegResults[SynthBB].push_back(Elt: CBR);
2811 } else {
2812 for (unsigned J = 0, E = ResultRegTypes.size(); J != E; ++J) {
2813 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: CBR, Idxs: J, Name: "asmresult");
2814 CBRRegResults[SynthBB].push_back(Elt: Tmp);
2815 }
2816 }
2817
2818 CGF.EmitBranch(Target: Dest);
2819 CGF.EmitBlock(BB: SynthBB);
2820 CBR->setIndirectDest(i: I++, B: SynthBB);
2821 }
2822 }
2823 } else if (HasUnwindClobber) {
2824 llvm::CallBase *Result = CGF.EmitCallOrInvoke(Callee: IA, Args, Name: "");
2825 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2826 /*HasUnwindClobber=*/true, NoMerge: CGF.InNoMergeAttributedStmt,
2827 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2828 } else {
2829 llvm::CallInst *Result =
2830 Builder.CreateCall(Callee: IA, Args, OpBundles: CGF.getBundlesForFunclet(Callee: IA));
2831 UpdateAsmCallInst(Result&: *Result, HasSideEffect,
2832 /*HasUnwindClobber=*/false, NoMerge: CGF.InNoMergeAttributedStmt,
2833 NoConvergent: CGF.InNoConvergentAttributedStmt, RegResults);
2834 }
2835
2836 EmitAsmStores(RegResults);
2837
2838 // If this is an asm goto with outputs, repeat EmitAsmStores, but with a
2839 // different insertion point; one for each indirect destination and with
2840 // CBRRegResults rather than RegResults.
2841 if (IsGCCAsmGoto && !CBRRegResults.empty()) {
2842 for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {
2843 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2844 Builder.SetInsertPoint(TheBB: Succ, IP: --(Succ->end()));
2845 EmitAsmStores(RegResults: CBRRegResults[Succ]);
2846 }
2847 }
2848}
2849
2850/// Gather and validate the output and input constraints for the given inline
2851/// assembly statement. This ensures that the constraints are valid for the
2852/// target and prepares them for further processing.
2853bool AsmConstraintsInfo::GetOutputAndInputConstraints() {
2854 bool IsValidTargetAsm = true;
2855 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2856 for (unsigned I = 0, E = S.getNumOutputs(); I != E && IsValidTargetAsm; I++) {
2857 StringRef Name;
2858 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2859 Name = GAS->getOutputName(i: I);
2860
2861 TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i: I), Name);
2862
2863 bool IsValid = getTarget().validateOutputConstraint(Info);
2864 if (IsHipStdPar && !IsValid)
2865 IsValidTargetAsm = false;
2866 else
2867 assert(IsValid && "Failed to parse output constraint");
2868
2869 OutputConstraintInfos.push_back(Elt: Info);
2870 }
2871
2872 for (unsigned I = 0, E = S.getNumInputs(); I != E && IsValidTargetAsm; I++) {
2873 StringRef Name;
2874 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2875 Name = GAS->getInputName(i: I);
2876
2877 TargetInfo::ConstraintInfo Info(S.getInputConstraint(i: I), Name);
2878
2879 bool IsValid =
2880 getTarget().validateInputConstraint(OutputConstraints: OutputConstraintInfos, info&: Info);
2881 if (IsHipStdPar && !IsValid)
2882 IsValidTargetAsm = false;
2883 else
2884 assert(IsValid && "Failed to parse input constraint");
2885
2886 InputConstraintInfos.push_back(Elt: Info);
2887 }
2888
2889 return IsValidTargetAsm;
2890}
2891
2892/// Process the output constraints of an inline assembly statement. This method
2893/// handles the complexity of determining whether an output should be a
2894/// register or memory operand, manages tied operands, and prepares the
2895/// necessary arguments for the LLVM inline asm call.
2896void AsmConstraintsInfo::HandleOutputConstraints() {
2897 // Keep track of defined physregs.
2898 llvm::SmallSet<std::string, 8> PhysRegOutputs;
2899
2900 for (unsigned I = 0, E = S.getNumOutputs(); I != E; I++) {
2901 TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[I];
2902
2903 // Simplify the output constraint.
2904 std::string OutputConstraint(S.getOutputConstraint(i: I));
2905 OutputConstraint = getTarget().simplifyConstraint(
2906 Constraint: StringRef(OutputConstraint).substr(Start: 1), OutCons: &OutputConstraintInfos);
2907
2908 const Expr *OutExpr = S.getOutputExpr(i: I);
2909 OutExpr = OutExpr->IgnoreParenNoopCasts(Ctx: getContext());
2910
2911 std::string GCCReg;
2912 OutputConstraint = S.addVariableConstraints(
2913 Constraint: OutputConstraint, AsmExpr: *OutExpr, Target: getTarget(), EarlyClobber: Info.earlyClobber(),
2914 UnsupportedCB: [&](const Stmt *UnspStmt, StringRef Msg) {
2915 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
2916 },
2917 GCCReg: &GCCReg);
2918
2919 // Give an error on multiple outputs to same physreg.
2920 if (!GCCReg.empty() && !PhysRegOutputs.insert(V: GCCReg).second)
2921 CGM.Error(loc: S.getAsmLoc(), error: "multiple outputs to hard register: " + GCCReg);
2922
2923 OutputConstraints.push_back(x: OutputConstraint);
2924 LValue Dest = CGF.EmitLValue(E: OutExpr);
2925 if (!Constraints.empty())
2926 Constraints += ',';
2927
2928 // If this is a register output, then make the inline asm return it
2929 // by-value. If this is a memory result, return the value by-reference.
2930 QualType QTy = OutExpr->getType();
2931 const bool IsScalarOrAggregate =
2932 CodeGenFunction::hasScalarEvaluationKind(T: QTy) ||
2933 CodeGenFunction::hasAggregateEvaluationKind(T: QTy);
2934
2935 if (!Info.allowsMemory() && IsScalarOrAggregate) {
2936 Constraints += "=" + OutputConstraint;
2937 ResultRegQualTys.push_back(x: QTy);
2938 ResultRegDests.push_back(x: Dest);
2939
2940 ResultBounds.emplace_back(args: Info.getOutputOperandBounds());
2941
2942 llvm::Type *Ty = CGF.ConvertTypeForMem(T: QTy);
2943 const bool RequiresCast =
2944 Info.allowsRegister() &&
2945 (getTargetHooks().isScalarizableAsmOperand(CGF, Ty) ||
2946 Ty->isAggregateType());
2947
2948 ResultTruncRegTypes.push_back(x: Ty);
2949 ResultTypeRequiresCast.push_back(Val: RequiresCast);
2950
2951 if (RequiresCast) {
2952 if (unsigned Size = getContext().getTypeSize(T: QTy))
2953 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2954 else
2955 CGM.Error(loc: OutExpr->getExprLoc(), error: "output size should not be zero");
2956 }
2957
2958 ResultRegTypes.push_back(x: Ty);
2959
2960 // If this output is tied to an input, and if the input is larger, then
2961 // we need to set the actual result type of the inline asm node to be the
2962 // same as the input type.
2963 if (Info.hasMatchingInput()) {
2964 unsigned InputNo;
2965 for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2966 TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2967 if (Input.hasTiedOperand() && Input.getTiedOperand() == I)
2968 break;
2969 }
2970 assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2971
2972 QualType InputTy = S.getInputExpr(i: InputNo)->getType();
2973 QualType OutputType = OutExpr->getType();
2974
2975 uint64_t InputSize = getContext().getTypeSize(T: InputTy);
2976 if (getContext().getTypeSize(T: OutputType) < InputSize)
2977 // Form the asm to return the value as a larger integer or fp type.
2978 ResultRegTypes.back() = CGF.ConvertType(T: InputTy);
2979 }
2980
2981 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
2982 CGF, Constraint: OutputConstraint, Ty: ResultRegTypes.back()))
2983 ResultRegTypes.back() = AdjTy;
2984 else
2985 CGM.getDiags().Report(Loc: S.getAsmLoc(),
2986 DiagID: diag::err_asm_invalid_type_in_input)
2987 << OutExpr->getType() << OutputConstraint;
2988
2989 // Update largest vector width for any vector types.
2990 if (auto *VT = dyn_cast<llvm::VectorType>(Val: ResultRegTypes.back()))
2991 CGF.LargestVectorWidth =
2992 std::max(a: (uint64_t)CGF.LargestVectorWidth,
2993 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
2994 } else {
2995 Address DestAddr = Dest.getAddress();
2996
2997 // Matrix types in memory are represented by arrays, but accessed through
2998 // vector pointers, with the alignment specified on the access operation.
2999 // For inline assembly, update pointer arguments to use vector pointers.
3000 // Otherwise there will be a mis-match if the matrix is also an
3001 // input-argument which is represented as vector.
3002 if (isa<MatrixType>(Val: OutExpr->getType().getCanonicalType()))
3003 DestAddr =
3004 DestAddr.withElementType(ElemTy: CGF.ConvertType(T: OutExpr->getType()));
3005
3006 ArgTypes.push_back(x: DestAddr.getType());
3007 ArgElemTypes.push_back(x: DestAddr.getElementType());
3008 Args.push_back(x: DestAddr.emitRawPointer(CGF));
3009
3010 Constraints += "=*" + OutputConstraint;
3011 ReadOnly = false;
3012 ReadNone = false;
3013 }
3014
3015 if (!Info.isReadWrite())
3016 continue;
3017
3018 InOutConstraints += ',';
3019
3020 const Expr *InputExpr = S.getOutputExpr(i: I);
3021 llvm::Value *Arg;
3022 llvm::Type *ArgElemType;
3023 std::tie(args&: Arg, args&: ArgElemType) =
3024 CGF.EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(),
3025 ConstraintStr&: InOutConstraints, Loc: InputExpr->getExprLoc());
3026
3027 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3028 CGF, Constraint: OutputConstraint, Ty: Arg->getType()))
3029 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3030
3031 // Update largest vector width for any vector types.
3032 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3033 CGF.LargestVectorWidth =
3034 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3035 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3036
3037 // Only tie earlyclobber physregs.
3038 if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
3039 InOutConstraints += llvm::utostr(X: I);
3040 else
3041 InOutConstraints += OutputConstraint;
3042
3043 InOutArgTypes.push_back(x: Arg->getType());
3044 InOutArgElemTypes.push_back(x: ArgElemType);
3045 InOutArgs.push_back(x: Arg);
3046 }
3047}
3048
3049/// Special handling for Microsoft-style inline assembly blocks. This ensures
3050/// that return registers (like EAX:EDX) are correctly mapped to the function's
3051/// return value slot when necessary.
3052void AsmConstraintsInfo::HandleMSStyleAsmBlob() {
3053 if (!isa<MSAsmStmt>(Val: &S))
3054 return;
3055
3056 const ABIArgInfo &RetAI = CGF.CurFnInfo->getReturnInfo();
3057 if (!RetAI.isDirect() && !RetAI.isExtend())
3058 return;
3059
3060 // Make a fake lvalue for the return value slot.
3061 LValue ReturnSlot =
3062 CGF.MakeAddrLValueWithoutTBAA(Addr: CGF.ReturnValue, T: CGF.FnRetTy);
3063 CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
3064 CGF, ReturnValue: ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
3065 ResultRegDests, AsmString, NumOutputs: S.getNumOutputs());
3066 CGF.SawAsmBlock = true;
3067}
3068
3069/// Process the input constraints of an inline assembly statement. It handles
3070/// type conversions, extensions for tied operands, and collects the necessary
3071/// LLVM values to be passed to the inline assembly call.
3072void AsmConstraintsInfo::HandleInputConstraints() {
3073 ASTContext &Ctx = getContext();
3074
3075 for (unsigned I = 0, E = S.getNumInputs(); I != E; I++) {
3076 TargetInfo::ConstraintInfo &Info = InputConstraintInfos[I];
3077 const Expr *InputExpr = S.getInputExpr(i: I);
3078
3079 if (Info.allowsMemory())
3080 ReadNone = false;
3081
3082 if (!Constraints.empty())
3083 Constraints += ',';
3084
3085 // Simplify the input constraint.
3086 std::string InputConstraint(S.getInputConstraint(i: I));
3087 InputConstraint =
3088 getTarget().simplifyConstraint(Constraint: InputConstraint, OutCons: &OutputConstraintInfos);
3089
3090 InputConstraint = S.addVariableConstraints(
3091 Constraint: InputConstraint, AsmExpr: *InputExpr->IgnoreParenNoopCasts(Ctx), Target: getTarget(),
3092 EarlyClobber: false /* No EarlyClobber */,
3093 UnsupportedCB: [&](const Stmt *UnspStmt, std::string_view Msg) {
3094 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
3095 });
3096
3097 std::string ReplaceConstraint(InputConstraint);
3098 llvm::Value *Arg;
3099 llvm::Type *ArgElemType;
3100 std::tie(args&: Arg, args&: ArgElemType) = CGF.EmitAsmInput(Info, InputExpr, ConstraintStr&: Constraints);
3101
3102 // If this input argument is tied to a larger output result, extend the
3103 // input to be the same size as the output. The LLVM backend wants to see
3104 // the input and output of a matching constraint be the same size. Note
3105 // that GCC does not define what the top bits are here. We use zext because
3106 // that is usually cheaper, but LLVM IR should really get an anyext someday.
3107 if (Info.hasTiedOperand()) {
3108 unsigned Output = Info.getTiedOperand();
3109 QualType OutputType = S.getOutputExpr(i: Output)->getType();
3110 QualType InputTy = InputExpr->getType();
3111
3112 if (Ctx.getTypeSize(T: OutputType) > Ctx.getTypeSize(T: InputTy)) {
3113 // Use ptrtoint as appropriate so that we can do our extension.
3114 if (isa<llvm::PointerType>(Val: Arg->getType()))
3115 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: CGF.IntPtrTy);
3116
3117 llvm::Type *OutputTy = CGF.ConvertType(T: OutputType);
3118 if (isa<llvm::IntegerType>(Val: OutputTy))
3119 Arg = Builder.CreateZExt(V: Arg, DestTy: OutputTy);
3120 else if (isa<llvm::PointerType>(Val: OutputTy))
3121 Arg = Builder.CreateZExt(V: Arg, DestTy: CGF.IntPtrTy);
3122 else if (OutputTy->isFloatingPointTy())
3123 Arg = Builder.CreateFPExt(V: Arg, DestTy: OutputTy);
3124 }
3125
3126 // Deal with the tied operands' constraint code in adjustInlineAsmType.
3127 ReplaceConstraint = OutputConstraints[Output];
3128 }
3129
3130 if (llvm::Type *AdjTy = getTargetHooks().adjustInlineAsmType(
3131 CGF, Constraint: ReplaceConstraint, Ty: Arg->getType()))
3132 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3133 else
3134 CGM.getDiags().Report(Loc: S.getAsmLoc(), DiagID: diag::err_asm_invalid_type_in_input)
3135 << InputExpr->getType() << InputConstraint;
3136
3137 // Update largest vector width for any vector types.
3138 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3139 CGF.LargestVectorWidth =
3140 std::max(a: (uint64_t)CGF.LargestVectorWidth,
3141 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3142
3143 ArgTypes.push_back(x: Arg->getType());
3144 ArgElemTypes.push_back(x: ArgElemType);
3145 Args.push_back(x: Arg);
3146
3147 Constraints += InputConstraint;
3148 }
3149
3150 // Append the "input" part of in/out constraints.
3151 for (unsigned I = 0, E = InOutArgs.size(); I != E; I++) {
3152 ArgTypes.push_back(x: InOutArgTypes[I]);
3153 ArgElemTypes.push_back(x: InOutArgElemTypes[I]);
3154 Args.push_back(x: InOutArgs[I]);
3155 }
3156
3157 Constraints += InOutConstraints;
3158}
3159
3160/// Handle labels in an 'asm goto' statement. This method resolves the symbolic
3161/// labels to LLVM basic blocks and updates the constraint string to reflect
3162/// the indirect jump targets.
3163bool AsmConstraintsInfo::HandleLabels() {
3164 if (const auto *GS = dyn_cast<GCCAsmStmt>(Val: &S); GS && GS->isAsmGoto()) {
3165 for (const auto *E : GS->labels()) {
3166 CodeGenFunction::JumpDest Dest = CGF.getJumpDestForLabel(D: E->getLabel());
3167 IndirectDests.push_back(Elt: Dest.getBlock());
3168
3169 if (!Constraints.empty())
3170 Constraints += ',';
3171
3172 Constraints += "!i";
3173 }
3174
3175 DefaultDest = CGF.createBasicBlock(name: "asm.fallthrough");
3176 return true;
3177 }
3178
3179 return false;
3180}
3181
3182/// Process clobber constraints for an inline assembly statement. This
3183/// identifies which registers or system state (like "memory" or "cc") are
3184/// modified by the assembly block, which is crucial for correct optimization
3185/// and side-effect modeling.
3186bool AsmConstraintsInfo::HandleClobbers() {
3187 bool HasUnwindClobber = false;
3188 for (unsigned I = 0, E = S.getNumClobbers(); I != E; I++) {
3189 std::string Clobber = S.getClobber(i: I);
3190
3191 if (Clobber == "unwind") {
3192 HasUnwindClobber = true;
3193 continue;
3194 }
3195
3196 if (Clobber == "memory") {
3197 ReadOnly = false;
3198 ReadNone = false;
3199 } else if (Clobber != "cc") {
3200 Clobber = getTarget().getNormalizedGCCRegisterName(Name: Clobber);
3201 if (CGM.getCodeGenOpts().StackClashProtector &&
3202 getTarget().isSPRegName(Clobber)) {
3203 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3204 DiagID: diag::warn_stack_clash_protection_inline_asm);
3205 }
3206 }
3207
3208 if (isa<MSAsmStmt>(Val: &S)) {
3209 if (Clobber == "eax" || Clobber == "edx") {
3210 if (Constraints.find(s: "=&A") != std::string::npos)
3211 continue;
3212
3213 std::string::size_type position1 =
3214 Constraints.find(str: "={" + Clobber + "}");
3215 if (position1 != std::string::npos) {
3216 Constraints.insert(pos: position1 + 1, s: "&");
3217 continue;
3218 }
3219
3220 std::string::size_type position2 = Constraints.find(s: "=A");
3221 if (position2 != std::string::npos) {
3222 Constraints.insert(pos: position2 + 1, s: "&");
3223 continue;
3224 }
3225 }
3226 }
3227
3228 if (!Constraints.empty())
3229 Constraints += ',';
3230
3231 Constraints += "~{" + Clobber + '}';
3232 }
3233
3234 return HasUnwindClobber;
3235}
3236
3237void AsmConstraintsInfo::UpdateAsmCallInst(
3238 llvm::CallBase &Result, bool HasSideEffect, bool HasUnwindClobber,
3239 bool NoMerge, bool NoConvergent, std::vector<llvm::Value *> &RegResults) {
3240 if (!HasUnwindClobber)
3241 Result.addFnAttr(Kind: llvm::Attribute::NoUnwind);
3242
3243 if (NoMerge)
3244 Result.addFnAttr(Kind: llvm::Attribute::NoMerge);
3245
3246 // Attach readnone and readonly attributes.
3247 if (!HasSideEffect) {
3248 if (ReadNone)
3249 Result.setDoesNotAccessMemory();
3250 else if (ReadOnly)
3251 Result.setOnlyReadsMemory();
3252 }
3253
3254 // Add elementtype attribute for indirect constraints.
3255 for (auto Pair : llvm::enumerate(First&: ArgElemTypes)) {
3256 if (Pair.value()) {
3257 auto Attr = llvm::Attribute::get(
3258 Context&: getLLVMContext(), Kind: llvm::Attribute::ElementType, Ty: Pair.value());
3259 Result.addParamAttr(ArgNo: Pair.index(), Attr);
3260 }
3261 }
3262
3263 // Slap the source location of the inline asm into a !srcloc metadata on the
3264 // call.
3265 const StringLiteral *SL;
3266 if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(Val: &S);
3267 gccAsmStmt &&
3268 (SL = dyn_cast<StringLiteral>(Val: gccAsmStmt->getAsmStringExpr()))) {
3269 Result.setMetadata(Kind: "srcloc", Node: getAsmSrcLocInfo(Str: SL, CGF));
3270 } else {
3271 // At least put the line number on MS inline asm blobs and GCC asm constexpr
3272 // strings.
3273 llvm::Constant *Loc =
3274 llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: S.getAsmLoc().getRawEncoding());
3275 Result.setMetadata(Kind: "srcloc",
3276 Node: llvm::MDNode::get(Context&: getLLVMContext(),
3277 MDs: llvm::ConstantAsMetadata::get(C: Loc)));
3278 }
3279
3280 // Make inline-asm calls Key for the debug info feature Key Instructions.
3281 CGF.addInstToNewSourceAtom(KeyInstruction: &Result, Backup: nullptr);
3282
3283 if (!NoConvergent && getLangOpts().assumeFunctionsAreConvergent())
3284 // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
3285 // convergent (meaning, they may call an intrinsically convergent op, such
3286 // as bar.sync, and so can't have certain optimizations applied around
3287 // them) unless it's explicitly marked 'noconvergent'.
3288 Result.addFnAttr(Kind: llvm::Attribute::Convergent);
3289
3290 // Extract all of the register value results from the asm.
3291 if (ResultRegTypes.size() == 1) {
3292 RegResults.push_back(x: &Result);
3293 } else {
3294 for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
3295 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: &Result, Idxs: i, Name: "asmresult");
3296 RegResults.push_back(x: Tmp);
3297 }
3298 }
3299}
3300
3301void AsmConstraintsInfo::EmitAsmStores(
3302 const llvm::ArrayRef<llvm::Value *> RegResults) {
3303 llvm::LLVMContext &CTX = getLLVMContext();
3304
3305 assert(RegResults.size() == ResultRegTypes.size());
3306 assert(RegResults.size() == ResultTruncRegTypes.size());
3307 assert(RegResults.size() == ResultRegDests.size());
3308
3309 // ResultRegDests can also be populated by addReturnRegisterOutputs() above,
3310 // in which case its size may grow.
3311 assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
3312 assert(ResultBounds.size() <= ResultRegDests.size());
3313
3314 for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
3315 llvm::Value *Tmp = RegResults[i];
3316 llvm::Type *TruncTy = ResultTruncRegTypes[i];
3317
3318 if (i < ResultBounds.size() && ResultBounds[i].has_value()) {
3319 const auto [LowerBound, UpperBound] = ResultBounds[i].value();
3320
3321 // FIXME: Support for nonzero lower bounds not yet implemented.
3322 assert(LowerBound == 0 && "Output operand lower bound is not zero.");
3323
3324 llvm::Constant *UpperBoundConst =
3325 llvm::ConstantInt::get(Ty: Tmp->getType(), V: UpperBound);
3326 llvm::Value *IsBooleanValue =
3327 Builder.CreateCmp(Pred: llvm::CmpInst::ICMP_ULT, LHS: Tmp, RHS: UpperBoundConst);
3328 llvm::Function *FnAssume = CGM.getIntrinsic(IID: llvm::Intrinsic::assume);
3329
3330 Builder.CreateCall(Callee: FnAssume, Args: IsBooleanValue);
3331 }
3332
3333 // If the result type of the LLVM IR asm doesn't match the result type of
3334 // the expression, do the conversion.
3335 if (ResultRegTypes[i] != TruncTy) {
3336 // Truncate the integer result to the right size, note that TruncTy can be
3337 // a pointer.
3338 if (TruncTy->isFloatingPointTy())
3339 Tmp = Builder.CreateFPTrunc(V: Tmp, DestTy: TruncTy);
3340 else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
3341 uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(Ty: TruncTy);
3342 Tmp = Builder.CreateTrunc(
3343 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)ResSize));
3344 Tmp = Builder.CreateIntToPtr(V: Tmp, DestTy: TruncTy);
3345 } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
3346 uint64_t TmpSize =
3347 CGM.getDataLayout().getTypeSizeInBits(Ty: Tmp->getType());
3348 Tmp = Builder.CreatePtrToInt(
3349 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)TmpSize));
3350 Tmp = Builder.CreateTrunc(V: Tmp, DestTy: TruncTy);
3351 } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
3352 Tmp = Builder.CreateZExtOrTrunc(V: Tmp, DestTy: TruncTy);
3353 } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
3354 Tmp = Builder.CreateBitCast(V: Tmp, DestTy: TruncTy);
3355 }
3356 }
3357
3358 ApplyAtomGroup Grp(CGF.getDebugInfo());
3359 LValue Dest = ResultRegDests[i];
3360
3361 // ResultTypeRequiresCast elements correspond to the first
3362 // ResultTypeRequiresCast.size() elements of RegResults.
3363 if (i < ResultTypeRequiresCast.size() && ResultTypeRequiresCast[i]) {
3364 unsigned Size = getContext().getTypeSize(T: ResultRegQualTys[i]);
3365 Address A = Dest.getAddress().withElementType(ElemTy: ResultRegTypes[i]);
3366
3367 if (getTargetHooks().isScalarizableAsmOperand(CGF, Ty: TruncTy)) {
3368 llvm::StoreInst *S = Builder.CreateStore(Val: Tmp, Addr: A);
3369 CGF.addInstToCurrentSourceAtom(KeyInstruction: S, Backup: S->getValueOperand());
3370 continue;
3371 }
3372
3373 QualType Ty = getContext().getIntTypeForBitwidth(DestWidth: Size, /*Signed=*/false);
3374 if (Ty.isNull()) {
3375 const Expr *OutExpr = S.getOutputExpr(i);
3376 CGM.getDiags().Report(Loc: OutExpr->getExprLoc(),
3377 DiagID: diag::err_store_value_to_reg);
3378 return;
3379 }
3380
3381 Dest = CGF.MakeAddrLValue(Addr: A, T: Ty);
3382 }
3383
3384 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Tmp), Dst: Dest);
3385 }
3386}
3387
3388LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
3389 const RecordDecl *RD = S.getCapturedRecordDecl();
3390 CanQualType RecordTy = getContext().getCanonicalTagType(TD: RD);
3391
3392 // Initialize the captured struct.
3393 LValue SlotLV =
3394 MakeAddrLValue(Addr: CreateMemTemp(T: RecordTy, Name: "agg.captured"), T: RecordTy);
3395
3396 RecordDecl::field_iterator CurField = RD->field_begin();
3397 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
3398 E = S.capture_init_end();
3399 I != E; ++I, ++CurField) {
3400 LValue LV = EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
3401 if (CurField->hasCapturedVLAType()) {
3402 EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
3403 } else {
3404 EmitInitializerForField(Field: *CurField, LHS: LV, Init: *I);
3405 }
3406 }
3407
3408 return SlotLV;
3409}
3410
3411/// Generate an outlined function for the body of a CapturedStmt, store any
3412/// captured variables into the captured struct, and call the outlined function.
3413llvm::Function *
3414CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
3415 LValue CapStruct = InitCapturedStruct(S);
3416
3417 // Emit the CapturedDecl
3418 CodeGenFunction CGF(CGM, true);
3419 CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
3420 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
3421 delete CGF.CapturedStmtInfo;
3422
3423 // Emit call to the helper function.
3424 EmitCallOrInvoke(Callee: F, Args: CapStruct.getPointer(CGF&: *this));
3425
3426 return F;
3427}
3428
3429Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
3430 LValue CapStruct = InitCapturedStruct(S);
3431 return CapStruct.getAddress();
3432}
3433
3434/// Creates the outlined function for a CapturedStmt.
3435llvm::Function *
3436CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
3437 assert(CapturedStmtInfo &&
3438 "CapturedStmtInfo should be set when generating the captured function");
3439 const CapturedDecl *CD = S.getCapturedDecl();
3440 const RecordDecl *RD = S.getCapturedRecordDecl();
3441 SourceLocation Loc = S.getBeginLoc();
3442 assert(CD->hasBody() && "missing CapturedDecl body");
3443
3444 // Build the argument list.
3445 ASTContext &Ctx = CGM.getContext();
3446 FunctionArgList Args;
3447 Args.append(in_start: CD->param_begin(), in_end: CD->param_end());
3448
3449 // Create the function declaration.
3450 const CGFunctionInfo &FuncInfo =
3451 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
3452 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
3453
3454 llvm::Function *F =
3455 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
3456 N: CapturedStmtInfo->getHelperName(), M: &CGM.getModule());
3457 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
3458 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3459 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3460 if (CD->isNothrow())
3461 F->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3462
3463 // Generate the function.
3464 StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc: CD->getLocation(),
3465 StartLoc: CD->getBody()->getBeginLoc());
3466 // Set the context parameter in CapturedStmtInfo.
3467 Address DeclPtr = GetAddrOfLocalVar(VD: CD->getContextParam());
3468 CapturedStmtInfo->setContextValue(Builder.CreateLoad(Addr: DeclPtr));
3469
3470 // Initialize variable-length arrays.
3471 LValue Base = MakeNaturalAlignRawAddrLValue(
3472 V: CapturedStmtInfo->getContextValue(), T: Ctx.getCanonicalTagType(TD: RD));
3473 for (auto *FD : RD->fields()) {
3474 if (FD->hasCapturedVLAType()) {
3475 auto *ExprArg =
3476 EmitLoadOfLValue(V: EmitLValueForField(Base, Field: FD), Loc: S.getBeginLoc())
3477 .getScalarVal();
3478 auto VAT = FD->getCapturedVLAType();
3479 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
3480 }
3481 }
3482
3483 // If 'this' is captured, load it into CXXThisValue.
3484 if (CapturedStmtInfo->isCXXThisExprCaptured()) {
3485 FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
3486 LValue ThisLValue = EmitLValueForField(Base, Field: FD);
3487 CXXThisValue = EmitLoadOfLValue(V: ThisLValue, Loc).getScalarVal();
3488 }
3489
3490 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
3491 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
3492 FinishFunction(EndLoc: CD->getBodyRBrace());
3493
3494 return F;
3495}
3496
3497// Returns the first convergence entry/loop/anchor instruction found in |BB|.
3498// std::nullptr otherwise.
3499static llvm::ConvergenceControlInst *getConvergenceToken(llvm::BasicBlock *BB) {
3500 for (auto &I : *BB) {
3501 if (auto *CI = dyn_cast<llvm::ConvergenceControlInst>(Val: &I))
3502 return CI;
3503 }
3504 return nullptr;
3505}
3506
3507llvm::CallBase *
3508CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input) {
3509 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3510 assert(ParentToken);
3511
3512 llvm::Value *bundleArgs[] = {ParentToken};
3513 llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
3514 auto *Output = llvm::CallBase::addOperandBundle(
3515 CB: Input, ID: llvm::LLVMContext::OB_convergencectrl, OB, InsertPt: Input->getIterator());
3516 Input->replaceAllUsesWith(V: Output);
3517 Input->eraseFromParent();
3518 return Output;
3519}
3520
3521llvm::ConvergenceControlInst *
3522CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {
3523 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3524 assert(ParentToken);
3525 return llvm::ConvergenceControlInst::CreateLoop(BB&: *BB, Parent: ParentToken);
3526}
3527
3528llvm::ConvergenceControlInst *
3529CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
3530 llvm::BasicBlock *BB = &F->getEntryBlock();
3531 llvm::ConvergenceControlInst *Token = getConvergenceToken(BB);
3532 if (Token)
3533 return Token;
3534
3535 // Adding a convergence token requires the function to be marked as
3536 // convergent.
3537 F->setConvergent();
3538 return llvm::ConvergenceControlInst::CreateEntry(BB&: *BB);
3539}
3540