1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Stmt nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGDebugInfo.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "CodeGenPGO.h"
18#include "TargetInfo.h"
19#include "clang/AST/Attr.h"
20#include "clang/AST/Expr.h"
21#include "clang/AST/Stmt.h"
22#include "clang/AST/StmtSYCL.h"
23#include "clang/AST/StmtVisitor.h"
24#include "clang/Basic/Builtins.h"
25#include "clang/Basic/DiagnosticSema.h"
26#include "clang/Basic/PrettyStackTrace.h"
27#include "clang/Basic/SourceManager.h"
28#include "clang/Basic/TargetInfo.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/StringExtras.h"
33#include "llvm/IR/Assumptions.h"
34#include "llvm/IR/DataLayout.h"
35#include "llvm/IR/InlineAsm.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/MDBuilder.h"
38#include "llvm/Support/SaveAndRestore.h"
39#include <optional>
40
41using namespace clang;
42using namespace CodeGen;
43
44//===----------------------------------------------------------------------===//
45// Statement Emission
46//===----------------------------------------------------------------------===//
47
48void CodeGenFunction::EmitStopPoint(const Stmt *S) {
49 if (CGDebugInfo *DI = getDebugInfo()) {
50 SourceLocation Loc;
51 Loc = S->getBeginLoc();
52 DI->EmitLocation(Builder, Loc);
53
54 LastStopPoint = Loc;
55 }
56}
57
58void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
59 assert(S && "Null statement?");
60 PGO->setCurrentStmt(S);
61
62 // These statements have their own debug info handling.
63 if (EmitSimpleStmt(S, Attrs))
64 return;
65
66 // Check if we are generating unreachable code.
67 if (!HaveInsertPoint()) {
68 // If so, and the statement doesn't contain a label, then we do not need to
69 // generate actual code. This is safe because (1) the current point is
70 // unreachable, so we don't need to execute the code, and (2) we've already
71 // handled the statements which update internal data structures (like the
72 // local variable map) which could be used by subsequent statements.
73 if (!ContainsLabel(S)) {
74 // Verify that any decl statements were handled as simple, they may be in
75 // scope of subsequent reachable statements.
76 assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
77 PGO->markStmtMaybeUsed(S);
78 return;
79 }
80
81 // Otherwise, make a new block to hold the code.
82 EnsureInsertPoint();
83 }
84
85 // Generate a stoppoint if we are emitting debug info.
86 EmitStopPoint(S);
87
88 // Ignore all OpenMP directives except for simd if OpenMP with Simd is
89 // enabled.
90 if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
91 if (const auto *D = dyn_cast<OMPExecutableDirective>(Val: S)) {
92 EmitSimpleOMPExecutableDirective(D: *D);
93 return;
94 }
95 }
96
97 switch (S->getStmtClass()) {
98 case Stmt::NoStmtClass:
99 case Stmt::CXXCatchStmtClass:
100 case Stmt::SEHExceptStmtClass:
101 case Stmt::SEHFinallyStmtClass:
102 case Stmt::MSDependentExistsStmtClass:
103 case Stmt::UnresolvedSYCLKernelCallStmtClass:
104 llvm_unreachable("invalid statement class to emit generically");
105 case Stmt::NullStmtClass:
106 case Stmt::CompoundStmtClass:
107 case Stmt::DeclStmtClass:
108 case Stmt::LabelStmtClass:
109 case Stmt::AttributedStmtClass:
110 case Stmt::GotoStmtClass:
111 case Stmt::BreakStmtClass:
112 case Stmt::ContinueStmtClass:
113 case Stmt::DefaultStmtClass:
114 case Stmt::CaseStmtClass:
115 case Stmt::DeferStmtClass:
116 case Stmt::SEHLeaveStmtClass:
117 case Stmt::SYCLKernelCallStmtClass:
118 llvm_unreachable("should have emitted these statements as simple");
119
120#define STMT(Type, Base)
121#define ABSTRACT_STMT(Op)
122#define EXPR(Type, Base) \
123 case Stmt::Type##Class:
124#include "clang/AST/StmtNodes.inc"
125 {
126 // Remember the block we came in on.
127 llvm::BasicBlock *incoming = Builder.GetInsertBlock();
128 assert(incoming && "expression emission must have an insertion point");
129
130 EmitIgnoredExpr(E: cast<Expr>(Val: S));
131
132 llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
133 assert(outgoing && "expression emission cleared block!");
134
135 // The expression emitters assume (reasonably!) that the insertion
136 // point is always set. To maintain that, the call-emission code
137 // for noreturn functions has to enter a new block with no
138 // predecessors. We want to kill that block and mark the current
139 // insertion point unreachable in the common case of a call like
140 // "exit();". Since expression emission doesn't otherwise create
141 // blocks with no predecessors, we can just test for that.
142 // However, we must be careful not to do this to our incoming
143 // block, because *statement* emission does sometimes create
144 // reachable blocks which will have no predecessors until later in
145 // the function. This occurs with, e.g., labels that are not
146 // reachable by fallthrough.
147 if (incoming != outgoing && outgoing->use_empty()) {
148 outgoing->eraseFromParent();
149 Builder.ClearInsertionPoint();
150 }
151 break;
152 }
153
154 case Stmt::IndirectGotoStmtClass:
155 EmitIndirectGotoStmt(S: cast<IndirectGotoStmt>(Val: *S)); break;
156
157 case Stmt::IfStmtClass: EmitIfStmt(S: cast<IfStmt>(Val: *S)); break;
158 case Stmt::WhileStmtClass: EmitWhileStmt(S: cast<WhileStmt>(Val: *S), Attrs); break;
159 case Stmt::DoStmtClass: EmitDoStmt(S: cast<DoStmt>(Val: *S), Attrs); break;
160 case Stmt::ForStmtClass: EmitForStmt(S: cast<ForStmt>(Val: *S), Attrs); break;
161
162 case Stmt::ReturnStmtClass: EmitReturnStmt(S: cast<ReturnStmt>(Val: *S)); break;
163
164 case Stmt::SwitchStmtClass: EmitSwitchStmt(S: cast<SwitchStmt>(Val: *S)); break;
165 case Stmt::GCCAsmStmtClass: // Intentional fall-through.
166 case Stmt::MSAsmStmtClass: EmitAsmStmt(S: cast<AsmStmt>(Val: *S)); break;
167 case Stmt::CoroutineBodyStmtClass:
168 EmitCoroutineBody(S: cast<CoroutineBodyStmt>(Val: *S));
169 break;
170 case Stmt::CoreturnStmtClass:
171 EmitCoreturnStmt(S: cast<CoreturnStmt>(Val: *S));
172 break;
173 case Stmt::CapturedStmtClass: {
174 const CapturedStmt *CS = cast<CapturedStmt>(Val: S);
175 EmitCapturedStmt(S: *CS, K: CS->getCapturedRegionKind());
176 }
177 break;
178 case Stmt::ObjCAtTryStmtClass:
179 EmitObjCAtTryStmt(S: cast<ObjCAtTryStmt>(Val: *S));
180 break;
181 case Stmt::ObjCAtCatchStmtClass:
182 llvm_unreachable(
183 "@catch statements should be handled by EmitObjCAtTryStmt");
184 case Stmt::ObjCAtFinallyStmtClass:
185 llvm_unreachable(
186 "@finally statements should be handled by EmitObjCAtTryStmt");
187 case Stmt::ObjCAtThrowStmtClass:
188 EmitObjCAtThrowStmt(S: cast<ObjCAtThrowStmt>(Val: *S));
189 break;
190 case Stmt::ObjCAtSynchronizedStmtClass:
191 EmitObjCAtSynchronizedStmt(S: cast<ObjCAtSynchronizedStmt>(Val: *S));
192 break;
193 case Stmt::ObjCForCollectionStmtClass:
194 EmitObjCForCollectionStmt(S: cast<ObjCForCollectionStmt>(Val: *S));
195 break;
196 case Stmt::ObjCAutoreleasePoolStmtClass:
197 EmitObjCAutoreleasePoolStmt(S: cast<ObjCAutoreleasePoolStmt>(Val: *S));
198 break;
199
200 case Stmt::CXXTryStmtClass:
201 EmitCXXTryStmt(S: cast<CXXTryStmt>(Val: *S));
202 break;
203 case Stmt::CXXForRangeStmtClass:
204 EmitCXXForRangeStmt(S: cast<CXXForRangeStmt>(Val: *S), Attrs);
205 break;
206 case Stmt::SEHTryStmtClass:
207 EmitSEHTryStmt(S: cast<SEHTryStmt>(Val: *S));
208 break;
209 case Stmt::OMPMetaDirectiveClass:
210 EmitOMPMetaDirective(S: cast<OMPMetaDirective>(Val: *S));
211 break;
212 case Stmt::OMPCanonicalLoopClass:
213 EmitOMPCanonicalLoop(S: cast<OMPCanonicalLoop>(Val: S));
214 break;
215 case Stmt::OMPParallelDirectiveClass:
216 EmitOMPParallelDirective(S: cast<OMPParallelDirective>(Val: *S));
217 break;
218 case Stmt::OMPSimdDirectiveClass:
219 EmitOMPSimdDirective(S: cast<OMPSimdDirective>(Val: *S));
220 break;
221 case Stmt::OMPTileDirectiveClass:
222 EmitOMPTileDirective(S: cast<OMPTileDirective>(Val: *S));
223 break;
224 case Stmt::OMPStripeDirectiveClass:
225 EmitOMPStripeDirective(S: cast<OMPStripeDirective>(Val: *S));
226 break;
227 case Stmt::OMPUnrollDirectiveClass:
228 EmitOMPUnrollDirective(S: cast<OMPUnrollDirective>(Val: *S));
229 break;
230 case Stmt::OMPReverseDirectiveClass:
231 EmitOMPReverseDirective(S: cast<OMPReverseDirective>(Val: *S));
232 break;
233 case Stmt::OMPInterchangeDirectiveClass:
234 EmitOMPInterchangeDirective(S: cast<OMPInterchangeDirective>(Val: *S));
235 break;
236 case Stmt::OMPFuseDirectiveClass:
237 EmitOMPFuseDirective(S: cast<OMPFuseDirective>(Val: *S));
238 break;
239 case Stmt::OMPForDirectiveClass:
240 EmitOMPForDirective(S: cast<OMPForDirective>(Val: *S));
241 break;
242 case Stmt::OMPForSimdDirectiveClass:
243 EmitOMPForSimdDirective(S: cast<OMPForSimdDirective>(Val: *S));
244 break;
245 case Stmt::OMPSectionsDirectiveClass:
246 EmitOMPSectionsDirective(S: cast<OMPSectionsDirective>(Val: *S));
247 break;
248 case Stmt::OMPSectionDirectiveClass:
249 EmitOMPSectionDirective(S: cast<OMPSectionDirective>(Val: *S));
250 break;
251 case Stmt::OMPSingleDirectiveClass:
252 EmitOMPSingleDirective(S: cast<OMPSingleDirective>(Val: *S));
253 break;
254 case Stmt::OMPMasterDirectiveClass:
255 EmitOMPMasterDirective(S: cast<OMPMasterDirective>(Val: *S));
256 break;
257 case Stmt::OMPCriticalDirectiveClass:
258 EmitOMPCriticalDirective(S: cast<OMPCriticalDirective>(Val: *S));
259 break;
260 case Stmt::OMPParallelForDirectiveClass:
261 EmitOMPParallelForDirective(S: cast<OMPParallelForDirective>(Val: *S));
262 break;
263 case Stmt::OMPParallelForSimdDirectiveClass:
264 EmitOMPParallelForSimdDirective(S: cast<OMPParallelForSimdDirective>(Val: *S));
265 break;
266 case Stmt::OMPParallelMasterDirectiveClass:
267 EmitOMPParallelMasterDirective(S: cast<OMPParallelMasterDirective>(Val: *S));
268 break;
269 case Stmt::OMPParallelSectionsDirectiveClass:
270 EmitOMPParallelSectionsDirective(S: cast<OMPParallelSectionsDirective>(Val: *S));
271 break;
272 case Stmt::OMPTaskDirectiveClass:
273 EmitOMPTaskDirective(S: cast<OMPTaskDirective>(Val: *S));
274 break;
275 case Stmt::OMPTaskyieldDirectiveClass:
276 EmitOMPTaskyieldDirective(S: cast<OMPTaskyieldDirective>(Val: *S));
277 break;
278 case Stmt::OMPErrorDirectiveClass:
279 EmitOMPErrorDirective(S: cast<OMPErrorDirective>(Val: *S));
280 break;
281 case Stmt::OMPBarrierDirectiveClass:
282 EmitOMPBarrierDirective(S: cast<OMPBarrierDirective>(Val: *S));
283 break;
284 case Stmt::OMPTaskwaitDirectiveClass:
285 EmitOMPTaskwaitDirective(S: cast<OMPTaskwaitDirective>(Val: *S));
286 break;
287 case Stmt::OMPTaskgroupDirectiveClass:
288 EmitOMPTaskgroupDirective(S: cast<OMPTaskgroupDirective>(Val: *S));
289 break;
290 case Stmt::OMPFlushDirectiveClass:
291 EmitOMPFlushDirective(S: cast<OMPFlushDirective>(Val: *S));
292 break;
293 case Stmt::OMPDepobjDirectiveClass:
294 EmitOMPDepobjDirective(S: cast<OMPDepobjDirective>(Val: *S));
295 break;
296 case Stmt::OMPScanDirectiveClass:
297 EmitOMPScanDirective(S: cast<OMPScanDirective>(Val: *S));
298 break;
299 case Stmt::OMPOrderedDirectiveClass:
300 EmitOMPOrderedDirective(S: cast<OMPOrderedDirective>(Val: *S));
301 break;
302 case Stmt::OMPAtomicDirectiveClass:
303 EmitOMPAtomicDirective(S: cast<OMPAtomicDirective>(Val: *S));
304 break;
305 case Stmt::OMPTargetDirectiveClass:
306 EmitOMPTargetDirective(S: cast<OMPTargetDirective>(Val: *S));
307 break;
308 case Stmt::OMPTeamsDirectiveClass:
309 EmitOMPTeamsDirective(S: cast<OMPTeamsDirective>(Val: *S));
310 break;
311 case Stmt::OMPCancellationPointDirectiveClass:
312 EmitOMPCancellationPointDirective(S: cast<OMPCancellationPointDirective>(Val: *S));
313 break;
314 case Stmt::OMPCancelDirectiveClass:
315 EmitOMPCancelDirective(S: cast<OMPCancelDirective>(Val: *S));
316 break;
317 case Stmt::OMPTargetDataDirectiveClass:
318 EmitOMPTargetDataDirective(S: cast<OMPTargetDataDirective>(Val: *S));
319 break;
320 case Stmt::OMPTargetEnterDataDirectiveClass:
321 EmitOMPTargetEnterDataDirective(S: cast<OMPTargetEnterDataDirective>(Val: *S));
322 break;
323 case Stmt::OMPTargetExitDataDirectiveClass:
324 EmitOMPTargetExitDataDirective(S: cast<OMPTargetExitDataDirective>(Val: *S));
325 break;
326 case Stmt::OMPTargetParallelDirectiveClass:
327 EmitOMPTargetParallelDirective(S: cast<OMPTargetParallelDirective>(Val: *S));
328 break;
329 case Stmt::OMPTargetParallelForDirectiveClass:
330 EmitOMPTargetParallelForDirective(S: cast<OMPTargetParallelForDirective>(Val: *S));
331 break;
332 case Stmt::OMPTaskLoopDirectiveClass:
333 EmitOMPTaskLoopDirective(S: cast<OMPTaskLoopDirective>(Val: *S));
334 break;
335 case Stmt::OMPTaskLoopSimdDirectiveClass:
336 EmitOMPTaskLoopSimdDirective(S: cast<OMPTaskLoopSimdDirective>(Val: *S));
337 break;
338 case Stmt::OMPMasterTaskLoopDirectiveClass:
339 EmitOMPMasterTaskLoopDirective(S: cast<OMPMasterTaskLoopDirective>(Val: *S));
340 break;
341 case Stmt::OMPMaskedTaskLoopDirectiveClass:
342 EmitOMPMaskedTaskLoopDirective(S: cast<OMPMaskedTaskLoopDirective>(Val: *S));
343 break;
344 case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
345 EmitOMPMasterTaskLoopSimdDirective(
346 S: cast<OMPMasterTaskLoopSimdDirective>(Val: *S));
347 break;
348 case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
349 EmitOMPMaskedTaskLoopSimdDirective(
350 S: cast<OMPMaskedTaskLoopSimdDirective>(Val: *S));
351 break;
352 case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
353 EmitOMPParallelMasterTaskLoopDirective(
354 S: cast<OMPParallelMasterTaskLoopDirective>(Val: *S));
355 break;
356 case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
357 EmitOMPParallelMaskedTaskLoopDirective(
358 S: cast<OMPParallelMaskedTaskLoopDirective>(Val: *S));
359 break;
360 case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
361 EmitOMPParallelMasterTaskLoopSimdDirective(
362 S: cast<OMPParallelMasterTaskLoopSimdDirective>(Val: *S));
363 break;
364 case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
365 EmitOMPParallelMaskedTaskLoopSimdDirective(
366 S: cast<OMPParallelMaskedTaskLoopSimdDirective>(Val: *S));
367 break;
368 case Stmt::OMPDistributeDirectiveClass:
369 EmitOMPDistributeDirective(S: cast<OMPDistributeDirective>(Val: *S));
370 break;
371 case Stmt::OMPTargetUpdateDirectiveClass:
372 EmitOMPTargetUpdateDirective(S: cast<OMPTargetUpdateDirective>(Val: *S));
373 break;
374 case Stmt::OMPDistributeParallelForDirectiveClass:
375 EmitOMPDistributeParallelForDirective(
376 S: cast<OMPDistributeParallelForDirective>(Val: *S));
377 break;
378 case Stmt::OMPDistributeParallelForSimdDirectiveClass:
379 EmitOMPDistributeParallelForSimdDirective(
380 S: cast<OMPDistributeParallelForSimdDirective>(Val: *S));
381 break;
382 case Stmt::OMPDistributeSimdDirectiveClass:
383 EmitOMPDistributeSimdDirective(S: cast<OMPDistributeSimdDirective>(Val: *S));
384 break;
385 case Stmt::OMPTargetParallelForSimdDirectiveClass:
386 EmitOMPTargetParallelForSimdDirective(
387 S: cast<OMPTargetParallelForSimdDirective>(Val: *S));
388 break;
389 case Stmt::OMPTargetSimdDirectiveClass:
390 EmitOMPTargetSimdDirective(S: cast<OMPTargetSimdDirective>(Val: *S));
391 break;
392 case Stmt::OMPTeamsDistributeDirectiveClass:
393 EmitOMPTeamsDistributeDirective(S: cast<OMPTeamsDistributeDirective>(Val: *S));
394 break;
395 case Stmt::OMPTeamsDistributeSimdDirectiveClass:
396 EmitOMPTeamsDistributeSimdDirective(
397 S: cast<OMPTeamsDistributeSimdDirective>(Val: *S));
398 break;
399 case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
400 EmitOMPTeamsDistributeParallelForSimdDirective(
401 S: cast<OMPTeamsDistributeParallelForSimdDirective>(Val: *S));
402 break;
403 case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
404 EmitOMPTeamsDistributeParallelForDirective(
405 S: cast<OMPTeamsDistributeParallelForDirective>(Val: *S));
406 break;
407 case Stmt::OMPTargetTeamsDirectiveClass:
408 EmitOMPTargetTeamsDirective(S: cast<OMPTargetTeamsDirective>(Val: *S));
409 break;
410 case Stmt::OMPTargetTeamsDistributeDirectiveClass:
411 EmitOMPTargetTeamsDistributeDirective(
412 S: cast<OMPTargetTeamsDistributeDirective>(Val: *S));
413 break;
414 case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
415 EmitOMPTargetTeamsDistributeParallelForDirective(
416 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: *S));
417 break;
418 case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
419 EmitOMPTargetTeamsDistributeParallelForSimdDirective(
420 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: *S));
421 break;
422 case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
423 EmitOMPTargetTeamsDistributeSimdDirective(
424 S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: *S));
425 break;
426 case Stmt::OMPInteropDirectiveClass:
427 EmitOMPInteropDirective(S: cast<OMPInteropDirective>(Val: *S));
428 break;
429 case Stmt::OMPDispatchDirectiveClass:
430 CGM.ErrorUnsupported(S, Type: "OpenMP dispatch directive");
431 break;
432 case Stmt::OMPScopeDirectiveClass:
433 EmitOMPScopeDirective(S: cast<OMPScopeDirective>(Val: *S));
434 break;
435 case Stmt::OMPMaskedDirectiveClass:
436 EmitOMPMaskedDirective(S: cast<OMPMaskedDirective>(Val: *S));
437 break;
438 case Stmt::OMPGenericLoopDirectiveClass:
439 EmitOMPGenericLoopDirective(S: cast<OMPGenericLoopDirective>(Val: *S));
440 break;
441 case Stmt::OMPTeamsGenericLoopDirectiveClass:
442 EmitOMPTeamsGenericLoopDirective(S: cast<OMPTeamsGenericLoopDirective>(Val: *S));
443 break;
444 case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
445 EmitOMPTargetTeamsGenericLoopDirective(
446 S: cast<OMPTargetTeamsGenericLoopDirective>(Val: *S));
447 break;
448 case Stmt::OMPParallelGenericLoopDirectiveClass:
449 EmitOMPParallelGenericLoopDirective(
450 S: cast<OMPParallelGenericLoopDirective>(Val: *S));
451 break;
452 case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
453 EmitOMPTargetParallelGenericLoopDirective(
454 S: cast<OMPTargetParallelGenericLoopDirective>(Val: *S));
455 break;
456 case Stmt::OMPParallelMaskedDirectiveClass:
457 EmitOMPParallelMaskedDirective(S: cast<OMPParallelMaskedDirective>(Val: *S));
458 break;
459 case Stmt::OMPAssumeDirectiveClass:
460 EmitOMPAssumeDirective(S: cast<OMPAssumeDirective>(Val: *S));
461 break;
462 case Stmt::OpenACCComputeConstructClass:
463 EmitOpenACCComputeConstruct(S: cast<OpenACCComputeConstruct>(Val: *S));
464 break;
465 case Stmt::OpenACCLoopConstructClass:
466 EmitOpenACCLoopConstruct(S: cast<OpenACCLoopConstruct>(Val: *S));
467 break;
468 case Stmt::OpenACCCombinedConstructClass:
469 EmitOpenACCCombinedConstruct(S: cast<OpenACCCombinedConstruct>(Val: *S));
470 break;
471 case Stmt::OpenACCDataConstructClass:
472 EmitOpenACCDataConstruct(S: cast<OpenACCDataConstruct>(Val: *S));
473 break;
474 case Stmt::OpenACCEnterDataConstructClass:
475 EmitOpenACCEnterDataConstruct(S: cast<OpenACCEnterDataConstruct>(Val: *S));
476 break;
477 case Stmt::OpenACCExitDataConstructClass:
478 EmitOpenACCExitDataConstruct(S: cast<OpenACCExitDataConstruct>(Val: *S));
479 break;
480 case Stmt::OpenACCHostDataConstructClass:
481 EmitOpenACCHostDataConstruct(S: cast<OpenACCHostDataConstruct>(Val: *S));
482 break;
483 case Stmt::OpenACCWaitConstructClass:
484 EmitOpenACCWaitConstruct(S: cast<OpenACCWaitConstruct>(Val: *S));
485 break;
486 case Stmt::OpenACCInitConstructClass:
487 EmitOpenACCInitConstruct(S: cast<OpenACCInitConstruct>(Val: *S));
488 break;
489 case Stmt::OpenACCShutdownConstructClass:
490 EmitOpenACCShutdownConstruct(S: cast<OpenACCShutdownConstruct>(Val: *S));
491 break;
492 case Stmt::OpenACCSetConstructClass:
493 EmitOpenACCSetConstruct(S: cast<OpenACCSetConstruct>(Val: *S));
494 break;
495 case Stmt::OpenACCUpdateConstructClass:
496 EmitOpenACCUpdateConstruct(S: cast<OpenACCUpdateConstruct>(Val: *S));
497 break;
498 case Stmt::OpenACCAtomicConstructClass:
499 EmitOpenACCAtomicConstruct(S: cast<OpenACCAtomicConstruct>(Val: *S));
500 break;
501 case Stmt::OpenACCCacheConstructClass:
502 EmitOpenACCCacheConstruct(S: cast<OpenACCCacheConstruct>(Val: *S));
503 break;
504 }
505}
506
507bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
508 ArrayRef<const Attr *> Attrs) {
509 switch (S->getStmtClass()) {
510 default:
511 return false;
512 case Stmt::NullStmtClass:
513 break;
514 case Stmt::CompoundStmtClass:
515 EmitCompoundStmt(S: cast<CompoundStmt>(Val: *S));
516 break;
517 case Stmt::DeclStmtClass:
518 EmitDeclStmt(S: cast<DeclStmt>(Val: *S));
519 break;
520 case Stmt::LabelStmtClass:
521 EmitLabelStmt(S: cast<LabelStmt>(Val: *S));
522 break;
523 case Stmt::AttributedStmtClass:
524 EmitAttributedStmt(S: cast<AttributedStmt>(Val: *S));
525 break;
526 case Stmt::GotoStmtClass:
527 EmitGotoStmt(S: cast<GotoStmt>(Val: *S));
528 break;
529 case Stmt::BreakStmtClass:
530 EmitBreakStmt(S: cast<BreakStmt>(Val: *S));
531 break;
532 case Stmt::ContinueStmtClass:
533 EmitContinueStmt(S: cast<ContinueStmt>(Val: *S));
534 break;
535 case Stmt::DefaultStmtClass:
536 EmitDefaultStmt(S: cast<DefaultStmt>(Val: *S), Attrs);
537 break;
538 case Stmt::CaseStmtClass:
539 EmitCaseStmt(S: cast<CaseStmt>(Val: *S), Attrs);
540 break;
541 case Stmt::DeferStmtClass:
542 EmitDeferStmt(S: cast<DeferStmt>(Val: *S));
543 break;
544 case Stmt::SEHLeaveStmtClass:
545 EmitSEHLeaveStmt(S: cast<SEHLeaveStmt>(Val: *S));
546 break;
547 case Stmt::SYCLKernelCallStmtClass:
548 EmitSYCLKernelCallStmt(S: cast<SYCLKernelCallStmt>(Val: *S));
549 break;
550 }
551 return true;
552}
553
554/// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true,
555/// this captures the expression result of the last sub-statement and returns it
556/// (for use by the statement expression extension).
557Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
558 AggValueSlot AggSlot) {
559 PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
560 "LLVM IR generation of compound statement ('{}')");
561
562 // Keep track of the current cleanup stack depth, including debug scopes.
563 LexicalScope Scope(*this, S.getSourceRange());
564
565 return EmitCompoundStmtWithoutScope(S, GetLast, AVS: AggSlot);
566}
567
568Address
569CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
570 bool GetLast,
571 AggValueSlot AggSlot) {
572
573 for (CompoundStmt::const_body_iterator I = S.body_begin(),
574 E = S.body_end() - GetLast;
575 I != E; ++I)
576 EmitStmt(S: *I);
577
578 Address RetAlloca = Address::invalid();
579 if (GetLast) {
580 // We have to special case labels here. They are statements, but when put
581 // at the end of a statement expression, they yield the value of their
582 // subexpression. Handle this by walking through all labels we encounter,
583 // emitting them before we evaluate the subexpr.
584 // Similar issues arise for attributed statements.
585 const Stmt *LastStmt = S.body_back();
586 while (!isa<Expr>(Val: LastStmt)) {
587 if (const auto *LS = dyn_cast<LabelStmt>(Val: LastStmt)) {
588 EmitLabel(D: LS->getDecl());
589 LastStmt = LS->getSubStmt();
590 } else if (const auto *AS = dyn_cast<AttributedStmt>(Val: LastStmt)) {
591 // FIXME: Update this if we ever have attributes that affect the
592 // semantics of an expression.
593 LastStmt = AS->getSubStmt();
594 } else {
595 llvm_unreachable("unknown value statement");
596 }
597 }
598
599 EnsureInsertPoint();
600
601 const Expr *E = cast<Expr>(Val: LastStmt);
602 QualType ExprTy = E->getType();
603 if (hasAggregateEvaluationKind(T: ExprTy)) {
604 EmitAggExpr(E, AS: AggSlot);
605 } else {
606 // We can't return an RValue here because there might be cleanups at
607 // the end of the StmtExpr. Because of that, we have to emit the result
608 // here into a temporary alloca.
609 RetAlloca = CreateMemTemp(T: ExprTy);
610 EmitAnyExprToMem(E, Location: RetAlloca, Quals: Qualifiers(),
611 /*IsInit*/ IsInitializer: false);
612 }
613 }
614
615 return RetAlloca;
616}
617
618void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
619 llvm::BranchInst *BI = dyn_cast<llvm::BranchInst>(Val: BB->getTerminator());
620
621 // If there is a cleanup stack, then we it isn't worth trying to
622 // simplify this block (we would need to remove it from the scope map
623 // and cleanup entry).
624 if (!EHStack.empty())
625 return;
626
627 // Can only simplify direct branches.
628 if (!BI || !BI->isUnconditional())
629 return;
630
631 // Can only simplify empty blocks.
632 if (BI->getIterator() != BB->begin())
633 return;
634
635 BB->replaceAllUsesWith(V: BI->getSuccessor(Idx: 0));
636 BI->eraseFromParent();
637 BB->eraseFromParent();
638}
639
640void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
641 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
642
643 // Fall out of the current block (if necessary).
644 EmitBranch(Block: BB);
645
646 if (IsFinished && BB->use_empty()) {
647 delete BB;
648 return;
649 }
650
651 // Place the block after the current block, if possible, or else at
652 // the end of the function.
653 if (CurBB && CurBB->getParent())
654 CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
655 else
656 CurFn->insert(Position: CurFn->end(), BB);
657 Builder.SetInsertPoint(BB);
658}
659
660void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
661 // Emit a branch from the current block to the target one if this
662 // was a real block. If this was just a fall-through block after a
663 // terminator, don't emit it.
664 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
665
666 if (!CurBB || CurBB->getTerminator()) {
667 // If there is no insert point or the previous block is already
668 // terminated, don't touch it.
669 } else {
670 // Otherwise, create a fall-through branch.
671 Builder.CreateBr(Dest: Target);
672 }
673
674 Builder.ClearInsertionPoint();
675}
676
677void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
678 bool inserted = false;
679 for (llvm::User *u : block->users()) {
680 if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(Val: u)) {
681 CurFn->insert(Position: std::next(x: insn->getParent()->getIterator()), BB: block);
682 inserted = true;
683 break;
684 }
685 }
686
687 if (!inserted)
688 CurFn->insert(Position: CurFn->end(), BB: block);
689
690 Builder.SetInsertPoint(block);
691}
692
693CodeGenFunction::JumpDest
694CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
695 JumpDest &Dest = LabelMap[D];
696 if (Dest.isValid()) return Dest;
697
698 // Create, but don't insert, the new block.
699 Dest = JumpDest(createBasicBlock(name: D->getName()),
700 EHScopeStack::stable_iterator::invalid(),
701 NextCleanupDestIndex++);
702 return Dest;
703}
704
705void CodeGenFunction::EmitLabel(const LabelDecl *D) {
706 // Add this label to the current lexical scope if we're within any
707 // normal cleanups. Jumps "in" to this label --- when permitted by
708 // the language --- may need to be routed around such cleanups.
709 if (EHStack.hasNormalCleanups() && CurLexicalScope)
710 CurLexicalScope->addLabel(label: D);
711
712 JumpDest &Dest = LabelMap[D];
713
714 // If we didn't need a forward reference to this label, just go
715 // ahead and create a destination at the current scope.
716 if (!Dest.isValid()) {
717 Dest = getJumpDestInCurrentScope(Name: D->getName());
718
719 // Otherwise, we need to give this label a target depth and remove
720 // it from the branch-fixups list.
721 } else {
722 assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
723 Dest.setScopeDepth(EHStack.stable_begin());
724 ResolveBranchFixups(Target: Dest.getBlock());
725 }
726
727 EmitBlock(BB: Dest.getBlock());
728
729 // Emit debug info for labels.
730 if (CGDebugInfo *DI = getDebugInfo()) {
731 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
732 DI->setLocation(D->getLocation());
733 DI->EmitLabel(D, Builder);
734 }
735 }
736
737 incrementProfileCounter(S: D->getStmt());
738}
739
740/// Change the cleanup scope of the labels in this lexical scope to
741/// match the scope of the enclosing context.
742void CodeGenFunction::LexicalScope::rescopeLabels() {
743 assert(!Labels.empty());
744 EHScopeStack::stable_iterator innermostScope
745 = CGF.EHStack.getInnermostNormalCleanup();
746
747 // Change the scope depth of all the labels.
748 for (const LabelDecl *Label : Labels) {
749 assert(CGF.LabelMap.count(Label));
750 JumpDest &dest = CGF.LabelMap.find(Val: Label)->second;
751 assert(dest.getScopeDepth().isValid());
752 assert(innermostScope.encloses(dest.getScopeDepth()));
753 dest.setScopeDepth(innermostScope);
754 }
755
756 // Reparent the labels if the new scope also has cleanups.
757 if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
758 ParentScope->Labels.append(in_start: Labels.begin(), in_end: Labels.end());
759 }
760}
761
762
763void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
764 EmitLabel(D: S.getDecl());
765
766 // IsEHa - emit eha.scope.begin if it's a side entry of a scope
767 if (getLangOpts().EHAsynch && S.isSideEntry())
768 EmitSehCppScopeBegin();
769
770 EmitStmt(S: S.getSubStmt());
771}
772
773void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
774 bool nomerge = false;
775 bool noinline = false;
776 bool alwaysinline = false;
777 bool noconvergent = false;
778 HLSLControlFlowHintAttr::Spelling flattenOrBranch =
779 HLSLControlFlowHintAttr::SpellingNotCalculated;
780 const CallExpr *musttail = nullptr;
781 const AtomicAttr *AA = nullptr;
782
783 for (const auto *A : S.getAttrs()) {
784 switch (A->getKind()) {
785 default:
786 break;
787 case attr::NoMerge:
788 nomerge = true;
789 break;
790 case attr::NoInline:
791 noinline = true;
792 break;
793 case attr::AlwaysInline:
794 alwaysinline = true;
795 break;
796 case attr::NoConvergent:
797 noconvergent = true;
798 break;
799 case attr::MustTail: {
800 const Stmt *Sub = S.getSubStmt();
801 const ReturnStmt *R = cast<ReturnStmt>(Val: Sub);
802 musttail = cast<CallExpr>(Val: R->getRetValue()->IgnoreParens());
803 } break;
804 case attr::CXXAssume: {
805 const Expr *Assumption = cast<CXXAssumeAttr>(Val: A)->getAssumption();
806 if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() &&
807 !Assumption->HasSideEffects(Ctx: getContext())) {
808 llvm::Value *AssumptionVal = EmitCheckedArgForAssume(E: Assumption);
809 Builder.CreateAssumption(Cond: AssumptionVal);
810 }
811 } break;
812 case attr::Atomic:
813 AA = cast<AtomicAttr>(Val: A);
814 break;
815 case attr::HLSLControlFlowHint: {
816 flattenOrBranch = cast<HLSLControlFlowHintAttr>(Val: A)->getSemanticSpelling();
817 } break;
818 }
819 }
820 SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);
821 SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);
822 SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);
823 SaveAndRestore save_noconvergent(InNoConvergentAttributedStmt, noconvergent);
824 SaveAndRestore save_musttail(MustTailCall, musttail);
825 SaveAndRestore save_flattenOrBranch(HLSLControlFlowAttr, flattenOrBranch);
826 CGAtomicOptionsRAII AORAII(CGM, AA);
827 EmitStmt(S: S.getSubStmt(), Attrs: S.getAttrs());
828}
829
830void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
831 // If this code is reachable then emit a stop point (if generating
832 // debug info). We have to do this ourselves because we are on the
833 // "simple" statement path.
834 if (HaveInsertPoint())
835 EmitStopPoint(S: &S);
836
837 ApplyAtomGroup Grp(getDebugInfo());
838 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: S.getLabel()));
839}
840
841
842void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
843 ApplyAtomGroup Grp(getDebugInfo());
844 if (const LabelDecl *Target = S.getConstantTarget()) {
845 EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: Target));
846 return;
847 }
848
849 // Ensure that we have an i8* for our PHI node.
850 llvm::Value *V = Builder.CreateBitCast(V: EmitScalarExpr(E: S.getTarget()),
851 DestTy: Int8PtrTy, Name: "addr");
852 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
853
854 // Get the basic block for the indirect goto.
855 llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
856
857 // The first instruction in the block has to be the PHI for the switch dest,
858 // add an entry for this branch.
859 cast<llvm::PHINode>(Val: IndGotoBB->begin())->addIncoming(V, BB: CurBB);
860
861 EmitBranch(Target: IndGotoBB);
862 if (CurBB && CurBB->getTerminator())
863 addInstToCurrentSourceAtom(KeyInstruction: CurBB->getTerminator(), Backup: nullptr);
864}
865
866void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
867 const Stmt *Else = S.getElse();
868
869 // The else branch of a consteval if statement is always the only branch that
870 // can be runtime evaluated.
871 if (S.isConsteval()) {
872 const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;
873 if (Executed) {
874 RunCleanupsScope ExecutedScope(*this);
875 EmitStmt(S: Executed);
876 }
877 return;
878 }
879
880 // C99 6.8.4.1: The first substatement is executed if the expression compares
881 // unequal to 0. The condition must be a scalar type.
882 LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
883 ApplyDebugLocation DL(*this, S.getCond());
884
885 if (S.getInit())
886 EmitStmt(S: S.getInit());
887
888 if (S.getConditionVariable())
889 EmitDecl(D: *S.getConditionVariable());
890
891 // If the condition constant folds and can be elided, try to avoid emitting
892 // the condition and the dead arm of the if/else.
893 bool CondConstant;
894 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: CondConstant,
895 AllowLabels: S.isConstexpr())) {
896 // Figure out which block (then or else) is executed.
897 const Stmt *Executed = S.getThen();
898 const Stmt *Skipped = Else;
899 if (!CondConstant) // Condition false?
900 std::swap(a&: Executed, b&: Skipped);
901
902 // If the skipped block has no labels in it, just emit the executed block.
903 // This avoids emitting dead code and simplifies the CFG substantially.
904 if (S.isConstexpr() || !ContainsLabel(S: Skipped)) {
905 incrementProfileCounter(ExecSkip: CondConstant ? UseExecPath : UseSkipPath, S: &S,
906 /*UseBoth=*/true);
907 if (Executed) {
908 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
909 RunCleanupsScope ExecutedScope(*this);
910 EmitStmt(S: Executed);
911 }
912 PGO->markStmtMaybeUsed(S: Skipped);
913 return;
914 }
915 }
916
917 auto HasSkip = hasSkipCounter(S: &S);
918
919 // Otherwise, the condition did not fold, or we couldn't elide it. Just emit
920 // the conditional branch.
921 llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.then");
922 llvm::BasicBlock *ContBlock = createBasicBlock(name: "if.end");
923 llvm::BasicBlock *ElseBlock =
924 (Else || HasSkip ? createBasicBlock(name: "if.else") : ContBlock);
925 // Prefer the PGO based weights over the likelihood attribute.
926 // When the build isn't optimized the metadata isn't used, so don't generate
927 // it.
928 // Also, differentiate between disabled PGO and a never executed branch with
929 // PGO. Assuming PGO is in use:
930 // - we want to ignore the [[likely]] attribute if the branch is never
931 // executed,
932 // - assuming the profile is poor, preserving the attribute may still be
933 // beneficial.
934 // As an approximation, preserve the attribute only if both the branch and the
935 // parent context were not executed.
936 Stmt::Likelihood LH = Stmt::LH_None;
937 uint64_t ThenCount = getProfileCount(S: S.getThen());
938 if (!ThenCount && !getCurrentProfileCount() &&
939 CGM.getCodeGenOpts().OptimizationLevel)
940 LH = Stmt::getLikelihood(Then: S.getThen(), Else);
941
942 // When measuring MC/DC, always fully evaluate the condition up front using
943 // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
944 // executing the body of the if.then or if.else. This is useful for when
945 // there is a 'return' within the body, but this is particularly beneficial
946 // when one if-stmt is nested within another if-stmt so that all of the MC/DC
947 // updates are kept linear and consistent.
948 if (!CGM.getCodeGenOpts().MCDCCoverage) {
949 EmitBranchOnBoolExpr(Cond: S.getCond(), TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount: ThenCount, LH,
950 /*ConditionalOp=*/nullptr,
951 /*ConditionalDecl=*/S.getConditionVariable());
952 } else {
953 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
954 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
955 Builder.CreateCondBr(Cond: BoolCondVal, True: ThenBlock, False: ElseBlock);
956 }
957
958 // Emit the 'then' code.
959 EmitBlock(BB: ThenBlock);
960 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
961 {
962 RunCleanupsScope ThenScope(*this);
963 EmitStmt(S: S.getThen());
964 }
965 EmitBranch(Target: ContBlock);
966
967 // Emit the 'else' code if present.
968 if (Else) {
969 {
970 // There is no need to emit line number for an unconditional branch.
971 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
972 EmitBlock(BB: ElseBlock);
973 }
974 // Add a counter to else block unless it has CounterExpr.
975 if (HasSkip)
976 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
977 {
978 RunCleanupsScope ElseScope(*this);
979 EmitStmt(S: Else);
980 }
981 {
982 // There is no need to emit line number for an unconditional branch.
983 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
984 EmitBranch(Target: ContBlock);
985 }
986 } else if (HasSkip) {
987 EmitBlock(BB: ElseBlock);
988 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
989 EmitBranch(Target: ContBlock);
990 }
991
992 // Emit the continuation block for code after the if.
993 EmitBlock(BB: ContBlock, IsFinished: true);
994}
995
996bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression,
997 bool HasEmptyBody) {
998 if (CGM.getCodeGenOpts().getFiniteLoops() ==
999 CodeGenOptions::FiniteLoopsKind::Never)
1000 return false;
1001
1002 // Now apply rules for plain C (see 6.8.5.6 in C11).
1003 // Loops with constant conditions do not have to make progress in any C
1004 // version.
1005 // As an extension, we consisider loops whose constant expression
1006 // can be constant-folded.
1007 Expr::EvalResult Result;
1008 bool CondIsConstInt =
1009 !ControllingExpression ||
1010 (ControllingExpression->EvaluateAsInt(Result, Ctx: getContext()) &&
1011 Result.Val.isInt());
1012
1013 bool CondIsTrue = CondIsConstInt && (!ControllingExpression ||
1014 Result.Val.getInt().getBoolValue());
1015
1016 // Loops with non-constant conditions must make progress in C11 and later.
1017 if (getLangOpts().C11 && !CondIsConstInt)
1018 return true;
1019
1020 // [C++26][intro.progress] (DR)
1021 // The implementation may assume that any thread will eventually do one of the
1022 // following:
1023 // [...]
1024 // - continue execution of a trivial infinite loop ([stmt.iter.general]).
1025 if (CGM.getCodeGenOpts().getFiniteLoops() ==
1026 CodeGenOptions::FiniteLoopsKind::Always ||
1027 getLangOpts().CPlusPlus11) {
1028 if (HasEmptyBody && CondIsTrue) {
1029 CurFn->removeFnAttr(Kind: llvm::Attribute::MustProgress);
1030 return false;
1031 }
1032 return true;
1033 }
1034 return false;
1035}
1036
1037// [C++26][stmt.iter.general] (DR)
1038// A trivially empty iteration statement is an iteration statement matching one
1039// of the following forms:
1040// - while ( expression ) ;
1041// - while ( expression ) { }
1042// - do ; while ( expression ) ;
1043// - do { } while ( expression ) ;
1044// - for ( init-statement expression(opt); ) ;
1045// - for ( init-statement expression(opt); ) { }
1046template <typename LoopStmt> static bool hasEmptyLoopBody(const LoopStmt &S) {
1047 if constexpr (std::is_same_v<LoopStmt, ForStmt>) {
1048 if (S.getInc())
1049 return false;
1050 }
1051 const Stmt *Body = S.getBody();
1052 if (!Body || isa<NullStmt>(Val: Body))
1053 return true;
1054 if (const CompoundStmt *Compound = dyn_cast<CompoundStmt>(Val: Body))
1055 return Compound->body_empty();
1056 return false;
1057}
1058
1059void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
1060 ArrayRef<const Attr *> WhileAttrs) {
1061 // Emit the header for the loop, which will also become
1062 // the continue target.
1063 JumpDest LoopHeader = getJumpDestInCurrentScope(Name: "while.cond");
1064 EmitBlock(BB: LoopHeader.getBlock());
1065
1066 if (CGM.shouldEmitConvergenceTokens())
1067 ConvergenceTokenStack.push_back(
1068 Elt: emitConvergenceLoopToken(BB: LoopHeader.getBlock()));
1069
1070 // Create an exit block for when the condition fails, which will
1071 // also become the break target.
1072 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "while.end");
1073
1074 // Store the blocks to use for break and continue.
1075 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopHeader));
1076
1077 // C++ [stmt.while]p2:
1078 // When the condition of a while statement is a declaration, the
1079 // scope of the variable that is declared extends from its point
1080 // of declaration (3.3.2) to the end of the while statement.
1081 // [...]
1082 // The object created in a condition is destroyed and created
1083 // with each iteration of the loop.
1084 RunCleanupsScope ConditionScope(*this);
1085
1086 if (S.getConditionVariable())
1087 EmitDecl(D: *S.getConditionVariable());
1088
1089 // Evaluate the conditional in the while header. C99 6.8.5.1: The
1090 // evaluation of the controlling expression takes place before each
1091 // execution of the loop body.
1092 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1093
1094 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1095
1096 // while(1) is common, avoid extra exit blocks. Be sure
1097 // to correctly handle break/continue though.
1098 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1099 bool EmitBoolCondBranch = !C || !C->isOne();
1100 const SourceRange &R = S.getSourceRange();
1101 LoopStack.push(Header: LoopHeader.getBlock(), Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
1102 Attrs: WhileAttrs, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1103 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1104 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1105
1106 // As long as the condition is true, go to the loop body.
1107 llvm::BasicBlock *LoopBody = createBasicBlock(name: "while.body");
1108 if (EmitBoolCondBranch) {
1109 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1110 if (hasSkipCounter(S: &S) || ConditionScope.requiresCleanups())
1111 ExitBlock = createBasicBlock(name: "while.exit");
1112 llvm::MDNode *Weights =
1113 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1114 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1115 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1116 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1117 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock, BranchWeights: Weights);
1118 // Key Instructions: Emit the condition and branch as separate source
1119 // location atoms otherwise we may omit a step onto the loop condition in
1120 // favour of the `while` keyword.
1121 // FIXME: We could have the branch as the backup location for the condition,
1122 // which would probably be a better experience. Explore this later.
1123 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1124 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1125 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1126
1127 if (ExitBlock != LoopExit.getBlock()) {
1128 EmitBlock(BB: ExitBlock);
1129 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1130 EmitBranchThroughCleanup(Dest: LoopExit);
1131 }
1132 } else if (const Attr *A = Stmt::getLikelihoodAttr(S: S.getBody())) {
1133 CGM.getDiags().Report(Loc: A->getLocation(),
1134 DiagID: diag::warn_attribute_has_no_effect_on_infinite_loop)
1135 << A << A->getRange();
1136 CGM.getDiags().Report(
1137 Loc: S.getWhileLoc(),
1138 DiagID: diag::note_attribute_has_no_effect_on_infinite_loop_here)
1139 << SourceRange(S.getWhileLoc(), S.getRParenLoc());
1140 }
1141
1142 // Emit the loop body. We have to emit this in a cleanup scope
1143 // because it might be a singleton DeclStmt.
1144 {
1145 RunCleanupsScope BodyScope(*this);
1146 EmitBlock(BB: LoopBody);
1147 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1148 EmitStmt(S: S.getBody());
1149 }
1150
1151 BreakContinueStack.pop_back();
1152
1153 // Immediately force cleanup.
1154 ConditionScope.ForceCleanup();
1155
1156 EmitStopPoint(S: &S);
1157 // Branch to the loop header again.
1158 EmitBranch(Target: LoopHeader.getBlock());
1159
1160 LoopStack.pop();
1161
1162 // Emit the exit block.
1163 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1164
1165 // The LoopHeader typically is just a branch if we skipped emitting
1166 // a branch, try to erase it.
1167 if (!EmitBoolCondBranch) {
1168 SimplifyForwardingBlocks(BB: LoopHeader.getBlock());
1169 PGO->markStmtAsUsed(Skipped: true, S: &S);
1170 }
1171
1172 if (CGM.shouldEmitConvergenceTokens())
1173 ConvergenceTokenStack.pop_back();
1174}
1175
1176void CodeGenFunction::EmitDoStmt(const DoStmt &S,
1177 ArrayRef<const Attr *> DoAttrs) {
1178 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "do.end");
1179 JumpDest LoopCond = getJumpDestInCurrentScope(Name: "do.cond");
1180
1181 uint64_t ParentCount = getCurrentProfileCount();
1182
1183 // Store the blocks to use for break and continue.
1184 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, LoopCond));
1185
1186 // Emit the body of the loop.
1187 llvm::BasicBlock *LoopBody = createBasicBlock(name: "do.body");
1188
1189 EmitBlockWithFallThrough(BB: LoopBody, S: &S);
1190
1191 if (CGM.shouldEmitConvergenceTokens())
1192 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: LoopBody));
1193
1194 {
1195 RunCleanupsScope BodyScope(*this);
1196 EmitStmt(S: S.getBody());
1197 }
1198
1199 EmitBlock(BB: LoopCond.getBlock());
1200
1201 // C99 6.8.5.2: "The evaluation of the controlling expression takes place
1202 // after each execution of the loop body."
1203
1204 // Evaluate the conditional in the while header.
1205 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1206 // compares unequal to 0. The condition must be a scalar type.
1207 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1208
1209 BreakContinueStack.pop_back();
1210
1211 // "do {} while (0)" is common in macros, avoid extra blocks. Be sure
1212 // to correctly handle break/continue though.
1213 llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1214 bool EmitBoolCondBranch = !C || !C->isZero();
1215
1216 const SourceRange &R = S.getSourceRange();
1217 LoopStack.push(Header: LoopBody, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: DoAttrs,
1218 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1219 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1220 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1221
1222 auto *LoopFalse = (hasSkipCounter(S: &S) ? createBasicBlock(name: "do.loopfalse")
1223 : LoopExit.getBlock());
1224
1225 // As long as the condition is true, iterate the loop.
1226 if (EmitBoolCondBranch) {
1227 uint64_t BackedgeCount = getProfileCount(S: S.getBody()) - ParentCount;
1228 auto *I = Builder.CreateCondBr(
1229 Cond: BoolCondVal, True: LoopBody, False: LoopFalse,
1230 BranchWeights: createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: BackedgeCount));
1231
1232 // Key Instructions: Emit the condition and branch as separate source
1233 // location atoms otherwise we may omit a step onto the loop condition in
1234 // favour of the closing brace.
1235 // FIXME: We could have the branch as the backup location for the condition,
1236 // which would probably be a better experience (no jumping to the brace).
1237 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1238 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1239 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1240 }
1241
1242 LoopStack.pop();
1243
1244 if (LoopFalse != LoopExit.getBlock()) {
1245 EmitBlock(BB: LoopFalse);
1246 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S, /*UseBoth=*/true);
1247 }
1248
1249 // Emit the exit block.
1250 EmitBlock(BB: LoopExit.getBlock());
1251
1252 // The DoCond block typically is just a branch if we skipped
1253 // emitting a branch, try to erase it.
1254 if (!EmitBoolCondBranch)
1255 SimplifyForwardingBlocks(BB: LoopCond.getBlock());
1256
1257 if (CGM.shouldEmitConvergenceTokens())
1258 ConvergenceTokenStack.pop_back();
1259}
1260
1261void CodeGenFunction::EmitForStmt(const ForStmt &S,
1262 ArrayRef<const Attr *> ForAttrs) {
1263 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1264
1265 std::optional<LexicalScope> ForScope;
1266 if (getLangOpts().C99 || getLangOpts().CPlusPlus)
1267 ForScope.emplace(args&: *this, args: S.getSourceRange());
1268
1269 // Evaluate the first part before the loop.
1270 if (S.getInit())
1271 EmitStmt(S: S.getInit());
1272
1273 // Start the loop with a block that tests the condition.
1274 // If there's an increment, the continue scope will be overwritten
1275 // later.
1276 JumpDest CondDest = getJumpDestInCurrentScope(Name: "for.cond");
1277 llvm::BasicBlock *CondBlock = CondDest.getBlock();
1278 EmitBlock(BB: CondBlock);
1279
1280 if (CGM.shouldEmitConvergenceTokens())
1281 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1282
1283 const SourceRange &R = S.getSourceRange();
1284 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1285 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1286 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1287 MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1288
1289 // Create a cleanup scope for the condition variable cleanups.
1290 LexicalScope ConditionScope(*this, S.getSourceRange());
1291
1292 // If the for loop doesn't have an increment we can just use the condition as
1293 // the continue block. Otherwise, if there is no condition variable, we can
1294 // form the continue block now. If there is a condition variable, we can't
1295 // form the continue block until after we've emitted the condition, because
1296 // the condition is in scope in the increment, but Sema's jump diagnostics
1297 // ensure that there are no continues from the condition variable that jump
1298 // to the loop increment.
1299 JumpDest Continue;
1300 if (!S.getInc())
1301 Continue = CondDest;
1302 else if (!S.getConditionVariable())
1303 Continue = getJumpDestInCurrentScope(Name: "for.inc");
1304 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1305
1306 if (S.getCond()) {
1307 // If the for statement has a condition scope, emit the local variable
1308 // declaration.
1309 if (S.getConditionVariable()) {
1310 EmitDecl(D: *S.getConditionVariable());
1311
1312 // We have entered the condition variable's scope, so we're now able to
1313 // jump to the continue block.
1314 Continue = S.getInc() ? getJumpDestInCurrentScope(Name: "for.inc") : CondDest;
1315 BreakContinueStack.back().ContinueBlock = Continue;
1316 }
1317
1318 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1319 // If there are any cleanups between here and the loop-exit scope,
1320 // create a block to stage a loop exit along.
1321 if (hasSkipCounter(S: &S) || (ForScope && ForScope->requiresCleanups()))
1322 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1323
1324 // As long as the condition is true, iterate the loop.
1325 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1326
1327 // C99 6.8.5p2/p4: The first substatement is executed if the expression
1328 // compares unequal to 0. The condition must be a scalar type.
1329 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1330
1331 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1332
1333 llvm::MDNode *Weights =
1334 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1335 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1336 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1337 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1338
1339 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1340 // Key Instructions: Emit the condition and branch as separate atoms to
1341 // match existing loop stepping behaviour. FIXME: We could have the branch
1342 // as the backup location for the condition, which would probably be a
1343 // better experience (no jumping to the brace).
1344 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1345 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1346 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1347
1348 if (ExitBlock != LoopExit.getBlock()) {
1349 EmitBlock(BB: ExitBlock);
1350 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1351 EmitBranchThroughCleanup(Dest: LoopExit);
1352 }
1353
1354 EmitBlock(BB: ForBody);
1355 } else {
1356 // Treat it as a non-zero constant. Don't even create a new block for the
1357 // body, just fall into it.
1358 PGO->markStmtAsUsed(Skipped: true, S: &S);
1359 }
1360
1361 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1362
1363 {
1364 // Create a separate cleanup scope for the body, in case it is not
1365 // a compound statement.
1366 RunCleanupsScope BodyScope(*this);
1367 EmitStmt(S: S.getBody());
1368 }
1369
1370 // The last block in the loop's body (which unconditionally branches to the
1371 // `inc` block if there is one).
1372 auto *FinalBodyBB = Builder.GetInsertBlock();
1373
1374 // If there is an increment, emit it next.
1375 if (S.getInc()) {
1376 EmitBlock(BB: Continue.getBlock());
1377 EmitStmt(S: S.getInc());
1378 }
1379
1380 BreakContinueStack.pop_back();
1381
1382 ConditionScope.ForceCleanup();
1383
1384 EmitStopPoint(S: &S);
1385 EmitBranch(Target: CondBlock);
1386
1387 if (ForScope)
1388 ForScope->ForceCleanup();
1389
1390 LoopStack.pop();
1391
1392 // Emit the fall-through block.
1393 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1394
1395 if (CGM.shouldEmitConvergenceTokens())
1396 ConvergenceTokenStack.pop_back();
1397
1398 if (FinalBodyBB) {
1399 // Key Instructions: We want the for closing brace to be step-able on to
1400 // match existing behaviour.
1401 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1402 }
1403}
1404
1405void
1406CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1407 ArrayRef<const Attr *> ForAttrs) {
1408 JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1409
1410 LexicalScope ForScope(*this, S.getSourceRange());
1411
1412 // Evaluate the first pieces before the loop.
1413 if (S.getInit())
1414 EmitStmt(S: S.getInit());
1415 EmitStmt(S: S.getRangeStmt());
1416 EmitStmt(S: S.getBeginStmt());
1417 EmitStmt(S: S.getEndStmt());
1418
1419 // Start the loop with a block that tests the condition.
1420 // If there's an increment, the continue scope will be overwritten
1421 // later.
1422 llvm::BasicBlock *CondBlock = createBasicBlock(name: "for.cond");
1423 EmitBlock(BB: CondBlock);
1424
1425 if (CGM.shouldEmitConvergenceTokens())
1426 ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1427
1428 const SourceRange &R = S.getSourceRange();
1429 LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1430 StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1431 EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
1432
1433 // If there are any cleanups between here and the loop-exit scope,
1434 // create a block to stage a loop exit along.
1435 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1436 if (hasSkipCounter(S: &S) || ForScope.requiresCleanups())
1437 ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1438
1439 // The loop body, consisting of the specified body and the loop variable.
1440 llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1441
1442 // The body is executed if the expression, contextually converted
1443 // to bool, is true.
1444 llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1445 llvm::MDNode *Weights =
1446 createProfileWeightsForLoop(Cond: S.getCond(), LoopCount: getProfileCount(S: S.getBody()));
1447 if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1448 BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1449 Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1450 auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1451 // Key Instructions: Emit the condition and branch as separate atoms to
1452 // match existing loop stepping behaviour. FIXME: We could have the branch as
1453 // the backup location for the condition, which would probably be a better
1454 // experience.
1455 if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1456 addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1457 addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1458
1459 if (ExitBlock != LoopExit.getBlock()) {
1460 EmitBlock(BB: ExitBlock);
1461 incrementProfileCounter(ExecSkip: UseSkipPath, S: &S);
1462 EmitBranchThroughCleanup(Dest: LoopExit);
1463 }
1464
1465 EmitBlock(BB: ForBody);
1466 incrementProfileCounter(ExecSkip: UseExecPath, S: &S);
1467
1468 // Create a block for the increment. In case of a 'continue', we jump there.
1469 JumpDest Continue = getJumpDestInCurrentScope(Name: "for.inc");
1470
1471 // Store the blocks to use for break and continue.
1472 BreakContinueStack.push_back(Elt: BreakContinue(S, LoopExit, Continue));
1473
1474 {
1475 // Create a separate cleanup scope for the loop variable and body.
1476 LexicalScope BodyScope(*this, S.getSourceRange());
1477 EmitStmt(S: S.getLoopVarStmt());
1478 EmitStmt(S: S.getBody());
1479 }
1480 // The last block in the loop's body (which unconditionally branches to the
1481 // `inc` block if there is one).
1482 auto *FinalBodyBB = Builder.GetInsertBlock();
1483
1484 EmitStopPoint(S: &S);
1485 // If there is an increment, emit it next.
1486 EmitBlock(BB: Continue.getBlock());
1487 EmitStmt(S: S.getInc());
1488
1489 BreakContinueStack.pop_back();
1490
1491 EmitBranch(Target: CondBlock);
1492
1493 ForScope.ForceCleanup();
1494
1495 LoopStack.pop();
1496
1497 // Emit the fall-through block.
1498 EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1499
1500 if (CGM.shouldEmitConvergenceTokens())
1501 ConvergenceTokenStack.pop_back();
1502
1503 if (FinalBodyBB) {
1504 // We want the for closing brace to be step-able on to match existing
1505 // behaviour.
1506 addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1507 }
1508}
1509
1510void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1511 if (RV.isScalar()) {
1512 Builder.CreateStore(Val: RV.getScalarVal(), Addr: ReturnValue);
1513 } else if (RV.isAggregate()) {
1514 LValue Dest = MakeAddrLValue(Addr: ReturnValue, T: Ty);
1515 LValue Src = MakeAddrLValue(Addr: RV.getAggregateAddress(), T: Ty);
1516 EmitAggregateCopy(Dest, Src, EltTy: Ty, MayOverlap: getOverlapForReturnValue());
1517 } else {
1518 EmitStoreOfComplex(V: RV.getComplexVal(), dest: MakeAddrLValue(Addr: ReturnValue, T: Ty),
1519 /*init*/ isInit: true);
1520 }
1521 EmitBranchThroughCleanup(Dest: ReturnBlock);
1522}
1523
1524namespace {
1525// RAII struct used to save and restore a return statment's result expression.
1526struct SaveRetExprRAII {
1527 SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1528 : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1529 CGF.RetExpr = RetExpr;
1530 }
1531 ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1532 const Expr *OldRetExpr;
1533 CodeGenFunction &CGF;
1534};
1535} // namespace
1536
1537/// Determine if the given call uses the swiftasync calling convention.
1538static bool isSwiftAsyncCallee(const CallExpr *CE) {
1539 auto calleeQualType = CE->getCallee()->getType();
1540 const FunctionType *calleeType = nullptr;
1541 if (calleeQualType->isFunctionPointerType() ||
1542 calleeQualType->isFunctionReferenceType() ||
1543 calleeQualType->isBlockPointerType() ||
1544 calleeQualType->isMemberFunctionPointerType()) {
1545 calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1546 } else if (auto *ty = dyn_cast<FunctionType>(Val&: calleeQualType)) {
1547 calleeType = ty;
1548 } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(Val: CE)) {
1549 if (auto methodDecl = CMCE->getMethodDecl()) {
1550 // getMethodDecl() doesn't handle member pointers at the moment.
1551 calleeType = methodDecl->getType()->castAs<FunctionType>();
1552 } else {
1553 return false;
1554 }
1555 } else {
1556 return false;
1557 }
1558 return calleeType->getCallConv() == CallingConv::CC_SwiftAsync;
1559}
1560
1561/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1562/// if the function returns void, or may be missing one if the function returns
1563/// non-void. Fun stuff :).
1564void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1565 ApplyAtomGroup Grp(getDebugInfo());
1566 if (requiresReturnValueCheck()) {
1567 llvm::Constant *SLoc = EmitCheckSourceLocation(Loc: S.getBeginLoc());
1568 auto *SLocPtr =
1569 new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1570 llvm::GlobalVariable::PrivateLinkage, SLoc);
1571 SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1572 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: SLocPtr);
1573 assert(ReturnLocation.isValid() && "No valid return location");
1574 Builder.CreateStore(Val: SLocPtr, Addr: ReturnLocation);
1575 }
1576
1577 // Returning from an outlined SEH helper is UB, and we already warn on it.
1578 if (IsOutlinedSEHHelper) {
1579 Builder.CreateUnreachable();
1580 Builder.ClearInsertionPoint();
1581 }
1582
1583 // Emit the result value, even if unused, to evaluate the side effects.
1584 const Expr *RV = S.getRetValue();
1585
1586 // Record the result expression of the return statement. The recorded
1587 // expression is used to determine whether a block capture's lifetime should
1588 // end at the end of the full expression as opposed to the end of the scope
1589 // enclosing the block expression.
1590 //
1591 // This permits a small, easily-implemented exception to our over-conservative
1592 // rules about not jumping to statements following block literals with
1593 // non-trivial cleanups.
1594 SaveRetExprRAII SaveRetExpr(RV, *this);
1595
1596 RunCleanupsScope cleanupScope(*this);
1597 if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(Val: RV))
1598 RV = EWC->getSubExpr();
1599
1600 // If we're in a swiftasynccall function, and the return expression is a
1601 // call to a swiftasynccall function, mark the call as the musttail call.
1602 std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail;
1603 if (RV && CurFnInfo &&
1604 CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) {
1605 if (auto CE = dyn_cast<CallExpr>(Val: RV)) {
1606 if (isSwiftAsyncCallee(CE)) {
1607 SaveMustTail.emplace(args&: MustTailCall, args&: CE);
1608 }
1609 }
1610 }
1611
1612 // FIXME: Clean this up by using an LValue for ReturnTemp,
1613 // EmitStoreThroughLValue, and EmitAnyExpr.
1614 // Check if the NRVO candidate was not globalized in OpenMP mode.
1615 if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1616 S.getNRVOCandidate()->isNRVOVariable() &&
1617 (!getLangOpts().OpenMP ||
1618 !CGM.getOpenMPRuntime()
1619 .getAddressOfLocalVariable(CGF&: *this, VD: S.getNRVOCandidate())
1620 .isValid())) {
1621 // Apply the named return value optimization for this return statement,
1622 // which means doing nothing: the appropriate result has already been
1623 // constructed into the NRVO variable.
1624
1625 // If there is an NRVO flag for this variable, set it to 1 into indicate
1626 // that the cleanup code should not destroy the variable.
1627 if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1628 Builder.CreateFlagStore(Value: Builder.getTrue(), Addr: NRVOFlag);
1629 } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1630 // Make sure not to return anything, but evaluate the expression
1631 // for side effects.
1632 if (RV) {
1633 EmitAnyExpr(E: RV);
1634 }
1635 } else if (!RV) {
1636 // Do nothing (return value is left uninitialized)
1637 } else if (FnRetTy->isReferenceType()) {
1638 // If this function returns a reference, take the address of the expression
1639 // rather than the value.
1640 RValue Result = EmitReferenceBindingToExpr(E: RV);
1641 auto *I = Builder.CreateStore(Val: Result.getScalarVal(), Addr: ReturnValue);
1642 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1643 } else {
1644 switch (getEvaluationKind(T: RV->getType())) {
1645 case TEK_Scalar: {
1646 llvm::Value *Ret = EmitScalarExpr(E: RV);
1647 if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1648 EmitStoreOfScalar(value: Ret, lvalue: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1649 /*isInit*/ true);
1650 } else {
1651 auto *I = Builder.CreateStore(Val: Ret, Addr: ReturnValue);
1652 addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1653 }
1654 break;
1655 }
1656 case TEK_Complex:
1657 EmitComplexExprIntoLValue(E: RV, dest: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1658 /*isInit*/ true);
1659 break;
1660 case TEK_Aggregate:
1661 EmitAggExpr(E: RV, AS: AggValueSlot::forAddr(
1662 addr: ReturnValue, quals: Qualifiers(),
1663 isDestructed: AggValueSlot::IsDestructed,
1664 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1665 isAliased: AggValueSlot::IsNotAliased,
1666 mayOverlap: getOverlapForReturnValue()));
1667 break;
1668 }
1669 }
1670
1671 ++NumReturnExprs;
1672 if (!RV || RV->isEvaluatable(Ctx: getContext()))
1673 ++NumSimpleReturnExprs;
1674
1675 cleanupScope.ForceCleanup();
1676 EmitBranchThroughCleanup(Dest: ReturnBlock);
1677}
1678
1679void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1680 // As long as debug info is modeled with instructions, we have to ensure we
1681 // have a place to insert here and write the stop point here.
1682 if (HaveInsertPoint())
1683 EmitStopPoint(S: &S);
1684
1685 for (const auto *I : S.decls())
1686 EmitDecl(D: *I, /*EvaluateConditionDecl=*/true);
1687}
1688
1689auto CodeGenFunction::GetDestForLoopControlStmt(const LoopControlStmt &S)
1690 -> const BreakContinue * {
1691 if (!S.hasLabelTarget())
1692 return &BreakContinueStack.back();
1693
1694 const Stmt *LoopOrSwitch = S.getNamedLoopOrSwitch();
1695 assert(LoopOrSwitch && "break/continue target not set?");
1696 for (const BreakContinue &BC : llvm::reverse(C&: BreakContinueStack))
1697 if (BC.LoopOrSwitch == LoopOrSwitch)
1698 return &BC;
1699
1700 llvm_unreachable("break/continue target not found");
1701}
1702
1703void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1704 assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1705
1706 // If this code is reachable then emit a stop point (if generating
1707 // debug info). We have to do this ourselves because we are on the
1708 // "simple" statement path.
1709 if (HaveInsertPoint())
1710 EmitStopPoint(S: &S);
1711
1712 ApplyAtomGroup Grp(getDebugInfo());
1713 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->BreakBlock);
1714}
1715
1716void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1717 assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1718
1719 // If this code is reachable then emit a stop point (if generating
1720 // debug info). We have to do this ourselves because we are on the
1721 // "simple" statement path.
1722 if (HaveInsertPoint())
1723 EmitStopPoint(S: &S);
1724
1725 ApplyAtomGroup Grp(getDebugInfo());
1726 EmitBranchThroughCleanup(Dest: GetDestForLoopControlStmt(S)->ContinueBlock);
1727}
1728
1729/// EmitCaseStmtRange - If case statement range is not too big then
1730/// add multiple cases to switch instruction, one for each value within
1731/// the range. If range is too big then emit "if" condition check.
1732void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1733 ArrayRef<const Attr *> Attrs) {
1734 assert(S.getRHS() && "Expected RHS value in CaseStmt");
1735
1736 llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(Ctx: getContext());
1737 llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(Ctx: getContext());
1738
1739 // Emit the code for this case. We do this first to make sure it is
1740 // properly chained from our predecessor before generating the
1741 // switch machinery to enter this block.
1742 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1743 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1744 EmitStmt(S: S.getSubStmt());
1745
1746 // If range is empty, do nothing.
1747 if (LHS.isSigned() ? RHS.slt(RHS: LHS) : RHS.ult(RHS: LHS))
1748 return;
1749
1750 Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1751 llvm::APInt Range = RHS - LHS;
1752 // FIXME: parameters such as this should not be hardcoded.
1753 if (Range.ult(RHS: llvm::APInt(Range.getBitWidth(), 64))) {
1754 // Range is small enough to add multiple switch instruction cases.
1755 uint64_t Total = getProfileCount(S: &S);
1756 unsigned NCases = Range.getZExtValue() + 1;
1757 // We only have one region counter for the entire set of cases here, so we
1758 // need to divide the weights evenly between the generated cases, ensuring
1759 // that the total weight is preserved. E.g., a weight of 5 over three cases
1760 // will be distributed as weights of 2, 2, and 1.
1761 uint64_t Weight = Total / NCases, Rem = Total % NCases;
1762 for (unsigned I = 0; I != NCases; ++I) {
1763 if (SwitchWeights)
1764 SwitchWeights->push_back(Elt: Weight + (Rem ? 1 : 0));
1765 else if (SwitchLikelihood)
1766 SwitchLikelihood->push_back(Elt: LH);
1767
1768 if (Rem)
1769 Rem--;
1770 SwitchInsn->addCase(OnVal: Builder.getInt(AI: LHS), Dest: CaseDest);
1771 ++LHS;
1772 }
1773 return;
1774 }
1775
1776 // The range is too big. Emit "if" condition into a new block,
1777 // making sure to save and restore the current insertion point.
1778 llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1779
1780 // Push this test onto the chain of range checks (which terminates
1781 // in the default basic block). The switch's default will be changed
1782 // to the top of this chain after switch emission is complete.
1783 llvm::BasicBlock *FalseDest = CaseRangeBlock;
1784 CaseRangeBlock = createBasicBlock(name: "sw.caserange");
1785
1786 CurFn->insert(Position: CurFn->end(), BB: CaseRangeBlock);
1787 Builder.SetInsertPoint(CaseRangeBlock);
1788
1789 // Emit range check.
1790 llvm::Value *Diff =
1791 Builder.CreateSub(LHS: SwitchInsn->getCondition(), RHS: Builder.getInt(AI: LHS));
1792 llvm::Value *Cond =
1793 Builder.CreateICmpULE(LHS: Diff, RHS: Builder.getInt(AI: Range), Name: "inbounds");
1794
1795 llvm::MDNode *Weights = nullptr;
1796 if (SwitchWeights) {
1797 uint64_t ThisCount = getProfileCount(S: &S);
1798 uint64_t DefaultCount = (*SwitchWeights)[0];
1799 Weights = createProfileWeights(TrueCount: ThisCount, FalseCount: DefaultCount);
1800
1801 // Since we're chaining the switch default through each large case range, we
1802 // need to update the weight for the default, ie, the first case, to include
1803 // this case.
1804 (*SwitchWeights)[0] += ThisCount;
1805 } else if (SwitchLikelihood)
1806 Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1807
1808 Builder.CreateCondBr(Cond, True: CaseDest, False: FalseDest, BranchWeights: Weights);
1809
1810 // Restore the appropriate insertion point.
1811 if (RestoreBB)
1812 Builder.SetInsertPoint(RestoreBB);
1813 else
1814 Builder.ClearInsertionPoint();
1815}
1816
1817void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1818 ArrayRef<const Attr *> Attrs) {
1819 // If there is no enclosing switch instance that we're aware of, then this
1820 // case statement and its block can be elided. This situation only happens
1821 // when we've constant-folded the switch, are emitting the constant case,
1822 // and part of the constant case includes another case statement. For
1823 // instance: switch (4) { case 4: do { case 5: } while (1); }
1824 if (!SwitchInsn) {
1825 EmitStmt(S: S.getSubStmt());
1826 return;
1827 }
1828
1829 // Handle case ranges.
1830 if (S.getRHS()) {
1831 EmitCaseStmtRange(S, Attrs);
1832 return;
1833 }
1834
1835 llvm::ConstantInt *CaseVal =
1836 Builder.getInt(AI: S.getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1837
1838 // Emit debuginfo for the case value if it is an enum value.
1839 const ConstantExpr *CE;
1840 if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: S.getLHS()))
1841 CE = dyn_cast<ConstantExpr>(Val: ICE->getSubExpr());
1842 else
1843 CE = dyn_cast<ConstantExpr>(Val: S.getLHS());
1844 if (CE) {
1845 if (auto DE = dyn_cast<DeclRefExpr>(Val: CE->getSubExpr()))
1846 if (CGDebugInfo *Dbg = getDebugInfo())
1847 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
1848 Dbg->EmitGlobalVariable(VD: DE->getDecl(),
1849 Init: APValue(llvm::APSInt(CaseVal->getValue())));
1850 }
1851
1852 if (SwitchLikelihood)
1853 SwitchLikelihood->push_back(Elt: Stmt::getLikelihood(Attrs));
1854
1855 // If the body of the case is just a 'break', try to not emit an empty block.
1856 // If we're profiling or we're not optimizing, leave the block in for better
1857 // debug and coverage analysis.
1858 if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1859 CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1860 isa<BreakStmt>(Val: S.getSubStmt())) {
1861 JumpDest Block = BreakContinueStack.back().BreakBlock;
1862
1863 // Only do this optimization if there are no cleanups that need emitting.
1864 if (isObviouslyBranchWithoutCleanups(Dest: Block)) {
1865 if (SwitchWeights)
1866 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1867 SwitchInsn->addCase(OnVal: CaseVal, Dest: Block.getBlock());
1868
1869 // If there was a fallthrough into this case, make sure to redirect it to
1870 // the end of the switch as well.
1871 if (Builder.GetInsertBlock()) {
1872 Builder.CreateBr(Dest: Block.getBlock());
1873 Builder.ClearInsertionPoint();
1874 }
1875 return;
1876 }
1877 }
1878
1879 llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1880 EmitBlockWithFallThrough(BB: CaseDest, S: &S);
1881 if (SwitchWeights)
1882 SwitchWeights->push_back(Elt: getProfileCount(S: &S));
1883 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1884
1885 // Recursively emitting the statement is acceptable, but is not wonderful for
1886 // code where we have many case statements nested together, i.e.:
1887 // case 1:
1888 // case 2:
1889 // case 3: etc.
1890 // Handling this recursively will create a new block for each case statement
1891 // that falls through to the next case which is IR intensive. It also causes
1892 // deep recursion which can run into stack depth limitations. Handle
1893 // sequential non-range case statements specially.
1894 //
1895 // TODO When the next case has a likelihood attribute the code returns to the
1896 // recursive algorithm. Maybe improve this case if it becomes common practice
1897 // to use a lot of attributes.
1898 const CaseStmt *CurCase = &S;
1899 const CaseStmt *NextCase = dyn_cast<CaseStmt>(Val: S.getSubStmt());
1900
1901 // Otherwise, iteratively add consecutive cases to this switch stmt.
1902 while (NextCase && NextCase->getRHS() == nullptr) {
1903 CurCase = NextCase;
1904 llvm::ConstantInt *CaseVal =
1905 Builder.getInt(AI: CurCase->getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1906
1907 if (SwitchWeights)
1908 SwitchWeights->push_back(Elt: getProfileCount(S: NextCase));
1909 if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1910 CaseDest = createBasicBlock(name: "sw.bb");
1911 EmitBlockWithFallThrough(BB: CaseDest, S: CurCase);
1912 }
1913 // Since this loop is only executed when the CaseStmt has no attributes
1914 // use a hard-coded value.
1915 if (SwitchLikelihood)
1916 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
1917
1918 SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1919 NextCase = dyn_cast<CaseStmt>(Val: CurCase->getSubStmt());
1920 }
1921
1922 // Generate a stop point for debug info if the case statement is
1923 // followed by a default statement. A fallthrough case before a
1924 // default case gets its own branch target.
1925 if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1926 EmitStopPoint(S: CurCase);
1927
1928 // Normal default recursion for non-cases.
1929 EmitStmt(S: CurCase->getSubStmt());
1930}
1931
1932void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1933 ArrayRef<const Attr *> Attrs) {
1934 // If there is no enclosing switch instance that we're aware of, then this
1935 // default statement can be elided. This situation only happens when we've
1936 // constant-folded the switch.
1937 if (!SwitchInsn) {
1938 EmitStmt(S: S.getSubStmt());
1939 return;
1940 }
1941
1942 llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1943 assert(DefaultBlock->empty() &&
1944 "EmitDefaultStmt: Default block already defined?");
1945
1946 if (SwitchLikelihood)
1947 SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1948
1949 EmitBlockWithFallThrough(BB: DefaultBlock, S: &S);
1950
1951 EmitStmt(S: S.getSubStmt());
1952}
1953
1954namespace {
1955struct EmitDeferredStatement final : EHScopeStack::Cleanup {
1956 const DeferStmt &Stmt;
1957 EmitDeferredStatement(const DeferStmt *Stmt) : Stmt(*Stmt) {}
1958
1959 void Emit(CodeGenFunction &CGF, Flags) override {
1960 // Take care that any cleanups pushed by the body of a '_Defer' statement
1961 // don't clobber the current cleanup slot value.
1962 //
1963 // Assume we have a scope that pushes a cleanup; when that scope is exited,
1964 // we need to run that cleanup; this is accomplished by emitting the cleanup
1965 // into a separate block and then branching to that block at scope exit.
1966 //
1967 // Where this gets complicated is if we exit the scope in multiple different
1968 // ways; e.g. in a 'for' loop, we may exit the scope of its body by falling
1969 // off the end (in which case we need to run the cleanup and then branch to
1970 // the increment), or by 'break'ing out of the loop (in which case we need
1971 // to run the cleanup and then branch to the loop exit block); in both cases
1972 // we first branch to the cleanup block to run the cleanup, but the block we
1973 // need to jump to *after* running the cleanup is different.
1974 //
1975 // This is accomplished using a local integer variable called the 'cleanup
1976 // slot': before branching to the cleanup block, we store a value into that
1977 // slot. Then, in the cleanup block, after running the cleanup, we load the
1978 // value of that variable and 'switch' on it to branch to the appropriate
1979 // continuation block.
1980 //
1981 // The problem that arises once '_Defer' statements are involved is that the
1982 // body of a '_Defer' is an arbitrary statement which itself can create more
1983 // cleanups. This means we may end up overwriting the cleanup slot before we
1984 // ever have a chance to 'switch' on it, which means that once we *do* get
1985 // to the 'switch', we end up in whatever block the cleanup code happened to
1986 // pick as the default 'switch' exit label!
1987 //
1988 // That is, what is normally supposed to happen is something like:
1989 //
1990 // 1. Store 'X' to cleanup slot.
1991 // 2. Branch to cleanup block.
1992 // 3. Execute cleanup.
1993 // 4. Read value from cleanup slot.
1994 // 5. Branch to the block associated with 'X'.
1995 //
1996 // But if we encounter a _Defer' statement that contains a cleanup, then
1997 // what might instead happen is:
1998 //
1999 // 1. Store 'X' to cleanup slot.
2000 // 2. Branch to cleanup block.
2001 // 3. Execute cleanup; this ends up pushing another cleanup, so:
2002 // 3a. Store 'Y' to cleanup slot.
2003 // 3b. Run steps 2–5 recursively.
2004 // 4. Read value from cleanup slot, which is now 'Y' instead of 'X'.
2005 // 5. Branch to the block associated with 'Y'... which doesn't even
2006 // exist because the value 'Y' is only meaningful for the inner
2007 // cleanup. The result is we just branch 'somewhere random'.
2008 //
2009 // The rest of the cleanup code simply isn't prepared to handle this case
2010 // because most other cleanups can't push more cleanups, and thus, emitting
2011 // other cleanups generally cannot clobber the cleanup slot.
2012 //
2013 // To prevent this from happening, save the current cleanup slot value and
2014 // restore it after emitting the '_Defer' statement.
2015 llvm::Value *SavedCleanupDest = nullptr;
2016 if (CGF.NormalCleanupDest.isValid())
2017 SavedCleanupDest =
2018 CGF.Builder.CreateLoad(Addr: CGF.NormalCleanupDest, Name: "cleanup.dest.saved");
2019
2020 CGF.EmitStmt(S: Stmt.getBody());
2021
2022 if (SavedCleanupDest && CGF.HaveInsertPoint())
2023 CGF.Builder.CreateStore(Val: SavedCleanupDest, Addr: CGF.NormalCleanupDest);
2024
2025 // Cleanups must end with an insert point.
2026 CGF.EnsureInsertPoint();
2027 }
2028};
2029} // namespace
2030
2031void CodeGenFunction::EmitDeferStmt(const DeferStmt &S) {
2032 EHStack.pushCleanup<EmitDeferredStatement>(Kind: NormalAndEHCleanup, A: &S);
2033}
2034
2035/// CollectStatementsForCase - Given the body of a 'switch' statement and a
2036/// constant value that is being switched on, see if we can dead code eliminate
2037/// the body of the switch to a simple series of statements to emit. Basically,
2038/// on a switch (5) we want to find these statements:
2039/// case 5:
2040/// printf(...); <--
2041/// ++i; <--
2042/// break;
2043///
2044/// and add them to the ResultStmts vector. If it is unsafe to do this
2045/// transformation (for example, one of the elided statements contains a label
2046/// that might be jumped to), return CSFC_Failure. If we handled it and 'S'
2047/// should include statements after it (e.g. the printf() line is a substmt of
2048/// the case) then return CSFC_FallThrough. If we handled it and found a break
2049/// statement, then return CSFC_Success.
2050///
2051/// If Case is non-null, then we are looking for the specified case, checking
2052/// that nothing we jump over contains labels. If Case is null, then we found
2053/// the case and are looking for the break.
2054///
2055/// If the recursive walk actually finds our Case, then we set FoundCase to
2056/// true.
2057///
2058enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
2059static CSFC_Result CollectStatementsForCase(const Stmt *S,
2060 const SwitchCase *Case,
2061 bool &FoundCase,
2062 SmallVectorImpl<const Stmt*> &ResultStmts) {
2063 // If this is a null statement, just succeed.
2064 if (!S)
2065 return Case ? CSFC_Success : CSFC_FallThrough;
2066
2067 // If this is the switchcase (case 4: or default) that we're looking for, then
2068 // we're in business. Just add the substatement.
2069 if (const SwitchCase *SC = dyn_cast<SwitchCase>(Val: S)) {
2070 if (S == Case) {
2071 FoundCase = true;
2072 return CollectStatementsForCase(S: SC->getSubStmt(), Case: nullptr, FoundCase,
2073 ResultStmts);
2074 }
2075
2076 // Otherwise, this is some other case or default statement, just ignore it.
2077 return CollectStatementsForCase(S: SC->getSubStmt(), Case, FoundCase,
2078 ResultStmts);
2079 }
2080
2081 // If we are in the live part of the code and we found our break statement,
2082 // return a success!
2083 if (!Case && isa<BreakStmt>(Val: S))
2084 return CSFC_Success;
2085
2086 // If this is a switch statement, then it might contain the SwitchCase, the
2087 // break, or neither.
2088 if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: S)) {
2089 // Handle this as two cases: we might be looking for the SwitchCase (if so
2090 // the skipped statements must be skippable) or we might already have it.
2091 CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
2092 bool StartedInLiveCode = FoundCase;
2093 unsigned StartSize = ResultStmts.size();
2094
2095 // If we've not found the case yet, scan through looking for it.
2096 if (Case) {
2097 // Keep track of whether we see a skipped declaration. The code could be
2098 // using the declaration even if it is skipped, so we can't optimize out
2099 // the decl if the kept statements might refer to it.
2100 bool HadSkippedDecl = false;
2101
2102 // If we're looking for the case, just see if we can skip each of the
2103 // substatements.
2104 for (; Case && I != E; ++I) {
2105 HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(S: *I);
2106
2107 switch (CollectStatementsForCase(S: *I, Case, FoundCase, ResultStmts)) {
2108 case CSFC_Failure: return CSFC_Failure;
2109 case CSFC_Success:
2110 // A successful result means that either 1) that the statement doesn't
2111 // have the case and is skippable, or 2) does contain the case value
2112 // and also contains the break to exit the switch. In the later case,
2113 // we just verify the rest of the statements are elidable.
2114 if (FoundCase) {
2115 // If we found the case and skipped declarations, we can't do the
2116 // optimization.
2117 if (HadSkippedDecl)
2118 return CSFC_Failure;
2119
2120 for (++I; I != E; ++I)
2121 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2122 return CSFC_Failure;
2123 return CSFC_Success;
2124 }
2125 break;
2126 case CSFC_FallThrough:
2127 // If we have a fallthrough condition, then we must have found the
2128 // case started to include statements. Consider the rest of the
2129 // statements in the compound statement as candidates for inclusion.
2130 assert(FoundCase && "Didn't find case but returned fallthrough?");
2131 // We recursively found Case, so we're not looking for it anymore.
2132 Case = nullptr;
2133
2134 // If we found the case and skipped declarations, we can't do the
2135 // optimization.
2136 if (HadSkippedDecl)
2137 return CSFC_Failure;
2138 break;
2139 }
2140 }
2141
2142 if (!FoundCase)
2143 return CSFC_Success;
2144
2145 assert(!HadSkippedDecl && "fallthrough after skipping decl");
2146 }
2147
2148 // If we have statements in our range, then we know that the statements are
2149 // live and need to be added to the set of statements we're tracking.
2150 bool AnyDecls = false;
2151 for (; I != E; ++I) {
2152 AnyDecls |= CodeGenFunction::mightAddDeclToScope(S: *I);
2153
2154 switch (CollectStatementsForCase(S: *I, Case: nullptr, FoundCase, ResultStmts)) {
2155 case CSFC_Failure: return CSFC_Failure;
2156 case CSFC_FallThrough:
2157 // A fallthrough result means that the statement was simple and just
2158 // included in ResultStmt, keep adding them afterwards.
2159 break;
2160 case CSFC_Success:
2161 // A successful result means that we found the break statement and
2162 // stopped statement inclusion. We just ensure that any leftover stmts
2163 // are skippable and return success ourselves.
2164 for (++I; I != E; ++I)
2165 if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2166 return CSFC_Failure;
2167 return CSFC_Success;
2168 }
2169 }
2170
2171 // If we're about to fall out of a scope without hitting a 'break;', we
2172 // can't perform the optimization if there were any decls in that scope
2173 // (we'd lose their end-of-lifetime).
2174 if (AnyDecls) {
2175 // If the entire compound statement was live, there's one more thing we
2176 // can try before giving up: emit the whole thing as a single statement.
2177 // We can do that unless the statement contains a 'break;'.
2178 // FIXME: Such a break must be at the end of a construct within this one.
2179 // We could emit this by just ignoring the BreakStmts entirely.
2180 if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
2181 ResultStmts.resize(N: StartSize);
2182 ResultStmts.push_back(Elt: S);
2183 } else {
2184 return CSFC_Failure;
2185 }
2186 }
2187
2188 return CSFC_FallThrough;
2189 }
2190
2191 // Okay, this is some other statement that we don't handle explicitly, like a
2192 // for statement or increment etc. If we are skipping over this statement,
2193 // just verify it doesn't have labels, which would make it invalid to elide.
2194 if (Case) {
2195 if (CodeGenFunction::ContainsLabel(S, IgnoreCaseStmts: true))
2196 return CSFC_Failure;
2197 return CSFC_Success;
2198 }
2199
2200 // Otherwise, we want to include this statement. Everything is cool with that
2201 // so long as it doesn't contain a break out of the switch we're in.
2202 if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
2203
2204 // Otherwise, everything is great. Include the statement and tell the caller
2205 // that we fall through and include the next statement as well.
2206 ResultStmts.push_back(Elt: S);
2207 return CSFC_FallThrough;
2208}
2209
2210/// FindCaseStatementsForValue - Find the case statement being jumped to and
2211/// then invoke CollectStatementsForCase to find the list of statements to emit
2212/// for a switch on constant. See the comment above CollectStatementsForCase
2213/// for more details.
2214static bool FindCaseStatementsForValue(const SwitchStmt &S,
2215 const llvm::APSInt &ConstantCondValue,
2216 SmallVectorImpl<const Stmt*> &ResultStmts,
2217 ASTContext &C,
2218 const SwitchCase *&ResultCase) {
2219 // First step, find the switch case that is being branched to. We can do this
2220 // efficiently by scanning the SwitchCase list.
2221 const SwitchCase *Case = S.getSwitchCaseList();
2222 const DefaultStmt *DefaultCase = nullptr;
2223
2224 for (; Case; Case = Case->getNextSwitchCase()) {
2225 // It's either a default or case. Just remember the default statement in
2226 // case we're not jumping to any numbered cases.
2227 if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Val: Case)) {
2228 DefaultCase = DS;
2229 continue;
2230 }
2231
2232 // Check to see if this case is the one we're looking for.
2233 const CaseStmt *CS = cast<CaseStmt>(Val: Case);
2234 // Don't handle case ranges yet.
2235 if (CS->getRHS()) return false;
2236
2237 // If we found our case, remember it as 'case'.
2238 if (CS->getLHS()->EvaluateKnownConstInt(Ctx: C) == ConstantCondValue)
2239 break;
2240 }
2241
2242 // If we didn't find a matching case, we use a default if it exists, or we
2243 // elide the whole switch body!
2244 if (!Case) {
2245 // It is safe to elide the body of the switch if it doesn't contain labels
2246 // etc. If it is safe, return successfully with an empty ResultStmts list.
2247 if (!DefaultCase)
2248 return !CodeGenFunction::ContainsLabel(S: &S);
2249 Case = DefaultCase;
2250 }
2251
2252 // Ok, we know which case is being jumped to, try to collect all the
2253 // statements that follow it. This can fail for a variety of reasons. Also,
2254 // check to see that the recursive walk actually found our case statement.
2255 // Insane cases like this can fail to find it in the recursive walk since we
2256 // don't handle every stmt kind:
2257 // switch (4) {
2258 // while (1) {
2259 // case 4: ...
2260 bool FoundCase = false;
2261 ResultCase = Case;
2262 return CollectStatementsForCase(S: S.getBody(), Case, FoundCase,
2263 ResultStmts) != CSFC_Failure &&
2264 FoundCase;
2265}
2266
2267static std::optional<SmallVector<uint64_t, 16>>
2268getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
2269 // Are there enough branches to weight them?
2270 if (Likelihoods.size() <= 1)
2271 return std::nullopt;
2272
2273 uint64_t NumUnlikely = 0;
2274 uint64_t NumNone = 0;
2275 uint64_t NumLikely = 0;
2276 for (const auto LH : Likelihoods) {
2277 switch (LH) {
2278 case Stmt::LH_Unlikely:
2279 ++NumUnlikely;
2280 break;
2281 case Stmt::LH_None:
2282 ++NumNone;
2283 break;
2284 case Stmt::LH_Likely:
2285 ++NumLikely;
2286 break;
2287 }
2288 }
2289
2290 // Is there a likelihood attribute used?
2291 if (NumUnlikely == 0 && NumLikely == 0)
2292 return std::nullopt;
2293
2294 // When multiple cases share the same code they can be combined during
2295 // optimization. In that case the weights of the branch will be the sum of
2296 // the individual weights. Make sure the combined sum of all neutral cases
2297 // doesn't exceed the value of a single likely attribute.
2298 // The additions both avoid divisions by 0 and make sure the weights of None
2299 // don't exceed the weight of Likely.
2300 const uint64_t Likely = INT32_MAX / (NumLikely + 2);
2301 const uint64_t None = Likely / (NumNone + 1);
2302 const uint64_t Unlikely = 0;
2303
2304 SmallVector<uint64_t, 16> Result;
2305 Result.reserve(N: Likelihoods.size());
2306 for (const auto LH : Likelihoods) {
2307 switch (LH) {
2308 case Stmt::LH_Unlikely:
2309 Result.push_back(Elt: Unlikely);
2310 break;
2311 case Stmt::LH_None:
2312 Result.push_back(Elt: None);
2313 break;
2314 case Stmt::LH_Likely:
2315 Result.push_back(Elt: Likely);
2316 break;
2317 }
2318 }
2319
2320 return Result;
2321}
2322
2323void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
2324 // Handle nested switch statements.
2325 llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
2326 SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
2327 SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
2328 llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
2329
2330 // See if we can constant fold the condition of the switch and therefore only
2331 // emit the live case statement (if any) of the switch.
2332 llvm::APSInt ConstantCondValue;
2333 if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: ConstantCondValue)) {
2334 SmallVector<const Stmt*, 4> CaseStmts;
2335 const SwitchCase *Case = nullptr;
2336 if (FindCaseStatementsForValue(S, ConstantCondValue, ResultStmts&: CaseStmts,
2337 C&: getContext(), ResultCase&: Case)) {
2338 if (Case)
2339 incrementProfileCounter(S: Case);
2340 RunCleanupsScope ExecutedScope(*this);
2341
2342 if (S.getInit())
2343 EmitStmt(S: S.getInit());
2344
2345 // Emit the condition variable if needed inside the entire cleanup scope
2346 // used by this special case for constant folded switches.
2347 if (S.getConditionVariable())
2348 EmitDecl(D: *S.getConditionVariable(), /*EvaluateConditionDecl=*/true);
2349
2350 // At this point, we are no longer "within" a switch instance, so
2351 // we can temporarily enforce this to ensure that any embedded case
2352 // statements are not emitted.
2353 SwitchInsn = nullptr;
2354
2355 // Okay, we can dead code eliminate everything except this case. Emit the
2356 // specified series of statements and we're good.
2357 for (const Stmt *CaseStmt : CaseStmts)
2358 EmitStmt(S: CaseStmt);
2359 incrementProfileCounter(S: &S);
2360 PGO->markStmtMaybeUsed(S: S.getBody());
2361
2362 // Now we want to restore the saved switch instance so that nested
2363 // switches continue to function properly
2364 SwitchInsn = SavedSwitchInsn;
2365
2366 return;
2367 }
2368 }
2369
2370 JumpDest SwitchExit = getJumpDestInCurrentScope(Name: "sw.epilog");
2371
2372 RunCleanupsScope ConditionScope(*this);
2373
2374 if (S.getInit())
2375 EmitStmt(S: S.getInit());
2376
2377 if (S.getConditionVariable())
2378 EmitDecl(D: *S.getConditionVariable());
2379 llvm::Value *CondV = EmitScalarExpr(E: S.getCond());
2380 MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
2381
2382 // Create basic block to hold stuff that comes after switch
2383 // statement. We also need to create a default block now so that
2384 // explicit case ranges tests can have a place to jump to on
2385 // failure.
2386 llvm::BasicBlock *DefaultBlock = createBasicBlock(name: "sw.default");
2387 SwitchInsn = Builder.CreateSwitch(V: CondV, Dest: DefaultBlock);
2388 addInstToNewSourceAtom(KeyInstruction: SwitchInsn, Backup: CondV);
2389
2390 if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {
2391 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2392 llvm::ConstantInt *BranchHintConstant =
2393 HLSLControlFlowAttr ==
2394 HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2395 ? llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 1)
2396 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: 2);
2397 llvm::Metadata *Vals[] = {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2398 MDHelper.createConstant(C: BranchHintConstant)};
2399 SwitchInsn->setMetadata(Kind: "hlsl.controlflow.hint",
2400 Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2401 }
2402
2403 if (PGO->haveRegionCounts()) {
2404 // Walk the SwitchCase list to find how many there are.
2405 uint64_t DefaultCount = 0;
2406 unsigned NumCases = 0;
2407 for (const SwitchCase *Case = S.getSwitchCaseList();
2408 Case;
2409 Case = Case->getNextSwitchCase()) {
2410 if (isa<DefaultStmt>(Val: Case))
2411 DefaultCount = getProfileCount(S: Case);
2412 NumCases += 1;
2413 }
2414 SwitchWeights = new SmallVector<uint64_t, 16>();
2415 SwitchWeights->reserve(N: NumCases);
2416 // The default needs to be first. We store the edge count, so we already
2417 // know the right weight.
2418 SwitchWeights->push_back(Elt: DefaultCount);
2419 } else if (CGM.getCodeGenOpts().OptimizationLevel) {
2420 SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
2421 // Initialize the default case.
2422 SwitchLikelihood->push_back(Elt: Stmt::LH_None);
2423 }
2424
2425 CaseRangeBlock = DefaultBlock;
2426
2427 // Clear the insertion point to indicate we are in unreachable code.
2428 Builder.ClearInsertionPoint();
2429
2430 // All break statements jump to NextBlock. If BreakContinueStack is non-empty
2431 // then reuse last ContinueBlock.
2432 JumpDest OuterContinue;
2433 if (!BreakContinueStack.empty())
2434 OuterContinue = BreakContinueStack.back().ContinueBlock;
2435
2436 BreakContinueStack.push_back(Elt: BreakContinue(S, SwitchExit, OuterContinue));
2437
2438 // Emit switch body.
2439 EmitStmt(S: S.getBody());
2440
2441 BreakContinueStack.pop_back();
2442
2443 // Update the default block in case explicit case range tests have
2444 // been chained on top.
2445 SwitchInsn->setDefaultDest(CaseRangeBlock);
2446
2447 // If a default was never emitted:
2448 if (!DefaultBlock->getParent()) {
2449 // If we have cleanups, emit the default block so that there's a
2450 // place to jump through the cleanups from.
2451 if (ConditionScope.requiresCleanups()) {
2452 EmitBlock(BB: DefaultBlock);
2453
2454 // Otherwise, just forward the default block to the switch end.
2455 } else {
2456 DefaultBlock->replaceAllUsesWith(V: SwitchExit.getBlock());
2457 delete DefaultBlock;
2458 }
2459 }
2460
2461 ConditionScope.ForceCleanup();
2462
2463 // Close the last case (or DefaultBlock).
2464 EmitBranch(Target: SwitchExit.getBlock());
2465
2466 // Insert a False Counter if SwitchStmt doesn't have DefaultStmt.
2467 if (hasSkipCounter(S: S.getCond())) {
2468 auto *ImplicitDefaultBlock = createBasicBlock(name: "sw.false");
2469 EmitBlock(BB: ImplicitDefaultBlock);
2470 incrementProfileCounter(ExecSkip: UseSkipPath, S: S.getCond());
2471 Builder.CreateBr(Dest: SwitchInsn->getDefaultDest());
2472 SwitchInsn->setDefaultDest(ImplicitDefaultBlock);
2473 }
2474
2475 // Emit continuation.
2476 EmitBlock(BB: SwitchExit.getBlock(), IsFinished: true);
2477 incrementProfileCounter(S: &S);
2478
2479 // If the switch has a condition wrapped by __builtin_unpredictable,
2480 // create metadata that specifies that the switch is unpredictable.
2481 // Don't bother if not optimizing because that metadata would not be used.
2482 auto *Call = dyn_cast<CallExpr>(Val: S.getCond());
2483 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2484 auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2485 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2486 llvm::MDBuilder MDHelper(getLLVMContext());
2487 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_unpredictable,
2488 Node: MDHelper.createUnpredictable());
2489 }
2490 }
2491
2492 if (SwitchWeights) {
2493 assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2494 "switch weights do not match switch cases");
2495 // If there's only one jump destination there's no sense weighting it.
2496 if (SwitchWeights->size() > 1)
2497 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2498 Node: createProfileWeights(Weights: *SwitchWeights));
2499 delete SwitchWeights;
2500 } else if (SwitchLikelihood) {
2501 assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2502 "switch likelihoods do not match switch cases");
2503 std::optional<SmallVector<uint64_t, 16>> LHW =
2504 getLikelihoodWeights(Likelihoods: *SwitchLikelihood);
2505 if (LHW) {
2506 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2507 SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2508 Node: createProfileWeights(Weights: *LHW));
2509 }
2510 delete SwitchLikelihood;
2511 }
2512 SwitchInsn = SavedSwitchInsn;
2513 SwitchWeights = SavedSwitchWeights;
2514 SwitchLikelihood = SavedSwitchLikelihood;
2515 CaseRangeBlock = SavedCRBlock;
2516}
2517
2518std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2519 const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2520 QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2521 if (Info.allowsRegister() || !Info.allowsMemory()) {
2522 if (CodeGenFunction::hasScalarEvaluationKind(T: InputType))
2523 return {EmitLoadOfLValue(V: InputValue, Loc).getScalarVal(), nullptr};
2524
2525 llvm::Type *Ty = ConvertType(T: InputType);
2526 uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2527 if ((Size <= 64 && llvm::isPowerOf2_64(Value: Size)) ||
2528 getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty)) {
2529 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2530
2531 return {Builder.CreateLoad(Addr: InputValue.getAddress().withElementType(ElemTy: Ty)),
2532 nullptr};
2533 }
2534 }
2535
2536 Address Addr = InputValue.getAddress();
2537 ConstraintStr += '*';
2538 return {InputValue.getPointer(CGF&: *this), Addr.getElementType()};
2539}
2540std::pair<llvm::Value *, llvm::Type *>
2541CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2542 const Expr *InputExpr,
2543 std::string &ConstraintStr) {
2544 // If this can't be a register or memory, i.e., has to be a constant
2545 // (immediate or symbolic), try to emit it as such.
2546 if (!Info.allowsRegister() && !Info.allowsMemory()) {
2547 if (Info.requiresImmediateConstant()) {
2548 Expr::EvalResult EVResult;
2549 InputExpr->EvaluateAsRValue(Result&: EVResult, Ctx: getContext(), InConstantContext: true);
2550
2551 llvm::APSInt IntResult;
2552 if (EVResult.Val.toIntegralConstant(Result&: IntResult, SrcTy: InputExpr->getType(),
2553 Ctx: getContext()))
2554 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: IntResult), nullptr};
2555 }
2556
2557 Expr::EvalResult Result;
2558 if (InputExpr->EvaluateAsInt(Result, Ctx: getContext()))
2559 return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: Result.Val.getInt()),
2560 nullptr};
2561 }
2562
2563 if (Info.allowsRegister() || !Info.allowsMemory())
2564 if (CodeGenFunction::hasScalarEvaluationKind(T: InputExpr->getType()))
2565 return {EmitScalarExpr(E: InputExpr), nullptr};
2566 if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2567 return {EmitScalarExpr(E: InputExpr), nullptr};
2568 InputExpr = InputExpr->IgnoreParenNoopCasts(Ctx: getContext());
2569 LValue Dest = EmitLValue(E: InputExpr);
2570 return EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr,
2571 Loc: InputExpr->getExprLoc());
2572}
2573
2574/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2575/// asm call instruction. The !srcloc MDNode contains a list of constant
2576/// integers which are the source locations of the start of each line in the
2577/// asm.
2578static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2579 CodeGenFunction &CGF) {
2580 SmallVector<llvm::Metadata *, 8> Locs;
2581 // Add the location of the first line to the MDNode.
2582 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
2583 Ty: CGF.Int64Ty, V: Str->getBeginLoc().getRawEncoding())));
2584 StringRef StrVal = Str->getString();
2585 if (!StrVal.empty()) {
2586 const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2587 const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2588 unsigned StartToken = 0;
2589 unsigned ByteOffset = 0;
2590
2591 // Add the location of the start of each subsequent line of the asm to the
2592 // MDNode.
2593 for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2594 if (StrVal[i] != '\n') continue;
2595 SourceLocation LineLoc = Str->getLocationOfByte(
2596 ByteNo: i + 1, SM, Features: LangOpts, Target: CGF.getTarget(), StartToken: &StartToken, StartTokenByteOffset: &ByteOffset);
2597 Locs.push_back(Elt: llvm::ConstantAsMetadata::get(
2598 C: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: LineLoc.getRawEncoding())));
2599 }
2600 }
2601
2602 return llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: Locs);
2603}
2604
2605static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2606 bool HasUnwindClobber, bool ReadOnly,
2607 bool ReadNone, bool NoMerge, bool NoConvergent,
2608 const AsmStmt &S,
2609 const std::vector<llvm::Type *> &ResultRegTypes,
2610 const std::vector<llvm::Type *> &ArgElemTypes,
2611 CodeGenFunction &CGF,
2612 std::vector<llvm::Value *> &RegResults) {
2613 if (!HasUnwindClobber)
2614 Result.addFnAttr(Kind: llvm::Attribute::NoUnwind);
2615
2616 if (NoMerge)
2617 Result.addFnAttr(Kind: llvm::Attribute::NoMerge);
2618 // Attach readnone and readonly attributes.
2619 if (!HasSideEffect) {
2620 if (ReadNone)
2621 Result.setDoesNotAccessMemory();
2622 else if (ReadOnly)
2623 Result.setOnlyReadsMemory();
2624 }
2625
2626 // Add elementtype attribute for indirect constraints.
2627 for (auto Pair : llvm::enumerate(First: ArgElemTypes)) {
2628 if (Pair.value()) {
2629 auto Attr = llvm::Attribute::get(
2630 Context&: CGF.getLLVMContext(), Kind: llvm::Attribute::ElementType, Ty: Pair.value());
2631 Result.addParamAttr(ArgNo: Pair.index(), Attr);
2632 }
2633 }
2634
2635 // Slap the source location of the inline asm into a !srcloc metadata on the
2636 // call.
2637 const StringLiteral *SL;
2638 if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(Val: &S);
2639 gccAsmStmt &&
2640 (SL = dyn_cast<StringLiteral>(Val: gccAsmStmt->getAsmStringExpr()))) {
2641 Result.setMetadata(Kind: "srcloc", Node: getAsmSrcLocInfo(Str: SL, CGF));
2642 } else {
2643 // At least put the line number on MS inline asm blobs and GCC asm constexpr
2644 // strings.
2645 llvm::Constant *Loc =
2646 llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: S.getAsmLoc().getRawEncoding());
2647 Result.setMetadata(Kind: "srcloc",
2648 Node: llvm::MDNode::get(Context&: CGF.getLLVMContext(),
2649 MDs: llvm::ConstantAsMetadata::get(C: Loc)));
2650 }
2651
2652 // Make inline-asm calls Key for the debug info feature Key Instructions.
2653 CGF.addInstToNewSourceAtom(KeyInstruction: &Result, Backup: nullptr);
2654
2655 if (!NoConvergent && CGF.getLangOpts().assumeFunctionsAreConvergent())
2656 // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
2657 // convergent (meaning, they may call an intrinsically convergent op, such
2658 // as bar.sync, and so can't have certain optimizations applied around
2659 // them) unless it's explicitly marked 'noconvergent'.
2660 Result.addFnAttr(Kind: llvm::Attribute::Convergent);
2661 // Extract all of the register value results from the asm.
2662 if (ResultRegTypes.size() == 1) {
2663 RegResults.push_back(x: &Result);
2664 } else {
2665 for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
2666 llvm::Value *Tmp = CGF.Builder.CreateExtractValue(Agg: &Result, Idxs: i, Name: "asmresult");
2667 RegResults.push_back(x: Tmp);
2668 }
2669 }
2670}
2671
2672static void
2673EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S,
2674 const llvm::ArrayRef<llvm::Value *> RegResults,
2675 const llvm::ArrayRef<llvm::Type *> ResultRegTypes,
2676 const llvm::ArrayRef<llvm::Type *> ResultTruncRegTypes,
2677 const llvm::ArrayRef<LValue> ResultRegDests,
2678 const llvm::ArrayRef<QualType> ResultRegQualTys,
2679 const llvm::BitVector &ResultTypeRequiresCast,
2680 const std::vector<std::optional<std::pair<unsigned, unsigned>>>
2681 &ResultBounds) {
2682 CGBuilderTy &Builder = CGF.Builder;
2683 CodeGenModule &CGM = CGF.CGM;
2684 llvm::LLVMContext &CTX = CGF.getLLVMContext();
2685
2686 assert(RegResults.size() == ResultRegTypes.size());
2687 assert(RegResults.size() == ResultTruncRegTypes.size());
2688 assert(RegResults.size() == ResultRegDests.size());
2689 // ResultRegDests can be also populated by addReturnRegisterOutputs() above,
2690 // in which case its size may grow.
2691 assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
2692 assert(ResultBounds.size() <= ResultRegDests.size());
2693
2694 for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
2695 llvm::Value *Tmp = RegResults[i];
2696 llvm::Type *TruncTy = ResultTruncRegTypes[i];
2697
2698 if ((i < ResultBounds.size()) && ResultBounds[i].has_value()) {
2699 const auto [LowerBound, UpperBound] = ResultBounds[i].value();
2700 // FIXME: Support for nonzero lower bounds not yet implemented.
2701 assert(LowerBound == 0 && "Output operand lower bound is not zero.");
2702 llvm::Constant *UpperBoundConst =
2703 llvm::ConstantInt::get(Ty: Tmp->getType(), V: UpperBound);
2704 llvm::Value *IsBooleanValue =
2705 Builder.CreateCmp(Pred: llvm::CmpInst::ICMP_ULT, LHS: Tmp, RHS: UpperBoundConst);
2706 llvm::Function *FnAssume = CGM.getIntrinsic(IID: llvm::Intrinsic::assume);
2707 Builder.CreateCall(Callee: FnAssume, Args: IsBooleanValue);
2708 }
2709
2710 // If the result type of the LLVM IR asm doesn't match the result type of
2711 // the expression, do the conversion.
2712 if (ResultRegTypes[i] != TruncTy) {
2713
2714 // Truncate the integer result to the right size, note that TruncTy can be
2715 // a pointer.
2716 if (TruncTy->isFloatingPointTy())
2717 Tmp = Builder.CreateFPTrunc(V: Tmp, DestTy: TruncTy);
2718 else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
2719 uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(Ty: TruncTy);
2720 Tmp = Builder.CreateTrunc(
2721 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)ResSize));
2722 Tmp = Builder.CreateIntToPtr(V: Tmp, DestTy: TruncTy);
2723 } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
2724 uint64_t TmpSize =
2725 CGM.getDataLayout().getTypeSizeInBits(Ty: Tmp->getType());
2726 Tmp = Builder.CreatePtrToInt(
2727 V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)TmpSize));
2728 Tmp = Builder.CreateTrunc(V: Tmp, DestTy: TruncTy);
2729 } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
2730 Tmp = Builder.CreateZExtOrTrunc(V: Tmp, DestTy: TruncTy);
2731 } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
2732 Tmp = Builder.CreateBitCast(V: Tmp, DestTy: TruncTy);
2733 }
2734 }
2735
2736 ApplyAtomGroup Grp(CGF.getDebugInfo());
2737 LValue Dest = ResultRegDests[i];
2738 // ResultTypeRequiresCast elements correspond to the first
2739 // ResultTypeRequiresCast.size() elements of RegResults.
2740 if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) {
2741 unsigned Size = CGF.getContext().getTypeSize(T: ResultRegQualTys[i]);
2742 Address A = Dest.getAddress().withElementType(ElemTy: ResultRegTypes[i]);
2743 if (CGF.getTargetHooks().isScalarizableAsmOperand(CGF, Ty: TruncTy)) {
2744 llvm::StoreInst *S = Builder.CreateStore(Val: Tmp, Addr: A);
2745 CGF.addInstToCurrentSourceAtom(KeyInstruction: S, Backup: S->getValueOperand());
2746 continue;
2747 }
2748
2749 QualType Ty =
2750 CGF.getContext().getIntTypeForBitwidth(DestWidth: Size, /*Signed=*/false);
2751 if (Ty.isNull()) {
2752 const Expr *OutExpr = S.getOutputExpr(i);
2753 CGM.getDiags().Report(Loc: OutExpr->getExprLoc(),
2754 DiagID: diag::err_store_value_to_reg);
2755 return;
2756 }
2757 Dest = CGF.MakeAddrLValue(Addr: A, T: Ty);
2758 }
2759 CGF.EmitStoreThroughLValue(Src: RValue::get(V: Tmp), Dst: Dest);
2760 }
2761}
2762
2763static void EmitHipStdParUnsupportedAsm(CodeGenFunction *CGF,
2764 const AsmStmt &S) {
2765 constexpr auto Name = "__ASM__hipstdpar_unsupported";
2766
2767 std::string Asm;
2768 if (auto GCCAsm = dyn_cast<GCCAsmStmt>(Val: &S))
2769 Asm = GCCAsm->getAsmString();
2770
2771 auto &Ctx = CGF->CGM.getLLVMContext();
2772
2773 auto StrTy = llvm::ConstantDataArray::getString(Context&: Ctx, Initializer: Asm);
2774 auto FnTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx),
2775 Params: {StrTy->getType()}, isVarArg: false);
2776 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2777
2778 CGF->Builder.CreateCall(Callee: UBF, Args: {StrTy});
2779}
2780
2781void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2782 // Pop all cleanup blocks at the end of the asm statement.
2783 CodeGenFunction::RunCleanupsScope Cleanups(*this);
2784
2785 // Assemble the final asm string.
2786 std::string AsmString = S.generateAsmString(C: getContext());
2787
2788 // Get all the output and input constraints together.
2789 SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2790 SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2791
2792 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2793 bool IsValidTargetAsm = true;
2794 for (unsigned i = 0, e = S.getNumOutputs(); i != e && IsValidTargetAsm; i++) {
2795 StringRef Name;
2796 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2797 Name = GAS->getOutputName(i);
2798 TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
2799 bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
2800 if (IsHipStdPar && !IsValid)
2801 IsValidTargetAsm = false;
2802 else
2803 assert(IsValid && "Failed to parse output constraint");
2804 OutputConstraintInfos.push_back(Elt: Info);
2805 }
2806
2807 for (unsigned i = 0, e = S.getNumInputs(); i != e && IsValidTargetAsm; i++) {
2808 StringRef Name;
2809 if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2810 Name = GAS->getInputName(i);
2811 TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
2812 bool IsValid =
2813 getTarget().validateInputConstraint(OutputConstraints: OutputConstraintInfos, info&: Info);
2814 if (IsHipStdPar && !IsValid)
2815 IsValidTargetAsm = false;
2816 else
2817 assert(IsValid && "Failed to parse input constraint");
2818 InputConstraintInfos.push_back(Elt: Info);
2819 }
2820
2821 if (!IsValidTargetAsm)
2822 return EmitHipStdParUnsupportedAsm(CGF: this, S);
2823
2824 std::string Constraints;
2825
2826 std::vector<LValue> ResultRegDests;
2827 std::vector<QualType> ResultRegQualTys;
2828 std::vector<llvm::Type *> ResultRegTypes;
2829 std::vector<llvm::Type *> ResultTruncRegTypes;
2830 std::vector<llvm::Type *> ArgTypes;
2831 std::vector<llvm::Type *> ArgElemTypes;
2832 std::vector<llvm::Value*> Args;
2833 llvm::BitVector ResultTypeRequiresCast;
2834 std::vector<std::optional<std::pair<unsigned, unsigned>>> ResultBounds;
2835
2836 // Keep track of inout constraints.
2837 std::string InOutConstraints;
2838 std::vector<llvm::Value*> InOutArgs;
2839 std::vector<llvm::Type*> InOutArgTypes;
2840 std::vector<llvm::Type*> InOutArgElemTypes;
2841
2842 // Keep track of out constraints for tied input operand.
2843 std::vector<std::string> OutputConstraints;
2844
2845 // Keep track of defined physregs.
2846 llvm::SmallSet<std::string, 8> PhysRegOutputs;
2847
2848 // An inline asm can be marked readonly if it meets the following conditions:
2849 // - it doesn't have any sideeffects
2850 // - it doesn't clobber memory
2851 // - it doesn't return a value by-reference
2852 // It can be marked readnone if it doesn't have any input memory constraints
2853 // in addition to meeting the conditions listed above.
2854 bool ReadOnly = true, ReadNone = true;
2855
2856 for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
2857 TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
2858
2859 // Simplify the output constraint.
2860 std::string OutputConstraint(S.getOutputConstraint(i));
2861 OutputConstraint = getTarget().simplifyConstraint(
2862 Constraint: StringRef(OutputConstraint).substr(Start: 1), OutCons: &OutputConstraintInfos);
2863
2864 const Expr *OutExpr = S.getOutputExpr(i);
2865 OutExpr = OutExpr->IgnoreParenNoopCasts(Ctx: getContext());
2866
2867 std::string GCCReg;
2868 OutputConstraint = S.addVariableConstraints(
2869 Constraint: OutputConstraint, AsmExpr: *OutExpr, Target: getTarget(), EarlyClobber: Info.earlyClobber(),
2870 UnsupportedCB: [&](const Stmt *UnspStmt, StringRef Msg) {
2871 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
2872 },
2873 GCCReg: &GCCReg);
2874 // Give an error on multiple outputs to same physreg.
2875 if (!GCCReg.empty() && !PhysRegOutputs.insert(V: GCCReg).second)
2876 CGM.Error(loc: S.getAsmLoc(), error: "multiple outputs to hard register: " + GCCReg);
2877
2878 OutputConstraints.push_back(x: OutputConstraint);
2879 LValue Dest = EmitLValue(E: OutExpr);
2880 if (!Constraints.empty())
2881 Constraints += ',';
2882
2883 // If this is a register output, then make the inline asm return it
2884 // by-value. If this is a memory result, return the value by-reference.
2885 QualType QTy = OutExpr->getType();
2886 const bool IsScalarOrAggregate = hasScalarEvaluationKind(T: QTy) ||
2887 hasAggregateEvaluationKind(T: QTy);
2888 if (!Info.allowsMemory() && IsScalarOrAggregate) {
2889
2890 Constraints += "=" + OutputConstraint;
2891 ResultRegQualTys.push_back(x: QTy);
2892 ResultRegDests.push_back(x: Dest);
2893
2894 ResultBounds.emplace_back(args: Info.getOutputOperandBounds());
2895
2896 llvm::Type *Ty = ConvertTypeForMem(T: QTy);
2897 const bool RequiresCast = Info.allowsRegister() &&
2898 (getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty) ||
2899 Ty->isAggregateType());
2900
2901 ResultTruncRegTypes.push_back(x: Ty);
2902 ResultTypeRequiresCast.push_back(Val: RequiresCast);
2903
2904 if (RequiresCast) {
2905 unsigned Size = getContext().getTypeSize(T: QTy);
2906 if (Size)
2907 Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2908 else
2909 CGM.Error(loc: OutExpr->getExprLoc(), error: "output size should not be zero");
2910 }
2911 ResultRegTypes.push_back(x: Ty);
2912 // If this output is tied to an input, and if the input is larger, then
2913 // we need to set the actual result type of the inline asm node to be the
2914 // same as the input type.
2915 if (Info.hasMatchingInput()) {
2916 unsigned InputNo;
2917 for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2918 TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2919 if (Input.hasTiedOperand() && Input.getTiedOperand() == i)
2920 break;
2921 }
2922 assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2923
2924 QualType InputTy = S.getInputExpr(i: InputNo)->getType();
2925 QualType OutputType = OutExpr->getType();
2926
2927 uint64_t InputSize = getContext().getTypeSize(T: InputTy);
2928 if (getContext().getTypeSize(T: OutputType) < InputSize) {
2929 // Form the asm to return the value as a larger integer or fp type.
2930 ResultRegTypes.back() = ConvertType(T: InputTy);
2931 }
2932 }
2933 if (llvm::Type* AdjTy =
2934 getTargetHooks().adjustInlineAsmType(CGF&: *this, Constraint: OutputConstraint,
2935 Ty: ResultRegTypes.back()))
2936 ResultRegTypes.back() = AdjTy;
2937 else {
2938 CGM.getDiags().Report(Loc: S.getAsmLoc(),
2939 DiagID: diag::err_asm_invalid_type_in_input)
2940 << OutExpr->getType() << OutputConstraint;
2941 }
2942
2943 // Update largest vector width for any vector types.
2944 if (auto *VT = dyn_cast<llvm::VectorType>(Val: ResultRegTypes.back()))
2945 LargestVectorWidth =
2946 std::max(a: (uint64_t)LargestVectorWidth,
2947 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
2948 } else {
2949 Address DestAddr = Dest.getAddress();
2950 // Matrix types in memory are represented by arrays, but accessed through
2951 // vector pointers, with the alignment specified on the access operation.
2952 // For inline assembly, update pointer arguments to use vector pointers.
2953 // Otherwise there will be a mis-match if the matrix is also an
2954 // input-argument which is represented as vector.
2955 if (isa<MatrixType>(Val: OutExpr->getType().getCanonicalType()))
2956 DestAddr = DestAddr.withElementType(ElemTy: ConvertType(T: OutExpr->getType()));
2957
2958 ArgTypes.push_back(x: DestAddr.getType());
2959 ArgElemTypes.push_back(x: DestAddr.getElementType());
2960 Args.push_back(x: DestAddr.emitRawPointer(CGF&: *this));
2961 Constraints += "=*";
2962 Constraints += OutputConstraint;
2963 ReadOnly = ReadNone = false;
2964 }
2965
2966 if (Info.isReadWrite()) {
2967 InOutConstraints += ',';
2968
2969 const Expr *InputExpr = S.getOutputExpr(i);
2970 llvm::Value *Arg;
2971 llvm::Type *ArgElemType;
2972 std::tie(args&: Arg, args&: ArgElemType) = EmitAsmInputLValue(
2973 Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr&: InOutConstraints,
2974 Loc: InputExpr->getExprLoc());
2975
2976 if (llvm::Type* AdjTy =
2977 getTargetHooks().adjustInlineAsmType(CGF&: *this, Constraint: OutputConstraint,
2978 Ty: Arg->getType()))
2979 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
2980
2981 // Update largest vector width for any vector types.
2982 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
2983 LargestVectorWidth =
2984 std::max(a: (uint64_t)LargestVectorWidth,
2985 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
2986 // Only tie earlyclobber physregs.
2987 if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
2988 InOutConstraints += llvm::utostr(X: i);
2989 else
2990 InOutConstraints += OutputConstraint;
2991
2992 InOutArgTypes.push_back(x: Arg->getType());
2993 InOutArgElemTypes.push_back(x: ArgElemType);
2994 InOutArgs.push_back(x: Arg);
2995 }
2996 }
2997
2998 // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2999 // to the return value slot. Only do this when returning in registers.
3000 if (isa<MSAsmStmt>(Val: &S)) {
3001 const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();
3002 if (RetAI.isDirect() || RetAI.isExtend()) {
3003 // Make a fake lvalue for the return value slot.
3004 LValue ReturnSlot = MakeAddrLValueWithoutTBAA(Addr: ReturnValue, T: FnRetTy);
3005 CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
3006 CGF&: *this, ReturnValue: ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
3007 ResultRegDests, AsmString, NumOutputs: S.getNumOutputs());
3008 SawAsmBlock = true;
3009 }
3010 }
3011
3012 for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
3013 const Expr *InputExpr = S.getInputExpr(i);
3014
3015 TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
3016
3017 if (Info.allowsMemory())
3018 ReadNone = false;
3019
3020 if (!Constraints.empty())
3021 Constraints += ',';
3022
3023 // Simplify the input constraint.
3024 std::string InputConstraint(S.getInputConstraint(i));
3025 InputConstraint =
3026 getTarget().simplifyConstraint(Constraint: InputConstraint, OutCons: &OutputConstraintInfos);
3027
3028 InputConstraint = S.addVariableConstraints(
3029 Constraint: InputConstraint, AsmExpr: *InputExpr->IgnoreParenNoopCasts(Ctx: getContext()),
3030 Target: getTarget(), EarlyClobber: false /* No EarlyClobber */,
3031 UnsupportedCB: [&](const Stmt *UnspStmt, std::string_view Msg) {
3032 CGM.ErrorUnsupported(S: UnspStmt, Type: Msg);
3033 });
3034
3035 std::string ReplaceConstraint (InputConstraint);
3036 llvm::Value *Arg;
3037 llvm::Type *ArgElemType;
3038 std::tie(args&: Arg, args&: ArgElemType) = EmitAsmInput(Info, InputExpr, ConstraintStr&: Constraints);
3039
3040 // If this input argument is tied to a larger output result, extend the
3041 // input to be the same size as the output. The LLVM backend wants to see
3042 // the input and output of a matching constraint be the same size. Note
3043 // that GCC does not define what the top bits are here. We use zext because
3044 // that is usually cheaper, but LLVM IR should really get an anyext someday.
3045 if (Info.hasTiedOperand()) {
3046 unsigned Output = Info.getTiedOperand();
3047 QualType OutputType = S.getOutputExpr(i: Output)->getType();
3048 QualType InputTy = InputExpr->getType();
3049
3050 if (getContext().getTypeSize(T: OutputType) >
3051 getContext().getTypeSize(T: InputTy)) {
3052 // Use ptrtoint as appropriate so that we can do our extension.
3053 if (isa<llvm::PointerType>(Val: Arg->getType()))
3054 Arg = Builder.CreatePtrToInt(V: Arg, DestTy: IntPtrTy);
3055 llvm::Type *OutputTy = ConvertType(T: OutputType);
3056 if (isa<llvm::IntegerType>(Val: OutputTy))
3057 Arg = Builder.CreateZExt(V: Arg, DestTy: OutputTy);
3058 else if (isa<llvm::PointerType>(Val: OutputTy))
3059 Arg = Builder.CreateZExt(V: Arg, DestTy: IntPtrTy);
3060 else if (OutputTy->isFloatingPointTy())
3061 Arg = Builder.CreateFPExt(V: Arg, DestTy: OutputTy);
3062 }
3063 // Deal with the tied operands' constraint code in adjustInlineAsmType.
3064 ReplaceConstraint = OutputConstraints[Output];
3065 }
3066 if (llvm::Type* AdjTy =
3067 getTargetHooks().adjustInlineAsmType(CGF&: *this, Constraint: ReplaceConstraint,
3068 Ty: Arg->getType()))
3069 Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3070 else
3071 CGM.getDiags().Report(Loc: S.getAsmLoc(), DiagID: diag::err_asm_invalid_type_in_input)
3072 << InputExpr->getType() << InputConstraint;
3073
3074 // Update largest vector width for any vector types.
3075 if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3076 LargestVectorWidth =
3077 std::max(a: (uint64_t)LargestVectorWidth,
3078 b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3079
3080 ArgTypes.push_back(x: Arg->getType());
3081 ArgElemTypes.push_back(x: ArgElemType);
3082 Args.push_back(x: Arg);
3083 Constraints += InputConstraint;
3084 }
3085
3086 // Append the "input" part of inout constraints.
3087 for (unsigned i = 0, e = InOutArgs.size(); i != e; i++) {
3088 ArgTypes.push_back(x: InOutArgTypes[i]);
3089 ArgElemTypes.push_back(x: InOutArgElemTypes[i]);
3090 Args.push_back(x: InOutArgs[i]);
3091 }
3092 Constraints += InOutConstraints;
3093
3094 // Labels
3095 SmallVector<llvm::BasicBlock *, 16> Transfer;
3096 llvm::BasicBlock *Fallthrough = nullptr;
3097 bool IsGCCAsmGoto = false;
3098 if (const auto *GS = dyn_cast<GCCAsmStmt>(Val: &S)) {
3099 IsGCCAsmGoto = GS->isAsmGoto();
3100 if (IsGCCAsmGoto) {
3101 for (const auto *E : GS->labels()) {
3102 JumpDest Dest = getJumpDestForLabel(D: E->getLabel());
3103 Transfer.push_back(Elt: Dest.getBlock());
3104 if (!Constraints.empty())
3105 Constraints += ',';
3106 Constraints += "!i";
3107 }
3108 Fallthrough = createBasicBlock(name: "asm.fallthrough");
3109 }
3110 }
3111
3112 bool HasUnwindClobber = false;
3113
3114 // Clobbers
3115 for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
3116 std::string Clobber = S.getClobber(i);
3117
3118 if (Clobber == "memory")
3119 ReadOnly = ReadNone = false;
3120 else if (Clobber == "unwind") {
3121 HasUnwindClobber = true;
3122 continue;
3123 } else if (Clobber != "cc") {
3124 Clobber = getTarget().getNormalizedGCCRegisterName(Name: Clobber);
3125 if (CGM.getCodeGenOpts().StackClashProtector &&
3126 getTarget().isSPRegName(Clobber)) {
3127 CGM.getDiags().Report(Loc: S.getAsmLoc(),
3128 DiagID: diag::warn_stack_clash_protection_inline_asm);
3129 }
3130 }
3131
3132 if (isa<MSAsmStmt>(Val: &S)) {
3133 if (Clobber == "eax" || Clobber == "edx") {
3134 if (Constraints.find(s: "=&A") != std::string::npos)
3135 continue;
3136 std::string::size_type position1 =
3137 Constraints.find(str: "={" + Clobber + "}");
3138 if (position1 != std::string::npos) {
3139 Constraints.insert(pos: position1 + 1, s: "&");
3140 continue;
3141 }
3142 std::string::size_type position2 = Constraints.find(s: "=A");
3143 if (position2 != std::string::npos) {
3144 Constraints.insert(pos: position2 + 1, s: "&");
3145 continue;
3146 }
3147 }
3148 }
3149 if (!Constraints.empty())
3150 Constraints += ',';
3151
3152 Constraints += "~{";
3153 Constraints += Clobber;
3154 Constraints += '}';
3155 }
3156
3157 assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
3158 "unwind clobber can't be used with asm goto");
3159
3160 // Add machine specific clobbers
3161 std::string_view MachineClobbers = getTarget().getClobbers();
3162 if (!MachineClobbers.empty()) {
3163 if (!Constraints.empty())
3164 Constraints += ',';
3165 Constraints += MachineClobbers;
3166 }
3167
3168 llvm::Type *ResultType;
3169 if (ResultRegTypes.empty())
3170 ResultType = VoidTy;
3171 else if (ResultRegTypes.size() == 1)
3172 ResultType = ResultRegTypes[0];
3173 else
3174 ResultType = llvm::StructType::get(Context&: getLLVMContext(), Elements: ResultRegTypes);
3175
3176 llvm::FunctionType *FTy =
3177 llvm::FunctionType::get(Result: ResultType, Params: ArgTypes, isVarArg: false);
3178
3179 bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
3180
3181 llvm::InlineAsm::AsmDialect GnuAsmDialect =
3182 CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
3183 ? llvm::InlineAsm::AD_ATT
3184 : llvm::InlineAsm::AD_Intel;
3185 llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(Val: &S) ?
3186 llvm::InlineAsm::AD_Intel : GnuAsmDialect;
3187
3188 llvm::InlineAsm *IA = llvm::InlineAsm::get(
3189 Ty: FTy, AsmString, Constraints, hasSideEffects: HasSideEffect,
3190 /* IsAlignStack */ isAlignStack: false, asmDialect: AsmDialect, canThrow: HasUnwindClobber);
3191 std::vector<llvm::Value*> RegResults;
3192 llvm::CallBrInst *CBR;
3193 llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>
3194 CBRRegResults;
3195 if (IsGCCAsmGoto) {
3196 CBR = Builder.CreateCallBr(Callee: IA, DefaultDest: Fallthrough, IndirectDests: Transfer, Args);
3197 EmitBlock(BB: Fallthrough);
3198 UpdateAsmCallInst(Result&: *CBR, HasSideEffect, /*HasUnwindClobber=*/false, ReadOnly,
3199 ReadNone, NoMerge: InNoMergeAttributedStmt,
3200 NoConvergent: InNoConvergentAttributedStmt, S, ResultRegTypes,
3201 ArgElemTypes, CGF&: *this, RegResults);
3202 // Because we are emitting code top to bottom, we don't have enough
3203 // information at this point to know precisely whether we have a critical
3204 // edge. If we have outputs, split all indirect destinations.
3205 if (!RegResults.empty()) {
3206 unsigned i = 0;
3207 for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {
3208 llvm::Twine SynthName = Dest->getName() + ".split";
3209 llvm::BasicBlock *SynthBB = createBasicBlock(name: SynthName);
3210 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
3211 Builder.SetInsertPoint(SynthBB);
3212
3213 if (ResultRegTypes.size() == 1) {
3214 CBRRegResults[SynthBB].push_back(Elt: CBR);
3215 } else {
3216 for (unsigned j = 0, e = ResultRegTypes.size(); j != e; ++j) {
3217 llvm::Value *Tmp = Builder.CreateExtractValue(Agg: CBR, Idxs: j, Name: "asmresult");
3218 CBRRegResults[SynthBB].push_back(Elt: Tmp);
3219 }
3220 }
3221
3222 EmitBranch(Target: Dest);
3223 EmitBlock(BB: SynthBB);
3224 CBR->setIndirectDest(i: i++, B: SynthBB);
3225 }
3226 }
3227 } else if (HasUnwindClobber) {
3228 llvm::CallBase *Result = EmitCallOrInvoke(Callee: IA, Args, Name: "");
3229 UpdateAsmCallInst(Result&: *Result, HasSideEffect, /*HasUnwindClobber=*/true,
3230 ReadOnly, ReadNone, NoMerge: InNoMergeAttributedStmt,
3231 NoConvergent: InNoConvergentAttributedStmt, S, ResultRegTypes,
3232 ArgElemTypes, CGF&: *this, RegResults);
3233 } else {
3234 llvm::CallInst *Result =
3235 Builder.CreateCall(Callee: IA, Args, OpBundles: getBundlesForFunclet(Callee: IA));
3236 UpdateAsmCallInst(Result&: *Result, HasSideEffect, /*HasUnwindClobber=*/false,
3237 ReadOnly, ReadNone, NoMerge: InNoMergeAttributedStmt,
3238 NoConvergent: InNoConvergentAttributedStmt, S, ResultRegTypes,
3239 ArgElemTypes, CGF&: *this, RegResults);
3240 }
3241
3242 EmitAsmStores(CGF&: *this, S, RegResults, ResultRegTypes, ResultTruncRegTypes,
3243 ResultRegDests, ResultRegQualTys, ResultTypeRequiresCast,
3244 ResultBounds);
3245
3246 // If this is an asm goto with outputs, repeat EmitAsmStores, but with a
3247 // different insertion point; one for each indirect destination and with
3248 // CBRRegResults rather than RegResults.
3249 if (IsGCCAsmGoto && !CBRRegResults.empty()) {
3250 for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {
3251 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
3252 Builder.SetInsertPoint(TheBB: Succ, IP: --(Succ->end()));
3253 EmitAsmStores(CGF&: *this, S, RegResults: CBRRegResults[Succ], ResultRegTypes,
3254 ResultTruncRegTypes, ResultRegDests, ResultRegQualTys,
3255 ResultTypeRequiresCast, ResultBounds);
3256 }
3257 }
3258}
3259
3260LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
3261 const RecordDecl *RD = S.getCapturedRecordDecl();
3262 CanQualType RecordTy = getContext().getCanonicalTagType(TD: RD);
3263
3264 // Initialize the captured struct.
3265 LValue SlotLV =
3266 MakeAddrLValue(Addr: CreateMemTemp(T: RecordTy, Name: "agg.captured"), T: RecordTy);
3267
3268 RecordDecl::field_iterator CurField = RD->field_begin();
3269 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
3270 E = S.capture_init_end();
3271 I != E; ++I, ++CurField) {
3272 LValue LV = EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
3273 if (CurField->hasCapturedVLAType()) {
3274 EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
3275 } else {
3276 EmitInitializerForField(Field: *CurField, LHS: LV, Init: *I);
3277 }
3278 }
3279
3280 return SlotLV;
3281}
3282
3283/// Generate an outlined function for the body of a CapturedStmt, store any
3284/// captured variables into the captured struct, and call the outlined function.
3285llvm::Function *
3286CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
3287 LValue CapStruct = InitCapturedStruct(S);
3288
3289 // Emit the CapturedDecl
3290 CodeGenFunction CGF(CGM, true);
3291 CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
3292 llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
3293 delete CGF.CapturedStmtInfo;
3294
3295 // Emit call to the helper function.
3296 EmitCallOrInvoke(Callee: F, Args: CapStruct.getPointer(CGF&: *this));
3297
3298 return F;
3299}
3300
3301Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
3302 LValue CapStruct = InitCapturedStruct(S);
3303 return CapStruct.getAddress();
3304}
3305
3306/// Creates the outlined function for a CapturedStmt.
3307llvm::Function *
3308CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
3309 assert(CapturedStmtInfo &&
3310 "CapturedStmtInfo should be set when generating the captured function");
3311 const CapturedDecl *CD = S.getCapturedDecl();
3312 const RecordDecl *RD = S.getCapturedRecordDecl();
3313 SourceLocation Loc = S.getBeginLoc();
3314 assert(CD->hasBody() && "missing CapturedDecl body");
3315
3316 // Build the argument list.
3317 ASTContext &Ctx = CGM.getContext();
3318 FunctionArgList Args;
3319 Args.append(in_start: CD->param_begin(), in_end: CD->param_end());
3320
3321 // Create the function declaration.
3322 const CGFunctionInfo &FuncInfo =
3323 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: Ctx.VoidTy, args: Args);
3324 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
3325
3326 llvm::Function *F =
3327 llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
3328 N: CapturedStmtInfo->getHelperName(), M: &CGM.getModule());
3329 CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
3330 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3331 F->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3332 if (CD->isNothrow())
3333 F->addFnAttr(Kind: llvm::Attribute::NoUnwind);
3334
3335 // Generate the function.
3336 StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc: CD->getLocation(),
3337 StartLoc: CD->getBody()->getBeginLoc());
3338 // Set the context parameter in CapturedStmtInfo.
3339 Address DeclPtr = GetAddrOfLocalVar(VD: CD->getContextParam());
3340 CapturedStmtInfo->setContextValue(Builder.CreateLoad(Addr: DeclPtr));
3341
3342 // Initialize variable-length arrays.
3343 LValue Base = MakeNaturalAlignRawAddrLValue(
3344 V: CapturedStmtInfo->getContextValue(), T: Ctx.getCanonicalTagType(TD: RD));
3345 for (auto *FD : RD->fields()) {
3346 if (FD->hasCapturedVLAType()) {
3347 auto *ExprArg =
3348 EmitLoadOfLValue(V: EmitLValueForField(Base, Field: FD), Loc: S.getBeginLoc())
3349 .getScalarVal();
3350 auto VAT = FD->getCapturedVLAType();
3351 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
3352 }
3353 }
3354
3355 // If 'this' is captured, load it into CXXThisValue.
3356 if (CapturedStmtInfo->isCXXThisExprCaptured()) {
3357 FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
3358 LValue ThisLValue = EmitLValueForField(Base, Field: FD);
3359 CXXThisValue = EmitLoadOfLValue(V: ThisLValue, Loc).getScalarVal();
3360 }
3361
3362 PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
3363 CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
3364 FinishFunction(EndLoc: CD->getBodyRBrace());
3365
3366 return F;
3367}
3368
3369// Returns the first convergence entry/loop/anchor instruction found in |BB|.
3370// std::nullptr otherwise.
3371static llvm::ConvergenceControlInst *getConvergenceToken(llvm::BasicBlock *BB) {
3372 for (auto &I : *BB) {
3373 if (auto *CI = dyn_cast<llvm::ConvergenceControlInst>(Val: &I))
3374 return CI;
3375 }
3376 return nullptr;
3377}
3378
3379llvm::CallBase *
3380CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input) {
3381 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3382 assert(ParentToken);
3383
3384 llvm::Value *bundleArgs[] = {ParentToken};
3385 llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
3386 auto *Output = llvm::CallBase::addOperandBundle(
3387 CB: Input, ID: llvm::LLVMContext::OB_convergencectrl, OB, InsertPt: Input->getIterator());
3388 Input->replaceAllUsesWith(V: Output);
3389 Input->eraseFromParent();
3390 return Output;
3391}
3392
3393llvm::ConvergenceControlInst *
3394CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {
3395 llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3396 assert(ParentToken);
3397 return llvm::ConvergenceControlInst::CreateLoop(BB&: *BB, Parent: ParentToken);
3398}
3399
3400llvm::ConvergenceControlInst *
3401CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
3402 llvm::BasicBlock *BB = &F->getEntryBlock();
3403 llvm::ConvergenceControlInst *Token = getConvergenceToken(BB);
3404 if (Token)
3405 return Token;
3406
3407 // Adding a convergence token requires the function to be marked as
3408 // convergent.
3409 F->setConvergent();
3410 return llvm::ConvergenceControlInst::CreateEntry(BB&: *BB);
3411}
3412