1//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit blocks.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBlocks.h"
14#include "CGCXXABI.h"
15#include "CGDebugInfo.h"
16#include "CGObjCRuntime.h"
17#include "CGOpenCLRuntime.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "CodeGenPGO.h"
21#include "ConstantEmitter.h"
22#include "TargetInfo.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/DeclObjC.h"
25#include "clang/CodeGen/ConstantInitBuilder.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Module.h"
28#include "llvm/Support/ScopedPrinter.h"
29#include <algorithm>
30#include <cstdio>
31
32using namespace clang;
33using namespace CodeGen;
34
35CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
36 : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
37 NoEscape(false), HasCXXObject(false), UsesStret(false),
38 HasCapturedVariableLayout(false), CapturesNonExternalType(false),
39 LocalAddress(RawAddress::invalid()), StructureType(nullptr),
40 Block(block) {
41
42 // Skip asm prefix, if any. 'name' is usually taken directly from
43 // the mangled name of the enclosing function.
44 name.consume_front(Prefix: "\01");
45}
46
47// Anchor the vtable to this translation unit.
48BlockByrefHelpers::~BlockByrefHelpers() {}
49
50/// Build the given block as a global block.
51static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
52 const CGBlockInfo &blockInfo,
53 llvm::Constant *blockFn);
54
55/// Build the helper function to copy a block.
56static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,
57 const CGBlockInfo &blockInfo) {
58 return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);
59}
60
61/// Build the helper function to dispose of a block.
62static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
63 const CGBlockInfo &blockInfo) {
64 return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
65}
66
67namespace {
68
69enum class CaptureStrKind {
70 // String for the copy helper.
71 CopyHelper,
72 // String for the dispose helper.
73 DisposeHelper,
74 // Merge the strings for the copy helper and dispose helper.
75 Merged
76};
77
78} // end anonymous namespace
79
80static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
81 CaptureStrKind StrKind,
82 CharUnits BlockAlignment,
83 CodeGenModule &CGM);
84
85static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo,
86 CodeGenModule &CGM) {
87 std::string Name = "__block_descriptor_";
88 Name += llvm::to_string(Value: BlockInfo.BlockSize.getQuantity()) + "_";
89
90 if (BlockInfo.NeedsCopyDispose) {
91 if (CGM.getLangOpts().Exceptions)
92 Name += "e";
93 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
94 Name += "a";
95 Name += llvm::to_string(Value: BlockInfo.BlockAlign.getQuantity()) + "_";
96
97 for (auto &Cap : BlockInfo.SortedCaptures) {
98 if (Cap.isConstantOrTrivial())
99 continue;
100
101 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
102
103 if (Cap.CopyKind == Cap.DisposeKind) {
104 // If CopyKind and DisposeKind are the same, merge the capture
105 // information.
106 assert(Cap.CopyKind != BlockCaptureEntityKind::None &&
107 "shouldn't see BlockCaptureManagedEntity that is None");
108 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::Merged,
109 BlockAlignment: BlockInfo.BlockAlign, CGM);
110 } else {
111 // If CopyKind and DisposeKind are not the same, which can happen when
112 // either Kind is None or the captured object is a __strong block,
113 // concatenate the copy and dispose strings.
114 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::CopyHelper,
115 BlockAlignment: BlockInfo.BlockAlign, CGM);
116 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::DisposeHelper,
117 BlockAlignment: BlockInfo.BlockAlign, CGM);
118 }
119 }
120 Name += "_";
121 }
122
123 std::string TypeAtEncoding;
124
125 if (!CGM.getCodeGenOpts().DisableBlockSignatureString) {
126 TypeAtEncoding =
127 CGM.getContext().getObjCEncodingForBlock(blockExpr: BlockInfo.getBlockExpr());
128 /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms
129 /// as a separator between symbol name and symbol version.
130 llvm::replace(Range&: TypeAtEncoding, OldValue: '@', NewValue: '\1');
131 }
132 Name += "e" + llvm::to_string(Value: TypeAtEncoding.size()) + "_" + TypeAtEncoding;
133 Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, blockInfo: BlockInfo);
134 return Name;
135}
136
137/// buildBlockDescriptor - Build the block descriptor meta-data for a block.
138/// buildBlockDescriptor is accessed from 5th field of the Block_literal
139/// meta-data and contains stationary information about the block literal.
140/// Its definition will have 4 (or optionally 6) words.
141/// \code
142/// struct Block_descriptor {
143/// unsigned long reserved;
144/// unsigned long size; // size of Block_literal metadata in bytes.
145/// void *copy_func_helper_decl; // optional copy helper.
146/// void *destroy_func_decl; // optional destructor helper.
147/// void *block_method_encoding_address; // @encode for block literal signature.
148/// void *block_layout_info; // encoding of captured block variables.
149/// };
150/// \endcode
151static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
152 const CGBlockInfo &blockInfo) {
153 ASTContext &C = CGM.getContext();
154
155 llvm::IntegerType *ulong =
156 cast<llvm::IntegerType>(Val: CGM.getTypes().ConvertType(T: C.UnsignedLongTy));
157 llvm::PointerType *i8p = nullptr;
158 if (CGM.getLangOpts().OpenCL)
159 i8p = llvm::PointerType::get(
160 C&: CGM.getLLVMContext(), AddressSpace: C.getTargetAddressSpace(AS: LangAS::opencl_constant));
161 else
162 i8p = CGM.VoidPtrTy;
163
164 std::string descName;
165
166 // If an equivalent block descriptor global variable exists, return it.
167 if (C.getLangOpts().ObjC &&
168 CGM.getLangOpts().getGC() == LangOptions::NonGC) {
169 descName = getBlockDescriptorName(BlockInfo: blockInfo, CGM);
170 if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(Name: descName))
171 return desc;
172 }
173
174 // If there isn't an equivalent block descriptor global variable, create a new
175 // one.
176 ConstantInitBuilder builder(CGM);
177 auto elements = builder.beginStruct();
178
179 // reserved
180 elements.addInt(intTy: ulong, value: 0);
181
182 // Size
183 // FIXME: What is the right way to say this doesn't fit? We should give
184 // a user diagnostic in that case. Better fix would be to change the
185 // API to size_t.
186 elements.addInt(intTy: ulong, value: blockInfo.BlockSize.getQuantity());
187
188 // Optional copy/dispose helpers.
189 bool hasInternalHelper = false;
190 if (blockInfo.NeedsCopyDispose) {
191 auto &Schema = CGM.getCodeGenOpts().PointerAuth.BlockHelperFunctionPointers;
192 // copy_func_helper_decl
193 llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);
194 elements.addSignedPointer(Pointer: copyHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
195
196 // destroy_func_decl
197 llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
198 elements.addSignedPointer(Pointer: disposeHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
199
200 if (cast<llvm::Function>(Val: copyHelper->stripPointerCasts())
201 ->hasInternalLinkage() ||
202 cast<llvm::Function>(Val: disposeHelper->stripPointerCasts())
203 ->hasInternalLinkage())
204 hasInternalHelper = true;
205 }
206
207 // Signature. Mandatory ObjC-style method descriptor @encode sequence.
208 if (CGM.getCodeGenOpts().DisableBlockSignatureString) {
209 elements.addNullPointer(ptrTy: i8p);
210 } else {
211 std::string typeAtEncoding =
212 CGM.getContext().getObjCEncodingForBlock(blockExpr: blockInfo.getBlockExpr());
213 elements.add(value: CGM.GetAddrOfConstantCString(Str: typeAtEncoding).getPointer());
214 }
215
216 // GC layout.
217 if (C.getLangOpts().ObjC) {
218 if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
219 elements.add(value: CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
220 else
221 elements.add(value: CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));
222 }
223 else
224 elements.addNullPointer(ptrTy: i8p);
225
226 unsigned AddrSpace = 0;
227 if (C.getLangOpts().OpenCL)
228 AddrSpace = C.getTargetAddressSpace(AS: LangAS::opencl_constant);
229
230 llvm::GlobalValue::LinkageTypes linkage;
231 if (descName.empty()) {
232 linkage = llvm::GlobalValue::InternalLinkage;
233 descName = "__block_descriptor_tmp";
234 } else if (hasInternalHelper) {
235 // If either the copy helper or the dispose helper has internal linkage,
236 // the block descriptor must have internal linkage too.
237 linkage = llvm::GlobalValue::InternalLinkage;
238 } else {
239 linkage = llvm::GlobalValue::LinkOnceODRLinkage;
240 }
241
242 llvm::GlobalVariable *global =
243 elements.finishAndCreateGlobal(args&: descName, args: CGM.getPointerAlign(),
244 /*constant*/ args: true, args&: linkage, args&: AddrSpace);
245
246 if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) {
247 if (CGM.supportsCOMDAT())
248 global->setComdat(CGM.getModule().getOrInsertComdat(Name: descName));
249 global->setVisibility(llvm::GlobalValue::HiddenVisibility);
250 global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
251 }
252
253 return global;
254}
255
256/*
257 Purely notional variadic template describing the layout of a block.
258
259 template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>
260 struct Block_literal {
261 /// Initialized to one of:
262 /// extern void *_NSConcreteStackBlock[];
263 /// extern void *_NSConcreteGlobalBlock[];
264 ///
265 /// In theory, we could start one off malloc'ed by setting
266 /// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using
267 /// this isa:
268 /// extern void *_NSConcreteMallocBlock[];
269 struct objc_class *isa;
270
271 /// These are the flags (with corresponding bit number) that the
272 /// compiler is actually supposed to know about.
273 /// 23. BLOCK_IS_NOESCAPE - indicates that the block is non-escaping
274 /// 25. BLOCK_HAS_COPY_DISPOSE - indicates that the block
275 /// descriptor provides copy and dispose helper functions
276 /// 26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured
277 /// object with a nontrivial destructor or copy constructor
278 /// 28. BLOCK_IS_GLOBAL - indicates that the block is allocated
279 /// as global memory
280 /// 29. BLOCK_USE_STRET - indicates that the block function
281 /// uses stret, which objc_msgSend needs to know about
282 /// 30. BLOCK_HAS_SIGNATURE - indicates that the block has an
283 /// @encoded signature string
284 /// And we're not supposed to manipulate these:
285 /// 24. BLOCK_NEEDS_FREE - indicates that the block has been moved
286 /// to malloc'ed memory
287 /// 27. BLOCK_IS_GC - indicates that the block has been moved to
288 /// to GC-allocated memory
289 /// Additionally, the bottom 16 bits are a reference count which
290 /// should be zero on the stack.
291 int flags;
292
293 /// Reserved; should be zero-initialized.
294 int reserved;
295
296 /// Function pointer generated from block literal.
297 _ResultType (*invoke)(Block_literal *, _ParamTypes...);
298
299 /// Block description metadata generated from block literal.
300 struct Block_descriptor *block_descriptor;
301
302 /// Captured values follow.
303 _CapturesTypes captures...;
304 };
305 */
306
307namespace {
308 /// A chunk of data that we actually have to capture in the block.
309 struct BlockLayoutChunk {
310 CharUnits Alignment;
311 CharUnits Size;
312 const BlockDecl::Capture *Capture; // null for 'this'
313 llvm::Type *Type;
314 QualType FieldType;
315 BlockCaptureEntityKind CopyKind, DisposeKind;
316 BlockFieldFlags CopyFlags, DisposeFlags;
317
318 BlockLayoutChunk(CharUnits align, CharUnits size,
319 const BlockDecl::Capture *capture, llvm::Type *type,
320 QualType fieldType, BlockCaptureEntityKind CopyKind,
321 BlockFieldFlags CopyFlags,
322 BlockCaptureEntityKind DisposeKind,
323 BlockFieldFlags DisposeFlags)
324 : Alignment(align), Size(size), Capture(capture), Type(type),
325 FieldType(fieldType), CopyKind(CopyKind), DisposeKind(DisposeKind),
326 CopyFlags(CopyFlags), DisposeFlags(DisposeFlags) {}
327
328 /// Tell the block info that this chunk has the given field index.
329 void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {
330 if (!Capture) {
331 info.CXXThisIndex = index;
332 info.CXXThisOffset = offset;
333 } else {
334 info.SortedCaptures.push_back(Elt: CGBlockInfo::Capture::makeIndex(
335 index, offset, FieldType, CopyKind, CopyFlags, DisposeKind,
336 DisposeFlags, Cap: Capture));
337 }
338 }
339
340 bool isTrivial() const {
341 return CopyKind == BlockCaptureEntityKind::None &&
342 DisposeKind == BlockCaptureEntityKind::None;
343 }
344 };
345
346 /// Order by 1) all __strong together 2) next, all block together 3) next,
347 /// all byref together 4) next, all __weak together. Preserve descending
348 /// alignment in all situations.
349 bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
350 if (left.Alignment != right.Alignment)
351 return left.Alignment > right.Alignment;
352
353 auto getPrefOrder = [](const BlockLayoutChunk &chunk) {
354 switch (chunk.CopyKind) {
355 case BlockCaptureEntityKind::ARCStrong:
356 return 0;
357 case BlockCaptureEntityKind::BlockObject:
358 switch (chunk.CopyFlags.getBitMask()) {
359 case BLOCK_FIELD_IS_OBJECT:
360 return 0;
361 case BLOCK_FIELD_IS_BLOCK:
362 return 1;
363 case BLOCK_FIELD_IS_BYREF:
364 return 2;
365 default:
366 break;
367 }
368 break;
369 case BlockCaptureEntityKind::ARCWeak:
370 return 3;
371 default:
372 break;
373 }
374 return 4;
375 };
376
377 return getPrefOrder(left) < getPrefOrder(right);
378 }
379} // end anonymous namespace
380
381static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
382computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
383 const LangOptions &LangOpts);
384
385static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
386computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
387 const LangOptions &LangOpts);
388
389static void addBlockLayout(CharUnits align, CharUnits size,
390 const BlockDecl::Capture *capture, llvm::Type *type,
391 QualType fieldType,
392 SmallVectorImpl<BlockLayoutChunk> &Layout,
393 CGBlockInfo &Info, CodeGenModule &CGM) {
394 if (!capture) {
395 // 'this' capture.
396 Layout.push_back(Elt: BlockLayoutChunk(
397 align, size, capture, type, fieldType, BlockCaptureEntityKind::None,
398 BlockFieldFlags(), BlockCaptureEntityKind::None, BlockFieldFlags()));
399 return;
400 }
401
402 const LangOptions &LangOpts = CGM.getLangOpts();
403 BlockCaptureEntityKind CopyKind, DisposeKind;
404 BlockFieldFlags CopyFlags, DisposeFlags;
405
406 std::tie(args&: CopyKind, args&: CopyFlags) =
407 computeCopyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
408 std::tie(args&: DisposeKind, args&: DisposeFlags) =
409 computeDestroyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
410 Layout.push_back(Elt: BlockLayoutChunk(align, size, capture, type, fieldType,
411 CopyKind, CopyFlags, DisposeKind,
412 DisposeFlags));
413
414 if (Info.NoEscape)
415 return;
416
417 if (!Layout.back().isTrivial())
418 Info.NeedsCopyDispose = true;
419}
420
421/// Determines if the given type is safe for constant capture in C++.
422static bool isSafeForCXXConstantCapture(QualType type) {
423 const auto *record = type->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
424
425 // Only records can be unsafe.
426 if (!record)
427 return true;
428
429 // Maintain semantics for classes with non-trivial dtors or copy ctors.
430 if (!record->hasTrivialDestructor()) return false;
431 if (record->hasNonTrivialCopyConstructor()) return false;
432
433 // Otherwise, we just have to make sure there aren't any mutable
434 // fields that might have changed since initialization.
435 return !record->hasMutableFields();
436}
437
438/// It is illegal to modify a const object after initialization.
439/// Therefore, if a const object has a constant initializer, we don't
440/// actually need to keep storage for it in the block; we'll just
441/// rematerialize it at the start of the block function. This is
442/// acceptable because we make no promises about address stability of
443/// captured variables.
444static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
445 CodeGenFunction *CGF,
446 const VarDecl *var) {
447 // Return if this is a function parameter. We shouldn't try to
448 // rematerialize default arguments of function parameters.
449 if (isa<ParmVarDecl>(Val: var))
450 return nullptr;
451
452 QualType type = var->getType();
453
454 // We can only do this if the variable is const.
455 if (!type.isConstQualified()) return nullptr;
456
457 // Furthermore, in C++ we have to worry about mutable fields:
458 // C++ [dcl.type.cv]p4:
459 // Except that any class member declared mutable can be
460 // modified, any attempt to modify a const object during its
461 // lifetime results in undefined behavior.
462 if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type))
463 return nullptr;
464
465 // If the variable doesn't have any initializer (shouldn't this be
466 // invalid?), it's not clear what we should do. Maybe capture as
467 // zero?
468 const Expr *init = var->getInit();
469 if (!init) return nullptr;
470
471 return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(D: *var);
472}
473
474/// Get the low bit of a nonzero character count. This is the
475/// alignment of the nth byte if the 0th byte is universally aligned.
476static CharUnits getLowBit(CharUnits v) {
477 return CharUnits::fromQuantity(Quantity: v.getQuantity() & (~v.getQuantity() + 1));
478}
479
480static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
481 SmallVectorImpl<llvm::Type*> &elementTypes) {
482
483 assert(elementTypes.empty());
484 if (CGM.getLangOpts().OpenCL) {
485 // The header is basically 'struct { int; int; generic void *;
486 // custom_fields; }'. Assert that struct is packed.
487 auto GenPtrAlign = CharUnits::fromQuantity(
488 Quantity: CGM.getTarget().getPointerAlign(AddrSpace: LangAS::opencl_generic) / 8);
489 auto GenPtrSize = CharUnits::fromQuantity(
490 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: LangAS::opencl_generic) / 8);
491 assert(CGM.getIntSize() <= GenPtrSize);
492 assert(CGM.getIntAlign() <= GenPtrAlign);
493 assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign));
494 elementTypes.push_back(Elt: CGM.IntTy); /* total size */
495 elementTypes.push_back(Elt: CGM.IntTy); /* align */
496 elementTypes.push_back(
497 Elt: CGM.getOpenCLRuntime()
498 .getGenericVoidPointerType()); /* invoke function */
499 unsigned Offset =
500 2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity();
501 unsigned BlockAlign = GenPtrAlign.getQuantity();
502 if (auto *Helper =
503 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
504 for (auto *I : Helper->getCustomFieldTypes()) /* custom fields */ {
505 // TargetOpenCLBlockHelp needs to make sure the struct is packed.
506 // If necessary, add padding fields to the custom fields.
507 unsigned Align = CGM.getDataLayout().getABITypeAlign(Ty: I).value();
508 if (BlockAlign < Align)
509 BlockAlign = Align;
510 assert(Offset % Align == 0);
511 Offset += CGM.getDataLayout().getTypeAllocSize(Ty: I);
512 elementTypes.push_back(Elt: I);
513 }
514 }
515 info.BlockAlign = CharUnits::fromQuantity(Quantity: BlockAlign);
516 info.BlockSize = CharUnits::fromQuantity(Quantity: Offset);
517 } else {
518 // The header is basically 'struct { void *; int; int; void *; void *; }'.
519 // Assert that the struct is packed.
520 assert(CGM.getIntSize() <= CGM.getPointerSize());
521 assert(CGM.getIntAlign() <= CGM.getPointerAlign());
522 assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));
523 info.BlockAlign = CGM.getPointerAlign();
524 info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();
525 elementTypes.push_back(Elt: CGM.VoidPtrTy);
526 elementTypes.push_back(Elt: CGM.IntTy);
527 elementTypes.push_back(Elt: CGM.IntTy);
528 elementTypes.push_back(Elt: CGM.VoidPtrTy);
529 elementTypes.push_back(Elt: CGM.getBlockDescriptorType());
530 }
531}
532
533static QualType getCaptureFieldType(const CodeGenFunction &CGF,
534 const BlockDecl::Capture &CI) {
535 const VarDecl *VD = CI.getVariable();
536
537 // If the variable is captured by an enclosing block or lambda expression,
538 // use the type of the capture field.
539 if (CGF.BlockInfo && CI.isNested())
540 return CGF.BlockInfo->getCapture(var: VD).fieldType();
541 if (auto *FD = CGF.LambdaCaptureFields.lookup(Val: VD))
542 return FD->getType();
543 // If the captured variable is a non-escaping __block variable, the field
544 // type is the reference type. If the variable is a __block variable that
545 // already has a reference type, the field type is the variable's type.
546 return VD->isNonEscapingByref() ?
547 CGF.getContext().getLValueReferenceType(T: VD->getType()) : VD->getType();
548}
549
550/// Compute the layout of the given block. Attempts to lay the block
551/// out with minimal space requirements.
552static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
553 CGBlockInfo &info) {
554 ASTContext &C = CGM.getContext();
555 const BlockDecl *block = info.getBlockDecl();
556
557 SmallVector<llvm::Type*, 8> elementTypes;
558 initializeForBlockHeader(CGM, info, elementTypes);
559 bool hasNonConstantCustomFields = false;
560 if (auto *OpenCLHelper =
561 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper())
562 hasNonConstantCustomFields =
563 !OpenCLHelper->areAllCustomFieldValuesConstant(Info: info);
564 if (!block->hasCaptures() && !hasNonConstantCustomFields) {
565 info.StructureType =
566 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
567 info.CanBeGlobal = true;
568 return;
569 } else if (C.getLangOpts().ObjC &&
570 CGM.getLangOpts().getGC() == LangOptions::NonGC)
571 info.HasCapturedVariableLayout = true;
572
573 if (block->doesNotEscape())
574 info.NoEscape = true;
575
576 // Collect the layout chunks.
577 SmallVector<BlockLayoutChunk, 16> layout;
578 layout.reserve(N: block->capturesCXXThis() +
579 (block->capture_end() - block->capture_begin()));
580
581 CharUnits maxFieldAlign;
582
583 // First, 'this'.
584 if (block->capturesCXXThis()) {
585 QualType thisType = block->getCXXThisType();
586
587 // Theoretically, this could be in a different address space, so
588 // don't assume standard pointer size/align.
589 llvm::Type *llvmType = CGM.getTypes().ConvertType(T: thisType);
590 auto TInfo = CGM.getContext().getTypeInfoInChars(T: thisType);
591 maxFieldAlign = std::max(a: maxFieldAlign, b: TInfo.Align);
592
593 addBlockLayout(align: TInfo.Align, size: TInfo.Width, capture: nullptr, type: llvmType, fieldType: thisType,
594 Layout&: layout, Info&: info, CGM);
595 }
596
597 // Next, all the block captures.
598 for (const auto &CI : block->captures()) {
599 const VarDecl *variable = CI.getVariable();
600
601 if (CI.isEscapingByref()) {
602 // Just use void* instead of a pointer to the byref type.
603 CharUnits align = CGM.getPointerAlign();
604 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
605
606 // Since a __block variable cannot be captured by lambdas, its type and
607 // the capture field type should always match.
608 assert(CGF && getCaptureFieldType(*CGF, CI) == variable->getType() &&
609 "capture type differs from the variable type");
610 addBlockLayout(align, size: CGM.getPointerSize(), capture: &CI, type: CGM.VoidPtrTy,
611 fieldType: variable->getType(), Layout&: layout, Info&: info, CGM);
612 continue;
613 }
614
615 // Otherwise, build a layout chunk with the size and alignment of
616 // the declaration.
617 if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, var: variable)) {
618 info.SortedCaptures.push_back(
619 Elt: CGBlockInfo::Capture::makeConstant(value: constant, Cap: &CI));
620 continue;
621 }
622
623 QualType VT = getCaptureFieldType(CGF: *CGF, CI);
624
625 if (CGM.getLangOpts().CPlusPlus)
626 if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl())
627 if (CI.hasCopyExpr() || !record->hasTrivialDestructor()) {
628 info.HasCXXObject = true;
629 if (!record->isExternallyVisible())
630 info.CapturesNonExternalType = true;
631 }
632
633 CharUnits size = C.getTypeSizeInChars(T: VT);
634 CharUnits align = C.getDeclAlign(D: variable);
635
636 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
637
638 llvm::Type *llvmType =
639 CGM.getTypes().ConvertTypeForMem(T: VT);
640
641 addBlockLayout(align, size, capture: &CI, type: llvmType, fieldType: VT, Layout&: layout, Info&: info, CGM);
642 }
643
644 // If that was everything, we're done here.
645 if (layout.empty()) {
646 info.StructureType =
647 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
648 info.CanBeGlobal = true;
649 info.buildCaptureMap();
650 return;
651 }
652
653 // Sort the layout by alignment. We have to use a stable sort here
654 // to get reproducible results. There should probably be an
655 // llvm::array_pod_stable_sort.
656 llvm::stable_sort(Range&: layout);
657
658 // Needed for blocks layout info.
659 info.BlockHeaderForcedGapOffset = info.BlockSize;
660 info.BlockHeaderForcedGapSize = CharUnits::Zero();
661
662 CharUnits &blockSize = info.BlockSize;
663 info.BlockAlign = std::max(a: maxFieldAlign, b: info.BlockAlign);
664
665 // Assuming that the first byte in the header is maximally aligned,
666 // get the alignment of the first byte following the header.
667 CharUnits endAlign = getLowBit(v: blockSize);
668
669 // If the end of the header isn't satisfactorily aligned for the
670 // maximum thing, look for things that are okay with the header-end
671 // alignment, and keep appending them until we get something that's
672 // aligned right. This algorithm is only guaranteed optimal if
673 // that condition is satisfied at some point; otherwise we can get
674 // things like:
675 // header // next byte has alignment 4
676 // something_with_size_5; // next byte has alignment 1
677 // something_with_alignment_8;
678 // which has 7 bytes of padding, as opposed to the naive solution
679 // which might have less (?).
680 if (endAlign < maxFieldAlign) {
681 SmallVectorImpl<BlockLayoutChunk>::iterator
682 li = layout.begin() + 1, le = layout.end();
683
684 // Look for something that the header end is already
685 // satisfactorily aligned for.
686 for (; li != le && endAlign < li->Alignment; ++li)
687 ;
688
689 // If we found something that's naturally aligned for the end of
690 // the header, keep adding things...
691 if (li != le) {
692 SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
693 for (; li != le; ++li) {
694 assert(endAlign >= li->Alignment);
695
696 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
697 elementTypes.push_back(Elt: li->Type);
698 blockSize += li->Size;
699 endAlign = getLowBit(v: blockSize);
700
701 // ...until we get to the alignment of the maximum field.
702 if (endAlign >= maxFieldAlign) {
703 ++li;
704 break;
705 }
706 }
707 // Don't re-append everything we just appended.
708 layout.erase(CS: first, CE: li);
709 }
710 }
711
712 assert(endAlign == getLowBit(blockSize));
713
714 // At this point, we just have to add padding if the end align still
715 // isn't aligned right.
716 if (endAlign < maxFieldAlign) {
717 CharUnits newBlockSize = blockSize.alignTo(Align: maxFieldAlign);
718 CharUnits padding = newBlockSize - blockSize;
719
720 // If we haven't yet added any fields, remember that there was an
721 // initial gap; this need to go into the block layout bit map.
722 if (blockSize == info.BlockHeaderForcedGapOffset) {
723 info.BlockHeaderForcedGapSize = padding;
724 }
725
726 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
727 NumElements: padding.getQuantity()));
728 blockSize = newBlockSize;
729 endAlign = getLowBit(v: blockSize); // might be > maxFieldAlign
730 }
731
732 assert(endAlign >= maxFieldAlign);
733 assert(endAlign == getLowBit(blockSize));
734 // Slam everything else on now. This works because they have
735 // strictly decreasing alignment and we expect that size is always a
736 // multiple of alignment.
737 for (SmallVectorImpl<BlockLayoutChunk>::iterator
738 li = layout.begin(), le = layout.end(); li != le; ++li) {
739 if (endAlign < li->Alignment) {
740 // size may not be multiple of alignment. This can only happen with
741 // an over-aligned variable. We will be adding a padding field to
742 // make the size be multiple of alignment.
743 CharUnits padding = li->Alignment - endAlign;
744 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
745 NumElements: padding.getQuantity()));
746 blockSize += padding;
747 endAlign = getLowBit(v: blockSize);
748 }
749 assert(endAlign >= li->Alignment);
750 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
751 elementTypes.push_back(Elt: li->Type);
752 blockSize += li->Size;
753 endAlign = getLowBit(v: blockSize);
754 }
755
756 info.buildCaptureMap();
757 info.StructureType =
758 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
759}
760
761/// Emit a block literal expression in the current function.
762llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
763 // If the block has no captures, we won't have a pre-computed
764 // layout for it.
765 if (!blockExpr->getBlockDecl()->hasCaptures())
766 // The block literal is emitted as a global variable, and the block invoke
767 // function has to be extracted from its initializer.
768 if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(BE: blockExpr))
769 return Block;
770
771 CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());
772 computeBlockInfo(CGM, CGF: this, info&: blockInfo);
773 blockInfo.BlockExpression = blockExpr;
774 if (!blockInfo.CanBeGlobal)
775 blockInfo.LocalAddress = CreateTempAlloca(Ty: blockInfo.StructureType,
776 align: blockInfo.BlockAlign, Name: "block");
777 return EmitBlockLiteral(Info: blockInfo);
778}
779
780llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
781 bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;
782 llvm::PointerType *GenVoidPtrTy =
783 IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
784 LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
785 auto GenVoidPtrSize = CharUnits::fromQuantity(
786 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: GenVoidPtrAddr) / 8);
787 // Using the computed layout, generate the actual block function.
788 bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();
789 CodeGenFunction BlockCGF{CGM, true};
790 BlockCGF.SanOpts = SanOpts;
791 auto *InvokeFn = BlockCGF.GenerateBlockFunction(
792 GD: CurGD, Info: blockInfo, ldm: LocalDeclMap, IsLambdaConversionToBlock: isLambdaConv, BuildGlobalBlock: blockInfo.CanBeGlobal);
793 auto *blockFn = llvm::ConstantExpr::getPointerCast(C: InvokeFn, Ty: GenVoidPtrTy);
794
795 // If there is nothing to capture, we can emit this as a global block.
796 if (blockInfo.CanBeGlobal)
797 return CGM.getAddrOfGlobalBlockIfEmitted(BE: blockInfo.BlockExpression);
798
799 // Otherwise, we have to emit this as a local block.
800
801 RawAddress blockAddr = blockInfo.LocalAddress;
802 assert(blockAddr.isValid() && "block has no address!");
803
804 llvm::Constant *isa;
805 llvm::Constant *descriptor;
806 BlockFlags flags;
807 if (!IsOpenCL) {
808 // If the block is non-escaping, set field 'isa 'to NSConcreteGlobalBlock
809 // and set the BLOCK_IS_GLOBAL bit of field 'flags'. Copying a non-escaping
810 // block just returns the original block and releasing it is a no-op.
811 llvm::Constant *blockISA = blockInfo.NoEscape
812 ? CGM.getNSConcreteGlobalBlock()
813 : CGM.getNSConcreteStackBlock();
814 isa = blockISA;
815
816 // Compute the initial on-stack block flags.
817 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
818 flags = BLOCK_HAS_SIGNATURE;
819 if (blockInfo.HasCapturedVariableLayout)
820 flags |= BLOCK_HAS_EXTENDED_LAYOUT;
821 if (blockInfo.NeedsCopyDispose)
822 flags |= BLOCK_HAS_COPY_DISPOSE;
823 if (blockInfo.HasCXXObject)
824 flags |= BLOCK_HAS_CXX_OBJ;
825 if (blockInfo.UsesStret)
826 flags |= BLOCK_USE_STRET;
827 if (blockInfo.NoEscape)
828 flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;
829
830 // Build the block descriptor.
831 descriptor = buildBlockDescriptor(CGM, blockInfo);
832 }
833
834 auto projectField = [&](unsigned index, const Twine &name) -> Address {
835 return Builder.CreateStructGEP(Addr: blockAddr, Index: index, Name: name);
836 };
837 auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) {
838 Builder.CreateStore(Val: value, Addr: projectField(index, name));
839 };
840
841 // Initialize the block header.
842 {
843 // We assume all the header fields are densely packed.
844 unsigned index = 0;
845 CharUnits offset;
846 auto addHeaderField = [&](llvm::Value *value, CharUnits size,
847 const Twine &name) {
848 storeField(value, index, name);
849 offset += size;
850 index++;
851 };
852 auto addSignedHeaderField =
853 [&](llvm::Value *Value, const PointerAuthSchema &Schema,
854 GlobalDecl Decl, QualType Type, CharUnits Size, const Twine &Name) {
855 auto StorageAddress = projectField(index, Name);
856 if (Schema) {
857 auto AuthInfo = EmitPointerAuthInfo(
858 Schema, StorageAddress: StorageAddress.emitRawPointer(CGF&: *this), SchemaDecl: Decl, SchemaType: Type);
859 Value = EmitPointerAuthSign(Info: AuthInfo, Pointer: Value);
860 }
861 Builder.CreateStore(Val: Value, Addr: StorageAddress);
862 offset += Size;
863 index++;
864 };
865
866 if (!IsOpenCL) {
867 addSignedHeaderField(
868 isa, CGM.getCodeGenOpts().PointerAuth.ObjCIsaPointers, GlobalDecl(),
869 QualType(), getPointerSize(), "block.isa");
870 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
871 getIntSize(), "block.flags");
872 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: 0), getIntSize(),
873 "block.reserved");
874 } else {
875 addHeaderField(
876 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockSize.getQuantity()),
877 getIntSize(), "block.size");
878 addHeaderField(
879 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockAlign.getQuantity()),
880 getIntSize(), "block.align");
881 }
882
883 if (!IsOpenCL) {
884 llvm::Value *blockFnPtr =
885 llvm::ConstantExpr::getBitCast(C: InvokeFn, Ty: VoidPtrTy);
886 QualType type = blockInfo.getBlockExpr()
887 ->getType()
888 ->castAs<BlockPointerType>()
889 ->getPointeeType();
890 addSignedHeaderField(
891 blockFnPtr,
892 CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers,
893 GlobalDecl(), type, getPointerSize(), "block.invoke");
894
895 addSignedHeaderField(
896 descriptor, CGM.getCodeGenOpts().PointerAuth.BlockDescriptorPointers,
897 GlobalDecl(), type, getPointerSize(), "block.descriptor");
898 } else if (auto *Helper =
899 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
900 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
901 for (auto I : Helper->getCustomFieldValues(CGF&: *this, Info: blockInfo)) {
902 addHeaderField(
903 I.first,
904 CharUnits::fromQuantity(
905 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: I.first->getType())),
906 I.second);
907 }
908 } else
909 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
910 }
911
912 // Finally, capture all the values into the block.
913 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
914
915 // First, 'this'.
916 if (blockDecl->capturesCXXThis()) {
917 Address addr =
918 projectField(blockInfo.CXXThisIndex, "block.captured-this.addr");
919 Builder.CreateStore(Val: LoadCXXThis(), Addr: addr);
920 }
921
922 // Next, captured variables.
923 for (const auto &CI : blockDecl->captures()) {
924 const VarDecl *variable = CI.getVariable();
925 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
926
927 // Ignore constant captures.
928 if (capture.isConstant()) continue;
929
930 QualType type = capture.fieldType();
931
932 // This will be a [[type]]*, except that a byref entry will just be
933 // an i8**.
934 Address blockField = projectField(capture.getIndex(), "block.captured");
935
936 // Compute the address of the thing we're going to move into the
937 // block literal.
938 Address src = Address::invalid();
939
940 if (blockDecl->isConversionFromLambda()) {
941 // The lambda capture in a lambda's conversion-to-block-pointer is
942 // special; we'll simply emit it directly.
943 src = Address::invalid();
944 } else if (CI.isEscapingByref()) {
945 if (BlockInfo && CI.isNested()) {
946 // We need to use the capture from the enclosing block.
947 const CGBlockInfo::Capture &enclosingCapture =
948 BlockInfo->getCapture(var: variable);
949
950 // This is a [[type]]*, except that a byref entry will just be an i8**.
951 src = Builder.CreateStructGEP(Addr: LoadBlockStruct(),
952 Index: enclosingCapture.getIndex(),
953 Name: "block.capture.addr");
954 } else {
955 auto I = LocalDeclMap.find(Val: variable);
956 assert(I != LocalDeclMap.end());
957 src = I->second;
958 }
959 } else {
960 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
961 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
962 type.getNonReferenceType(), VK_LValue,
963 SourceLocation());
964 src = EmitDeclRefLValue(E: &declRef).getAddress();
965 };
966
967 // For byrefs, we just write the pointer to the byref struct into
968 // the block field. There's no need to chase the forwarding
969 // pointer at this point, since we're building something that will
970 // live a shorter life than the stack byref anyway.
971 if (CI.isEscapingByref()) {
972 // Get a void* that points to the byref struct.
973 llvm::Value *byrefPointer;
974 if (CI.isNested())
975 byrefPointer = Builder.CreateLoad(Addr: src, Name: "byref.capture");
976 else
977 byrefPointer = src.emitRawPointer(CGF&: *this);
978
979 // Write that void* into the capture field.
980 Builder.CreateStore(Val: byrefPointer, Addr: blockField);
981
982 // If we have a copy constructor, evaluate that into the block field.
983 } else if (const Expr *copyExpr = CI.getCopyExpr()) {
984 if (blockDecl->isConversionFromLambda()) {
985 // If we have a lambda conversion, emit the expression
986 // directly into the block instead.
987 AggValueSlot Slot =
988 AggValueSlot::forAddr(addr: blockField, quals: Qualifiers(),
989 isDestructed: AggValueSlot::IsDestructed,
990 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
991 isAliased: AggValueSlot::IsNotAliased,
992 mayOverlap: AggValueSlot::DoesNotOverlap);
993 EmitAggExpr(E: copyExpr, AS: Slot);
994 } else {
995 EmitSynthesizedCXXCopyCtor(Dest: blockField, Src: src, Exp: copyExpr);
996 }
997
998 // If it's a reference variable, copy the reference into the block field.
999 } else if (type->getAs<ReferenceType>()) {
1000 Builder.CreateStore(Val: src.emitRawPointer(CGF&: *this), Addr: blockField);
1001
1002 // If type is const-qualified, copy the value into the block field.
1003 } else if (type.isConstQualified() &&
1004 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1005 CGM.getCodeGenOpts().OptimizationLevel != 0) {
1006 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "captured");
1007 Builder.CreateStore(Val: value, Addr: blockField);
1008
1009 // If this is an ARC __strong block-pointer variable, don't do a
1010 // block copy.
1011 //
1012 // TODO: this can be generalized into the normal initialization logic:
1013 // we should never need to do a block-copy when initializing a local
1014 // variable, because the local variable's lifetime should be strictly
1015 // contained within the stack block's.
1016 } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1017 type->isBlockPointerType()) {
1018 // Load the block and do a simple retain.
1019 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "block.captured_block");
1020 value = EmitARCRetainNonBlock(value);
1021
1022 // Do a primitive store to the block field.
1023 Builder.CreateStore(Val: value, Addr: blockField);
1024
1025 // Otherwise, fake up a POD copy into the block field.
1026 } else {
1027 // Fake up a new variable so that EmitScalarInit doesn't think
1028 // we're referring to the variable in its own initializer.
1029 auto *BlockFieldPseudoVar = ImplicitParamDecl::Create(
1030 C&: getContext(), T: type, ParamKind: ImplicitParamKind::Other);
1031
1032 // We use one of these or the other depending on whether the
1033 // reference is nested.
1034 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
1035 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
1036 type, VK_LValue, SourceLocation());
1037
1038 ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
1039 &declRef, VK_PRValue, FPOptionsOverride());
1040 // FIXME: Pass a specific location for the expr init so that the store is
1041 // attributed to a reasonable location - otherwise it may be attributed to
1042 // locations of subexpressions in the initialization.
1043 EmitExprAsInit(init: &l2r, D: BlockFieldPseudoVar,
1044 lvalue: MakeAddrLValue(Addr: blockField, T: type, Source: AlignmentSource::Decl),
1045 /*captured by init*/ capturedByInit: false);
1046 }
1047
1048 // Push a cleanup for the capture if necessary.
1049 if (!blockInfo.NoEscape && !blockInfo.NeedsCopyDispose)
1050 continue;
1051
1052 // Ignore __block captures; there's nothing special in the on-stack block
1053 // that we need to do for them.
1054 if (CI.isByRef())
1055 continue;
1056
1057 // Ignore objects that aren't destructed.
1058 QualType::DestructionKind dtorKind = type.isDestructedType();
1059 if (dtorKind == QualType::DK_none)
1060 continue;
1061
1062 CodeGenFunction::Destroyer *destroyer;
1063
1064 // Block captures count as local values and have imprecise semantics.
1065 // They also can't be arrays, so need to worry about that.
1066 //
1067 // For const-qualified captures, emit clang.arc.use to ensure the captured
1068 // object doesn't get released while we are still depending on its validity
1069 // within the block.
1070 if (type.isConstQualified() &&
1071 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1072 CGM.getCodeGenOpts().OptimizationLevel != 0) {
1073 assert(CGM.getLangOpts().ObjCAutoRefCount &&
1074 "expected ObjC ARC to be enabled");
1075 destroyer = emitARCIntrinsicUse;
1076 } else if (dtorKind == QualType::DK_objc_strong_lifetime) {
1077 destroyer = destroyARCStrongImprecise;
1078 } else {
1079 destroyer = getDestroyer(destructionKind: dtorKind);
1080 }
1081
1082 CleanupKind cleanupKind = NormalCleanup;
1083 bool useArrayEHCleanup = needsEHCleanup(kind: dtorKind);
1084 if (useArrayEHCleanup)
1085 cleanupKind = NormalAndEHCleanup;
1086
1087 // Extend the lifetime of the capture to the end of the scope enclosing the
1088 // block expression except when the block decl is in the list of RetExpr's
1089 // cleanup objects, in which case its lifetime ends after the full
1090 // expression.
1091 auto IsBlockDeclInRetExpr = [&]() {
1092 auto *EWC = llvm::dyn_cast_or_null<ExprWithCleanups>(Val: RetExpr);
1093 if (EWC)
1094 for (auto &C : EWC->getObjects())
1095 if (auto *BD = dyn_cast<BlockDecl *>(Val: C))
1096 if (BD == blockDecl)
1097 return true;
1098 return false;
1099 };
1100
1101 if (IsBlockDeclInRetExpr())
1102 pushDestroy(kind: cleanupKind, addr: blockField, type, destroyer, useEHCleanupForArray: useArrayEHCleanup);
1103 else
1104 pushLifetimeExtendedDestroy(kind: cleanupKind, addr: blockField, type, destroyer,
1105 useEHCleanupForArray: useArrayEHCleanup);
1106 }
1107
1108 // Cast to the converted block-pointer type, which happens (somewhat
1109 // unfortunately) to be a pointer to function type.
1110 llvm::Value *result = Builder.CreatePointerCast(
1111 V: blockAddr.getPointer(), DestTy: ConvertType(T: blockInfo.getBlockExpr()->getType()));
1112
1113 if (IsOpenCL) {
1114 CGM.getOpenCLRuntime().recordBlockInfo(E: blockInfo.BlockExpression, InvokeF: InvokeFn,
1115 Block: result, BlockTy: blockInfo.StructureType);
1116 }
1117
1118 return result;
1119}
1120
1121
1122llvm::Type *CodeGenModule::getBlockDescriptorType() {
1123 if (BlockDescriptorType)
1124 return BlockDescriptorType;
1125
1126 unsigned AddrSpace = 0;
1127 if (getLangOpts().OpenCL)
1128 AddrSpace = getContext().getTargetAddressSpace(AS: LangAS::opencl_constant);
1129 BlockDescriptorType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: AddrSpace);
1130 return BlockDescriptorType;
1131}
1132
1133llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
1134 if (GenericBlockLiteralType)
1135 return GenericBlockLiteralType;
1136
1137 llvm::Type *BlockDescPtrTy = getBlockDescriptorType();
1138
1139 if (getLangOpts().OpenCL) {
1140 // struct __opencl_block_literal_generic {
1141 // int __size;
1142 // int __align;
1143 // __generic void *__invoke;
1144 // /* custom fields */
1145 // };
1146 SmallVector<llvm::Type *, 8> StructFields(
1147 {IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});
1148 if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1149 llvm::append_range(C&: StructFields, R: Helper->getCustomFieldTypes());
1150 }
1151 GenericBlockLiteralType = llvm::StructType::create(
1152 Elements: StructFields, Name: "struct.__opencl_block_literal_generic");
1153 } else {
1154 // struct __block_literal_generic {
1155 // void *__isa;
1156 // int __flags;
1157 // int __reserved;
1158 // void (*__invoke)(void *);
1159 // struct __block_descriptor *__descriptor;
1160 // };
1161 GenericBlockLiteralType =
1162 llvm::StructType::create(Name: "struct.__block_literal_generic", elt1: VoidPtrTy,
1163 elts: IntTy, elts: IntTy, elts: VoidPtrTy, elts: BlockDescPtrTy);
1164 }
1165
1166 return GenericBlockLiteralType;
1167}
1168
1169RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
1170 ReturnValueSlot ReturnValue,
1171 llvm::CallBase **CallOrInvoke) {
1172 const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();
1173 llvm::Value *BlockPtr = EmitScalarExpr(E: E->getCallee());
1174 llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();
1175 llvm::Value *Func = nullptr;
1176 QualType FnType = BPT->getPointeeType();
1177 ASTContext &Ctx = getContext();
1178 CallArgList Args;
1179
1180 llvm::Value *FuncPtr = nullptr;
1181
1182 if (getLangOpts().OpenCL) {
1183 // For OpenCL, BlockPtr is already casted to generic block literal.
1184
1185 // First argument of a block call is a generic block literal casted to
1186 // generic void pointer, i.e. i8 addrspace(4)*
1187 llvm::Type *GenericVoidPtrTy =
1188 CGM.getOpenCLRuntime().getGenericVoidPointerType();
1189 llvm::Value *BlockDescriptor = Builder.CreatePointerCast(
1190 V: BlockPtr, DestTy: GenericVoidPtrTy);
1191 QualType VoidPtrQualTy = Ctx.getPointerType(
1192 T: Ctx.getAddrSpaceQualType(T: Ctx.VoidTy, AddressSpace: LangAS::opencl_generic));
1193 Args.add(rvalue: RValue::get(V: BlockDescriptor), type: VoidPtrQualTy);
1194 // And the rest of the arguments.
1195 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1196
1197 // We *can* call the block directly unless it is a function argument.
1198 if (!isa<ParmVarDecl>(Val: E->getCalleeDecl()))
1199 Func = CGM.getOpenCLRuntime().getInvokeFunction(E: E->getCallee());
1200 else {
1201 FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 2);
1202 Func = Builder.CreateAlignedLoad(Ty: GenericVoidPtrTy, Addr: FuncPtr,
1203 Align: getPointerAlign());
1204 }
1205 } else {
1206 // Bitcast the block literal to a generic block literal.
1207 BlockPtr =
1208 Builder.CreatePointerCast(V: BlockPtr, DestTy: DefaultPtrTy, Name: "block.literal");
1209 // Get pointer to the block invoke function
1210 FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 3);
1211
1212 // First argument is a block literal casted to a void pointer
1213 BlockPtr = Builder.CreatePointerCast(V: BlockPtr, DestTy: VoidPtrTy);
1214 Args.add(rvalue: RValue::get(V: BlockPtr), type: Ctx.VoidPtrTy);
1215 // And the rest of the arguments.
1216 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1217
1218 // Load the function.
1219 Func = Builder.CreateAlignedLoad(Ty: VoidPtrTy, Addr: FuncPtr, Align: getPointerAlign());
1220 }
1221
1222 const FunctionType *FuncTy = FnType->castAs<FunctionType>();
1223 const CGFunctionInfo &FnInfo =
1224 CGM.getTypes().arrangeBlockFunctionCall(args: Args, type: FuncTy);
1225
1226 // Prepare the callee.
1227 CGPointerAuthInfo PointerAuth;
1228 if (auto &AuthSchema =
1229 CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers) {
1230 assert(FuncPtr != nullptr && "Missing function pointer for AuthInfo");
1231 PointerAuth =
1232 EmitPointerAuthInfo(Schema: AuthSchema, StorageAddress: FuncPtr, SchemaDecl: GlobalDecl(), SchemaType: FnType);
1233 }
1234
1235 CGCallee Callee(CGCalleeInfo(), Func, PointerAuth);
1236
1237 // And call the block.
1238 return EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke);
1239}
1240
1241Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {
1242 assert(BlockInfo && "evaluating block ref without block information?");
1243 const CGBlockInfo::Capture &capture = BlockInfo->getCapture(var: variable);
1244
1245 // Handle constant captures.
1246 if (capture.isConstant()) return LocalDeclMap.find(Val: variable)->second;
1247
1248 Address addr = Builder.CreateStructGEP(Addr: LoadBlockStruct(), Index: capture.getIndex(),
1249 Name: "block.capture.addr");
1250
1251 if (variable->isEscapingByref()) {
1252 // addr should be a void** right now. Load, then cast the result
1253 // to byref*.
1254
1255 auto &byrefInfo = getBlockByrefInfo(var: variable);
1256 addr = Address(Builder.CreateLoad(Addr: addr), byrefInfo.Type,
1257 byrefInfo.ByrefAlignment);
1258
1259 addr = emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, /*follow*/ followForward: true,
1260 name: variable->getName());
1261 }
1262
1263 assert((!variable->isNonEscapingByref() ||
1264 capture.fieldType()->isReferenceType()) &&
1265 "the capture field of a non-escaping variable should have a "
1266 "reference type");
1267 if (capture.fieldType()->isReferenceType())
1268 addr = EmitLoadOfReference(RefLVal: MakeAddrLValue(Addr: addr, T: capture.fieldType()));
1269
1270 return addr;
1271}
1272
1273void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE,
1274 llvm::Constant *Addr) {
1275 bool Ok = EmittedGlobalBlocks.insert(KV: std::make_pair(x&: BE, y&: Addr)).second;
1276 (void)Ok;
1277 assert(Ok && "Trying to replace an already-existing global block!");
1278}
1279
1280llvm::Constant *
1281CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE,
1282 StringRef Name) {
1283 if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE))
1284 return Block;
1285
1286 CGBlockInfo blockInfo(BE->getBlockDecl(), Name);
1287 blockInfo.BlockExpression = BE;
1288
1289 // Compute information about the layout, etc., of this block.
1290 computeBlockInfo(CGM&: *this, CGF: nullptr, info&: blockInfo);
1291
1292 // Using that metadata, generate the actual block function.
1293 {
1294 CodeGenFunction::DeclMapTy LocalDeclMap;
1295 CodeGenFunction(*this).GenerateBlockFunction(
1296 GD: GlobalDecl(), Info: blockInfo, ldm: LocalDeclMap,
1297 /*IsLambdaConversionToBlock*/ false, /*BuildGlobalBlock*/ true);
1298 }
1299
1300 return getAddrOfGlobalBlockIfEmitted(BE);
1301}
1302
1303static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
1304 const CGBlockInfo &blockInfo,
1305 llvm::Constant *blockFn) {
1306 assert(blockInfo.CanBeGlobal);
1307 // Callers should detect this case on their own: calling this function
1308 // generally requires computing layout information, which is a waste of time
1309 // if we've already emitted this block.
1310 assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) &&
1311 "Refusing to re-emit a global block.");
1312
1313 // Generate the constants for the block literal initializer.
1314 ConstantInitBuilder builder(CGM);
1315 auto fields = builder.beginStruct();
1316
1317 bool IsOpenCL = CGM.getLangOpts().OpenCL;
1318 bool IsWindows = CGM.getTarget().getTriple().isOSWindows();
1319 auto &CGOPointerAuth = CGM.getCodeGenOpts().PointerAuth;
1320 if (!IsOpenCL) {
1321 // isa
1322 if (IsWindows)
1323 fields.addNullPointer(ptrTy: CGM.Int8PtrPtrTy);
1324 else
1325 fields.addSignedPointer(Pointer: CGM.getNSConcreteGlobalBlock(),
1326 Schema: CGOPointerAuth.ObjCIsaPointers, CalleeDecl: GlobalDecl(),
1327 CalleeType: QualType());
1328
1329 // __flags
1330 BlockFlags flags = BLOCK_IS_GLOBAL;
1331 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
1332 flags |= BLOCK_HAS_SIGNATURE;
1333 if (blockInfo.UsesStret)
1334 flags |= BLOCK_USE_STRET;
1335
1336 fields.addInt(intTy: CGM.IntTy, value: flags.getBitMask());
1337
1338 // Reserved
1339 fields.addInt(intTy: CGM.IntTy, value: 0);
1340 } else {
1341 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockSize.getQuantity());
1342 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockAlign.getQuantity());
1343 }
1344
1345 // Function
1346 if (auto &Schema = CGOPointerAuth.BlockInvocationFunctionPointers) {
1347 QualType FnType = blockInfo.getBlockExpr()
1348 ->getType()
1349 ->castAs<BlockPointerType>()
1350 ->getPointeeType();
1351 fields.addSignedPointer(Pointer: blockFn, Schema, CalleeDecl: GlobalDecl(), CalleeType: FnType);
1352 } else
1353 fields.add(value: blockFn);
1354
1355 if (!IsOpenCL) {
1356 // Descriptor
1357 llvm::Constant *Descriptor = buildBlockDescriptor(CGM, blockInfo);
1358 fields.addSignedPointer(Pointer: Descriptor, Schema: CGOPointerAuth.BlockDescriptorPointers,
1359 CalleeDecl: GlobalDecl(), CalleeType: QualType());
1360 } else if (auto *Helper =
1361 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1362 for (auto *I : Helper->getCustomFieldValues(CGM, Info: blockInfo)) {
1363 fields.add(value: I);
1364 }
1365 }
1366
1367 unsigned AddrSpace = 0;
1368 if (CGM.getContext().getLangOpts().OpenCL)
1369 AddrSpace = CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
1370
1371 llvm::GlobalVariable *literal = fields.finishAndCreateGlobal(
1372 args: "__block_literal_global", args: blockInfo.BlockAlign,
1373 /*constant*/ args: !IsWindows, args: llvm::GlobalVariable::InternalLinkage, args&: AddrSpace);
1374
1375 literal->addAttribute(Kind: "objc_arc_inert");
1376
1377 // Windows does not allow globals to be initialised to point to globals in
1378 // different DLLs. Any such variables must run code to initialise them.
1379 if (IsWindows) {
1380 auto *Init = llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy,
1381 isVarArg: {}), Linkage: llvm::GlobalValue::InternalLinkage, N: ".block_isa_init",
1382 M: &CGM.getModule());
1383 llvm::IRBuilder<> b(llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "entry",
1384 Parent: Init));
1385 b.CreateAlignedStore(Val: CGM.getNSConcreteGlobalBlock(),
1386 Ptr: b.CreateStructGEP(Ty: literal->getValueType(), Ptr: literal, Idx: 0),
1387 Align: CGM.getPointerAlign().getAsAlign());
1388 b.CreateRetVoid();
1389 // We can't use the normal LLVM global initialisation array, because we
1390 // need to specify that this runs early in library initialisation.
1391 auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
1392 /*isConstant*/true, llvm::GlobalValue::InternalLinkage,
1393 Init, ".block_isa_init_ptr");
1394 InitVar->setSection(".CRT$XCLa");
1395 CGM.addUsedGlobal(GV: InitVar);
1396 }
1397
1398 // Return a constant of the appropriately-casted type.
1399 llvm::Type *RequiredType =
1400 CGM.getTypes().ConvertType(T: blockInfo.getBlockExpr()->getType());
1401 llvm::Constant *Result =
1402 llvm::ConstantExpr::getPointerCast(C: literal, Ty: RequiredType);
1403 CGM.setAddrOfGlobalBlock(BE: blockInfo.BlockExpression, Addr: Result);
1404 if (CGM.getContext().getLangOpts().OpenCL)
1405 CGM.getOpenCLRuntime().recordBlockInfo(
1406 E: blockInfo.BlockExpression,
1407 InvokeF: cast<llvm::Function>(Val: blockFn->stripPointerCasts()), Block: Result,
1408 BlockTy: literal->getValueType());
1409 return Result;
1410}
1411
1412void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
1413 unsigned argNum,
1414 llvm::Value *arg) {
1415 assert(BlockInfo && "not emitting prologue of block invocation function?!");
1416
1417 // Allocate a stack slot like for any local variable to guarantee optimal
1418 // debug info at -O0. The mem2reg pass will eliminate it when optimizing.
1419 RawAddress alloc =
1420 CreateMemTempWithoutCast(T: D->getType(), Name: D->getName() + ".addr");
1421 Builder.CreateStore(Val: arg, Addr: alloc);
1422 if (CGDebugInfo *DI = getDebugInfo()) {
1423 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1424 DI->setLocation(D->getLocation());
1425 DI->EmitDeclareOfBlockLiteralArgVariable(
1426 block: *BlockInfo, Name: D->getName(), ArgNo: argNum,
1427 LocalAddr: cast<llvm::AllocaInst>(Val: alloc.getPointer()->stripPointerCasts()),
1428 Builder);
1429 }
1430 }
1431
1432 SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc();
1433 ApplyDebugLocation Scope(*this, StartLoc);
1434
1435 // Instead of messing around with LocalDeclMap, just set the value
1436 // directly as BlockPointer.
1437 BlockPointer = Builder.CreatePointerCast(
1438 V: arg,
1439 DestTy: llvm::PointerType::get(
1440 C&: getLLVMContext(),
1441 AddressSpace: getContext().getLangOpts().OpenCL
1442 ? getContext().getTargetAddressSpace(AS: LangAS::opencl_generic)
1443 : 0),
1444 Name: "block");
1445}
1446
1447Address CodeGenFunction::LoadBlockStruct() {
1448 assert(BlockInfo && "not in a block invocation function!");
1449 assert(BlockPointer && "no block pointer set!");
1450 return Address(BlockPointer, BlockInfo->StructureType, BlockInfo->BlockAlign);
1451}
1452
1453llvm::Function *CodeGenFunction::GenerateBlockFunction(
1454 GlobalDecl GD, const CGBlockInfo &blockInfo, const DeclMapTy &ldm,
1455 bool IsLambdaConversionToBlock, bool BuildGlobalBlock) {
1456 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
1457
1458 CurGD = GD;
1459
1460 CurEHLocation = blockInfo.getBlockExpr()->getEndLoc();
1461
1462 BlockInfo = &blockInfo;
1463
1464 // Arrange for local static and local extern declarations to appear
1465 // to be local to this function as well, in case they're directly
1466 // referenced in a block.
1467 for (const auto &KV : ldm) {
1468 const auto *var = dyn_cast<VarDecl>(Val: KV.first);
1469 if (var && !var->hasLocalStorage())
1470 setAddrOfLocalVar(VD: var, Addr: KV.second);
1471 }
1472
1473 // Begin building the function declaration.
1474
1475 // Build the argument list.
1476 FunctionArgList args;
1477
1478 // The first argument is the block pointer. Just take it as a void*
1479 // and cast it later.
1480 QualType selfTy = getContext().VoidPtrTy;
1481
1482 // For OpenCL passed block pointer can be private AS local variable or
1483 // global AS program scope variable (for the case with and without captures).
1484 // Generic AS is used therefore to be able to accommodate both private and
1485 // generic AS in one implementation.
1486 if (getLangOpts().OpenCL)
1487 selfTy = getContext().getPointerType(T: getContext().getAddrSpaceQualType(
1488 T: getContext().VoidTy, AddressSpace: LangAS::opencl_generic));
1489
1490 const IdentifierInfo *II = &CGM.getContext().Idents.get(Name: ".block_descriptor");
1491
1492 auto *SelfDecl = ImplicitParamDecl::Create(
1493 C&: getContext(), DC: const_cast<BlockDecl *>(blockDecl), IdLoc: SourceLocation(), Id: II,
1494 T: selfTy, ParamKind: ImplicitParamKind::ObjCSelf);
1495 args.push_back(Elt: SelfDecl);
1496
1497 // Now add the rest of the parameters.
1498 args.append(in_start: blockDecl->param_begin(), in_end: blockDecl->param_end());
1499
1500 // Create the function declaration.
1501 const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();
1502 const CGFunctionInfo &fnInfo =
1503 CGM.getTypes().arrangeBlockFunctionDeclaration(type: fnType, args);
1504 if (CGM.ReturnSlotInterferesWithArgs(FI: fnInfo))
1505 blockInfo.UsesStret = true;
1506
1507 llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(Info: fnInfo);
1508
1509 StringRef name = CGM.getBlockMangledName(GD, BD: blockDecl);
1510 llvm::Function *fn = llvm::Function::Create(
1511 Ty: fnLLVMType, Linkage: llvm::GlobalValue::InternalLinkage, N: name, M: &CGM.getModule());
1512 CGM.SetInternalFunctionAttributes(GD: blockDecl, F: fn, FI: fnInfo);
1513
1514 if (BuildGlobalBlock) {
1515 auto GenVoidPtrTy = getContext().getLangOpts().OpenCL
1516 ? CGM.getOpenCLRuntime().getGenericVoidPointerType()
1517 : VoidPtrTy;
1518 buildGlobalBlock(CGM, blockInfo,
1519 blockFn: llvm::ConstantExpr::getPointerCast(C: fn, Ty: GenVoidPtrTy));
1520 }
1521
1522 // Begin generating the function.
1523 StartFunction(GD: blockDecl, RetTy: fnType->getReturnType(), Fn: fn, FnInfo: fnInfo, Args: args,
1524 Loc: blockDecl->getLocation(),
1525 StartLoc: blockInfo.getBlockExpr()->getBody()->getBeginLoc());
1526
1527 // Okay. Undo some of what StartFunction did.
1528
1529 // At -O0 we generate an explicit alloca for the BlockPointer, so the RA
1530 // won't delete the dbg.declare intrinsics for captured variables.
1531 llvm::Value *BlockPointerDbgLoc = BlockPointer;
1532 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
1533 // Allocate a stack slot for it, so we can point the debugger to it
1534 Address Alloca = CreateTempAlloca(Ty: BlockPointer->getType(),
1535 align: getPointerAlign(),
1536 Name: "block.addr");
1537 // Set the DebugLocation to empty, so the store is recognized as a
1538 // frame setup instruction by llvm::DwarfDebug::beginFunction().
1539 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1540 Builder.CreateStore(Val: BlockPointer, Addr: Alloca);
1541 BlockPointerDbgLoc = Alloca.emitRawPointer(CGF&: *this);
1542 }
1543
1544 // If we have a C++ 'this' reference, go ahead and force it into
1545 // existence now.
1546 if (blockDecl->capturesCXXThis()) {
1547 Address addr = Builder.CreateStructGEP(
1548 Addr: LoadBlockStruct(), Index: blockInfo.CXXThisIndex, Name: "block.captured-this");
1549 CXXThisValue = Builder.CreateLoad(Addr: addr, Name: "this");
1550 }
1551
1552 // Also force all the constant captures.
1553 for (const auto &CI : blockDecl->captures()) {
1554 const VarDecl *variable = CI.getVariable();
1555 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1556 if (!capture.isConstant()) continue;
1557
1558 CharUnits align = getContext().getDeclAlign(D: variable);
1559 Address alloca = CreateMemTempWithoutCast(T: variable->getType(), Align: align,
1560 Name: "block.captured-const");
1561
1562 Builder.CreateStore(Val: capture.getConstant(), Addr: alloca);
1563
1564 setAddrOfLocalVar(VD: variable, Addr: alloca);
1565 }
1566
1567 // Save a spot to insert the debug information for all the DeclRefExprs.
1568 llvm::BasicBlock *entry = Builder.GetInsertBlock();
1569 llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();
1570 --entry_ptr;
1571
1572 if (IsLambdaConversionToBlock)
1573 EmitLambdaBlockInvokeBody();
1574 else {
1575 PGO->assignRegionCounters(GD: GlobalDecl(blockDecl), Fn: fn);
1576 incrementProfileCounter(S: blockDecl->getBody());
1577 maybeCreateMCDCCondBitmap();
1578 EmitStmt(S: blockDecl->getBody());
1579 }
1580
1581 // Remember where we were...
1582 llvm::BasicBlock *resume = Builder.GetInsertBlock();
1583
1584 // Go back to the entry.
1585 if (entry_ptr->getNextNode())
1586 entry_ptr = entry_ptr->getNextNode()->getIterator();
1587 else
1588 entry_ptr = entry->end();
1589 Builder.SetInsertPoint(entry_ptr);
1590
1591 // Emit debug information for all the DeclRefExprs.
1592 // FIXME: also for 'this'
1593 if (CGDebugInfo *DI = getDebugInfo()) {
1594 for (const auto &CI : blockDecl->captures()) {
1595 const VarDecl *variable = CI.getVariable();
1596 DI->EmitLocation(Builder, Loc: variable->getLocation());
1597
1598 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1599 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1600 if (capture.isConstant()) {
1601 auto addr = LocalDeclMap.find(Val: variable)->second;
1602 (void)DI->EmitDeclareOfAutoVariable(
1603 Decl: variable, AI: addr.emitRawPointer(CGF&: *this), Builder);
1604 continue;
1605 }
1606
1607 DI->EmitDeclareOfBlockDeclRefVariable(
1608 variable, storage: BlockPointerDbgLoc, Builder, blockInfo,
1609 InsertPoint: entry_ptr == entry->end() ? nullptr : &*entry_ptr);
1610 }
1611 }
1612 // Recover location if it was changed in the above loop.
1613 DI->EmitLocation(Builder,
1614 Loc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1615 }
1616
1617 // And resume where we left off.
1618 if (resume == nullptr)
1619 Builder.ClearInsertionPoint();
1620 else
1621 Builder.SetInsertPoint(resume);
1622
1623 FinishFunction(EndLoc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1624
1625 return fn;
1626}
1627
1628static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
1629computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
1630 const LangOptions &LangOpts) {
1631 if (CI.getCopyExpr()) {
1632 assert(!CI.isByRef());
1633 // don't bother computing flags
1634 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
1635 }
1636 BlockFieldFlags Flags;
1637 if (CI.isEscapingByref()) {
1638 Flags = BLOCK_FIELD_IS_BYREF;
1639 if (T.isObjCGCWeak())
1640 Flags |= BLOCK_FIELD_IS_WEAK;
1641 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1642 }
1643
1644 if (T.hasAddressDiscriminatedPointerAuth())
1645 return std::make_pair(
1646 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth, y&: Flags);
1647
1648 Flags = BLOCK_FIELD_IS_OBJECT;
1649 bool isBlockPointer = T->isBlockPointerType();
1650 if (isBlockPointer)
1651 Flags = BLOCK_FIELD_IS_BLOCK;
1652
1653 switch (T.isNonTrivialToPrimitiveCopy()) {
1654 case QualType::PCK_Struct:
1655 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
1656 y: BlockFieldFlags());
1657 case QualType::PCK_ARCWeak:
1658 // We need to register __weak direct captures with the runtime.
1659 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak, y&: Flags);
1660 case QualType::PCK_ARCStrong:
1661 // We need to retain the copied value for __strong direct captures.
1662 // If it's a block pointer, we have to copy the block and assign that to
1663 // the destination pointer, so we might as well use _Block_object_assign.
1664 // Otherwise we can avoid that.
1665 return std::make_pair(x: !isBlockPointer ? BlockCaptureEntityKind::ARCStrong
1666 : BlockCaptureEntityKind::BlockObject,
1667 y&: Flags);
1668 case QualType::PCK_PtrAuth:
1669 return std::make_pair(
1670 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth,
1671 y: BlockFieldFlags());
1672 case QualType::PCK_Trivial:
1673 case QualType::PCK_VolatileTrivial: {
1674 if (!T->isObjCRetainableType())
1675 // For all other types, the memcpy is fine.
1676 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1677
1678 // Honor the inert __unsafe_unretained qualifier, which doesn't actually
1679 // make it into the type system.
1680 if (T->isObjCInertUnsafeUnretainedType())
1681 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1682
1683 // Special rules for ARC captures:
1684 Qualifiers QS = T.getQualifiers();
1685
1686 // Non-ARC captures of retainable pointers are strong and
1687 // therefore require a call to _Block_object_assign.
1688 if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount)
1689 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1690
1691 // Otherwise the memcpy is fine.
1692 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1693 }
1694 }
1695 llvm_unreachable("after exhaustive PrimitiveCopyKind switch");
1696}
1697
1698namespace {
1699/// Release a __block variable.
1700struct CallBlockRelease final : EHScopeStack::Cleanup {
1701 Address Addr;
1702 BlockFieldFlags FieldFlags;
1703 bool LoadBlockVarAddr, CanThrow;
1704
1705 CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,
1706 bool CT)
1707 : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),
1708 CanThrow(CT) {}
1709
1710 void Emit(CodeGenFunction &CGF, Flags flags) override {
1711 llvm::Value *BlockVarAddr;
1712 if (LoadBlockVarAddr) {
1713 BlockVarAddr = CGF.Builder.CreateLoad(Addr);
1714 } else {
1715 BlockVarAddr = Addr.emitRawPointer(CGF);
1716 }
1717
1718 CGF.BuildBlockRelease(DeclPtr: BlockVarAddr, flags: FieldFlags, CanThrow);
1719 }
1720};
1721} // end anonymous namespace
1722
1723/// Check if \p T is a C++ class that has a destructor that can throw.
1724bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {
1725 if (const auto *RD = T->getAsCXXRecordDecl())
1726 if (const CXXDestructorDecl *DD = RD->getDestructor())
1727 return DD->getType()->castAs<FunctionProtoType>()->canThrow();
1728 return false;
1729}
1730
1731// Return a string that has the information about a capture.
1732static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
1733 CaptureStrKind StrKind,
1734 CharUnits BlockAlignment,
1735 CodeGenModule &CGM) {
1736 std::string Str;
1737 ASTContext &Ctx = CGM.getContext();
1738 const BlockDecl::Capture &CI = *Cap.Cap;
1739 QualType CaptureTy = CI.getVariable()->getType();
1740
1741 BlockCaptureEntityKind Kind;
1742 BlockFieldFlags Flags;
1743
1744 // CaptureStrKind::Merged should be passed only when the operations and the
1745 // flags are the same for copy and dispose.
1746 assert((StrKind != CaptureStrKind::Merged ||
1747 (Cap.CopyKind == Cap.DisposeKind &&
1748 Cap.CopyFlags == Cap.DisposeFlags)) &&
1749 "different operations and flags");
1750
1751 if (StrKind == CaptureStrKind::DisposeHelper) {
1752 Kind = Cap.DisposeKind;
1753 Flags = Cap.DisposeFlags;
1754 } else {
1755 Kind = Cap.CopyKind;
1756 Flags = Cap.CopyFlags;
1757 }
1758
1759 switch (Kind) {
1760 case BlockCaptureEntityKind::CXXRecord: {
1761 Str += "c";
1762 SmallString<256> TyStr;
1763 llvm::raw_svector_ostream Out(TyStr);
1764 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: CaptureTy, Out);
1765 Str += llvm::to_string(Value: TyStr.size()) + TyStr.c_str();
1766 break;
1767 }
1768 case BlockCaptureEntityKind::ARCWeak:
1769 Str += "w";
1770 break;
1771 case BlockCaptureEntityKind::ARCStrong:
1772 Str += "s";
1773 break;
1774 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1775 auto PtrAuth = CaptureTy.getPointerAuth();
1776 assert(PtrAuth && PtrAuth.isAddressDiscriminated());
1777 Str += "p" + llvm::to_string(Value: PtrAuth.getKey()) + "d" +
1778 llvm::to_string(Value: PtrAuth.getExtraDiscriminator());
1779 break;
1780 }
1781 case BlockCaptureEntityKind::BlockObject: {
1782 const VarDecl *Var = CI.getVariable();
1783 unsigned F = Flags.getBitMask();
1784 if (F & BLOCK_FIELD_IS_BYREF) {
1785 Str += "r";
1786 if (F & BLOCK_FIELD_IS_WEAK)
1787 Str += "w";
1788 else {
1789 // If CaptureStrKind::Merged is passed, check both the copy expression
1790 // and the destructor.
1791 if (StrKind != CaptureStrKind::DisposeHelper) {
1792 if (Ctx.getBlockVarCopyInit(VD: Var).canThrow())
1793 Str += "c";
1794 }
1795 if (StrKind != CaptureStrKind::CopyHelper) {
1796 if (CodeGenFunction::cxxDestructorCanThrow(T: CaptureTy))
1797 Str += "d";
1798 }
1799 }
1800 } else {
1801 assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");
1802 if (F == BLOCK_FIELD_IS_BLOCK)
1803 Str += "b";
1804 else
1805 Str += "o";
1806 }
1807 break;
1808 }
1809 case BlockCaptureEntityKind::NonTrivialCStruct: {
1810 bool IsVolatile = CaptureTy.isVolatileQualified();
1811 CharUnits Alignment = BlockAlignment.alignmentAtOffset(offset: Cap.getOffset());
1812
1813 Str += "n";
1814 std::string FuncStr;
1815 if (StrKind == CaptureStrKind::DisposeHelper)
1816 FuncStr = CodeGenFunction::getNonTrivialDestructorStr(
1817 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1818 else
1819 // If CaptureStrKind::Merged is passed, use the copy constructor string.
1820 // It has all the information that the destructor string has.
1821 FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr(
1822 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1823 // The underscore is necessary here because non-trivial copy constructor
1824 // and destructor strings can start with a number.
1825 Str += llvm::to_string(Value: FuncStr.size()) + "_" + FuncStr;
1826 break;
1827 }
1828 case BlockCaptureEntityKind::None:
1829 break;
1830 }
1831
1832 return Str;
1833}
1834
1835static std::string getCopyDestroyHelperFuncName(
1836 const SmallVectorImpl<CGBlockInfo::Capture> &Captures,
1837 CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) {
1838 assert((StrKind == CaptureStrKind::CopyHelper ||
1839 StrKind == CaptureStrKind::DisposeHelper) &&
1840 "unexpected CaptureStrKind");
1841 std::string Name = StrKind == CaptureStrKind::CopyHelper
1842 ? "__copy_helper_block_"
1843 : "__destroy_helper_block_";
1844 if (CGM.getLangOpts().Exceptions)
1845 Name += "e";
1846 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
1847 Name += "a";
1848 Name += llvm::to_string(Value: BlockAlignment.getQuantity()) + "_";
1849
1850 for (auto &Cap : Captures) {
1851 if (Cap.isConstantOrTrivial())
1852 continue;
1853 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
1854 Name += getBlockCaptureStr(Cap, StrKind, BlockAlignment, CGM);
1855 }
1856
1857 return Name;
1858}
1859
1860static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
1861 Address Field, QualType CaptureType,
1862 BlockFieldFlags Flags, bool ForCopyHelper,
1863 VarDecl *Var, CodeGenFunction &CGF) {
1864 bool EHOnly = ForCopyHelper;
1865
1866 switch (CaptureKind) {
1867 case BlockCaptureEntityKind::CXXRecord:
1868 case BlockCaptureEntityKind::ARCWeak:
1869 case BlockCaptureEntityKind::NonTrivialCStruct:
1870 case BlockCaptureEntityKind::ARCStrong: {
1871 if (CaptureType.isDestructedType() &&
1872 (!EHOnly || CGF.needsEHCleanup(kind: CaptureType.isDestructedType()))) {
1873 CodeGenFunction::Destroyer *Destroyer =
1874 CaptureKind == BlockCaptureEntityKind::ARCStrong
1875 ? CodeGenFunction::destroyARCStrongImprecise
1876 : CGF.getDestroyer(destructionKind: CaptureType.isDestructedType());
1877 CleanupKind Kind =
1878 EHOnly ? EHCleanup
1879 : CGF.getCleanupKind(kind: CaptureType.isDestructedType());
1880 CGF.pushDestroy(kind: Kind, addr: Field, type: CaptureType, destroyer: Destroyer, useEHCleanupForArray: Kind & EHCleanup);
1881 }
1882 break;
1883 }
1884 case BlockCaptureEntityKind::BlockObject: {
1885 if (!EHOnly || CGF.getLangOpts().Exceptions) {
1886 CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;
1887 // Calls to _Block_object_dispose along the EH path in the copy helper
1888 // function don't throw as newly-copied __block variables always have a
1889 // reference count of 2.
1890 bool CanThrow =
1891 !ForCopyHelper && CGF.cxxDestructorCanThrow(T: CaptureType);
1892 CGF.enterByrefCleanup(Kind, Addr: Field, Flags, /*LoadBlockVarAddr*/ true,
1893 CanThrow);
1894 }
1895 break;
1896 }
1897 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth:
1898 case BlockCaptureEntityKind::None:
1899 break;
1900 }
1901}
1902
1903static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,
1904 llvm::Function *Fn,
1905 const CGFunctionInfo &FI,
1906 CodeGenModule &CGM) {
1907 if (CapturesNonExternalType) {
1908 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
1909 } else {
1910 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1911 Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1912 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
1913 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
1914 }
1915}
1916/// Generate the copy-helper function for a block closure object:
1917/// static void block_copy_helper(block_t *dst, block_t *src);
1918/// The runtime will have previously initialized 'dst' by doing a
1919/// bit-copy of 'src'.
1920///
1921/// Note that this copies an entire block closure object to the heap;
1922/// it should not be confused with a 'byref copy helper', which moves
1923/// the contents of an individual __block variable to the heap.
1924llvm::Constant *
1925CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
1926 std::string FuncName = getCopyDestroyHelperFuncName(
1927 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
1928 StrKind: CaptureStrKind::CopyHelper, CGM);
1929
1930 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
1931 return Func;
1932
1933 ASTContext &C = getContext();
1934
1935 QualType ReturnTy = C.VoidTy;
1936
1937 auto *DstDecl =
1938 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1939 auto *SrcDecl =
1940 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1941
1942 FunctionArgList args{DstDecl, SrcDecl};
1943 const CGFunctionInfo &FI =
1944 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
1945
1946 // FIXME: it would be nice if these were mergeable with things with
1947 // identical semantics.
1948 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
1949
1950 llvm::Function *Fn =
1951 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
1952 N: FuncName, M: &CGM.getModule());
1953 if (CGM.supportsCOMDAT())
1954 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
1955
1956 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
1957 CGM);
1958 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
1959 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1960
1961 Address src = GetAddrOfLocalVar(VD: SrcDecl);
1962 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
1963 blockInfo.BlockAlign);
1964
1965 Address dst = GetAddrOfLocalVar(VD: DstDecl);
1966 dst = Address(Builder.CreateLoad(Addr: dst), blockInfo.StructureType,
1967 blockInfo.BlockAlign);
1968
1969 for (auto &capture : blockInfo.SortedCaptures) {
1970 if (capture.isConstantOrTrivial())
1971 continue;
1972
1973 const BlockDecl::Capture &CI = *capture.Cap;
1974 QualType captureType = CI.getVariable()->getType();
1975 BlockFieldFlags flags = capture.CopyFlags;
1976
1977 unsigned index = capture.getIndex();
1978 Address srcField = Builder.CreateStructGEP(Addr: src, Index: index);
1979 Address dstField = Builder.CreateStructGEP(Addr: dst, Index: index);
1980
1981 switch (capture.CopyKind) {
1982 case BlockCaptureEntityKind::CXXRecord:
1983 // If there's an explicit copy expression, we do that.
1984 assert(CI.getCopyExpr() && "copy expression for variable is missing");
1985 EmitSynthesizedCXXCopyCtor(Dest: dstField, Src: srcField, Exp: CI.getCopyExpr());
1986 break;
1987 case BlockCaptureEntityKind::ARCWeak:
1988 EmitARCCopyWeak(dst: dstField, src: srcField);
1989 break;
1990 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1991 QualType Type = CI.getVariable()->getType();
1992 PointerAuthQualifier PointerAuth = Type.getPointerAuth();
1993 assert(PointerAuth && PointerAuth.isAddressDiscriminated());
1994 EmitPointerAuthCopy(Qualifier: PointerAuth, Type, DestField: dstField, SrcField: srcField);
1995 // We don't need to push cleanups for ptrauth types.
1996 continue;
1997 }
1998 case BlockCaptureEntityKind::NonTrivialCStruct: {
1999 // If this is a C struct that requires non-trivial copy construction,
2000 // emit a call to its copy constructor.
2001 QualType varType = CI.getVariable()->getType();
2002 callCStructCopyConstructor(Dst: MakeAddrLValue(Addr: dstField, T: varType),
2003 Src: MakeAddrLValue(Addr: srcField, T: varType));
2004 break;
2005 }
2006 case BlockCaptureEntityKind::ARCStrong: {
2007 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2008 // At -O0, store null into the destination field (so that the
2009 // storeStrong doesn't over-release) and then call storeStrong.
2010 // This is a workaround to not having an initStrong call.
2011 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
2012 auto *ty = cast<llvm::PointerType>(Val: srcValue->getType());
2013 llvm::Value *null = llvm::ConstantPointerNull::get(T: ty);
2014 Builder.CreateStore(Val: null, Addr: dstField);
2015 EmitARCStoreStrongCall(addr: dstField, value: srcValue, resultIgnored: true);
2016
2017 // With optimization enabled, take advantage of the fact that
2018 // the blocks runtime guarantees a memcpy of the block data, and
2019 // just emit a retain of the src field.
2020 } else {
2021 EmitARCRetainNonBlock(value: srcValue);
2022
2023 // Unless EH cleanup is required, we don't need this anymore, so kill
2024 // it. It's not quite worth the annoyance to avoid creating it in the
2025 // first place.
2026 if (!needsEHCleanup(kind: captureType.isDestructedType()))
2027 if (auto *I = cast_or_null<llvm::Instruction>(
2028 Val: dstField.getPointerIfNotSigned()))
2029 I->eraseFromParent();
2030 }
2031 break;
2032 }
2033 case BlockCaptureEntityKind::BlockObject: {
2034 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2035 llvm::Value *dstAddr = dstField.emitRawPointer(CGF&: *this);
2036 llvm::Value *args[] = {
2037 dstAddr, srcValue, llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())
2038 };
2039
2040 if (CI.isByRef() && C.getBlockVarCopyInit(VD: CI.getVariable()).canThrow())
2041 EmitRuntimeCallOrInvoke(callee: CGM.getBlockObjectAssign(), args);
2042 else
2043 EmitNounwindRuntimeCall(callee: CGM.getBlockObjectAssign(), args);
2044 break;
2045 }
2046 case BlockCaptureEntityKind::None:
2047 continue;
2048 }
2049
2050 // Ensure that we destroy the copied object if an exception is thrown later
2051 // in the helper function.
2052 pushCaptureCleanup(CaptureKind: capture.CopyKind, Field: dstField, CaptureType: captureType, Flags: flags,
2053 /*ForCopyHelper*/ true, Var: CI.getVariable(), CGF&: *this);
2054 }
2055
2056 FinishFunction();
2057
2058 return Fn;
2059}
2060
2061static BlockFieldFlags
2062getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI,
2063 QualType T) {
2064 BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT;
2065 if (T->isBlockPointerType())
2066 Flags = BLOCK_FIELD_IS_BLOCK;
2067 return Flags;
2068}
2069
2070static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
2071computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
2072 const LangOptions &LangOpts) {
2073 if (CI.isEscapingByref()) {
2074 BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
2075 if (T.isObjCGCWeak())
2076 Flags |= BLOCK_FIELD_IS_WEAK;
2077 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
2078 }
2079
2080 switch (T.isDestructedType()) {
2081 case QualType::DK_cxx_destructor:
2082 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
2083 case QualType::DK_objc_strong_lifetime:
2084 // Use objc_storeStrong for __strong direct captures; the
2085 // dynamic tools really like it when we do this.
2086 return std::make_pair(x: BlockCaptureEntityKind::ARCStrong,
2087 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2088 case QualType::DK_objc_weak_lifetime:
2089 // Support __weak direct captures.
2090 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak,
2091 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2092 case QualType::DK_nontrivial_c_struct:
2093 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
2094 y: BlockFieldFlags());
2095 case QualType::DK_none: {
2096 // Non-ARC captures are strong, and we need to use _Block_object_dispose.
2097 // But honor the inert __unsafe_unretained qualifier, which doesn't actually
2098 // make it into the type system.
2099 if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() &&
2100 !LangOpts.ObjCAutoRefCount && !T->isObjCInertUnsafeUnretainedType())
2101 return std::make_pair(x: BlockCaptureEntityKind::BlockObject,
2102 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2103 // Otherwise, we have nothing to do.
2104 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
2105 }
2106 }
2107 llvm_unreachable("after exhaustive DestructionKind switch");
2108}
2109
2110/// Generate the destroy-helper function for a block closure object:
2111/// static void block_destroy_helper(block_t *theBlock);
2112///
2113/// Note that this destroys a heap-allocated block closure object;
2114/// it should not be confused with a 'byref destroy helper', which
2115/// destroys the heap-allocated contents of an individual __block
2116/// variable.
2117llvm::Constant *
2118CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
2119 std::string FuncName = getCopyDestroyHelperFuncName(
2120 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
2121 StrKind: CaptureStrKind::DisposeHelper, CGM);
2122
2123 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
2124 return Func;
2125
2126 ASTContext &C = getContext();
2127
2128 QualType ReturnTy = C.VoidTy;
2129
2130 auto *SrcDecl =
2131 ImplicitParamDecl::Create(C, T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2132
2133 FunctionArgList args{SrcDecl};
2134 const CGFunctionInfo &FI =
2135 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2136
2137 // FIXME: We'd like to put these into a mergable by content, with
2138 // internal linkage.
2139 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
2140
2141 llvm::Function *Fn =
2142 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
2143 N: FuncName, M: &CGM.getModule());
2144 if (CGM.supportsCOMDAT())
2145 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
2146
2147 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
2148 CGM);
2149 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2150 markAsIgnoreThreadCheckingAtRuntime(Fn);
2151
2152 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2153
2154 Address src = GetAddrOfLocalVar(VD: SrcDecl);
2155 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
2156 blockInfo.BlockAlign);
2157
2158 CodeGenFunction::RunCleanupsScope cleanups(*this);
2159
2160 for (auto &capture : blockInfo.SortedCaptures) {
2161 if (capture.isConstantOrTrivial())
2162 continue;
2163
2164 const BlockDecl::Capture &CI = *capture.Cap;
2165 BlockFieldFlags flags = capture.DisposeFlags;
2166
2167 Address srcField = Builder.CreateStructGEP(Addr: src, Index: capture.getIndex());
2168
2169 pushCaptureCleanup(CaptureKind: capture.DisposeKind, Field: srcField,
2170 CaptureType: CI.getVariable()->getType(), Flags: flags,
2171 /*ForCopyHelper*/ false, Var: CI.getVariable(), CGF&: *this);
2172 }
2173
2174 cleanups.ForceCleanup();
2175
2176 FinishFunction();
2177
2178 return Fn;
2179}
2180
2181namespace {
2182
2183/// Emits the copy/dispose helper functions for a __block object of id type.
2184class ObjectByrefHelpers final : public BlockByrefHelpers {
2185 BlockFieldFlags Flags;
2186
2187public:
2188 ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
2189 : BlockByrefHelpers(alignment), Flags(flags) {}
2190
2191 void emitCopy(CodeGenFunction &CGF, Address destField,
2192 Address srcField) override {
2193 destField = destField.withElementType(ElemTy: CGF.Int8Ty);
2194
2195 srcField = srcField.withElementType(ElemTy: CGF.Int8PtrTy);
2196 llvm::Value *srcValue = CGF.Builder.CreateLoad(Addr: srcField);
2197
2198 unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
2199
2200 llvm::Value *flagsVal = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: flags);
2201 llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
2202
2203 llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};
2204 CGF.EmitNounwindRuntimeCall(callee: fn, args);
2205 }
2206
2207 void emitDispose(CodeGenFunction &CGF, Address field) override {
2208 field = field.withElementType(ElemTy: CGF.Int8PtrTy);
2209 llvm::Value *value = CGF.Builder.CreateLoad(Addr: field);
2210
2211 CGF.BuildBlockRelease(DeclPtr: value, flags: Flags | BLOCK_BYREF_CALLER, CanThrow: false);
2212 }
2213
2214 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2215 id.AddInteger(I: Flags.getBitMask());
2216 }
2217};
2218
2219/// Emits the copy/dispose helpers for an ARC __block __weak variable.
2220class ARCWeakByrefHelpers final : public BlockByrefHelpers {
2221public:
2222 ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2223
2224 void emitCopy(CodeGenFunction &CGF, Address destField,
2225 Address srcField) override {
2226 CGF.EmitARCMoveWeak(dst: destField, src: srcField);
2227 }
2228
2229 void emitDispose(CodeGenFunction &CGF, Address field) override {
2230 CGF.EmitARCDestroyWeak(addr: field);
2231 }
2232
2233 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2234 // 0 is distinguishable from all pointers and byref flags
2235 id.AddInteger(I: 0);
2236 }
2237};
2238
2239/// Emits the copy/dispose helpers for an ARC __block __strong variable
2240/// that's not of block-pointer type.
2241class ARCStrongByrefHelpers final : public BlockByrefHelpers {
2242public:
2243 ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2244
2245 void emitCopy(CodeGenFunction &CGF, Address destField,
2246 Address srcField) override {
2247 // Do a "move" by copying the value and then zeroing out the old
2248 // variable.
2249
2250 llvm::Value *value = CGF.Builder.CreateLoad(Addr: srcField);
2251
2252 llvm::Value *null =
2253 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: value->getType()));
2254
2255 if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {
2256 CGF.Builder.CreateStore(Val: null, Addr: destField);
2257 CGF.EmitARCStoreStrongCall(addr: destField, value, /*ignored*/ resultIgnored: true);
2258 CGF.EmitARCStoreStrongCall(addr: srcField, value: null, /*ignored*/ resultIgnored: true);
2259 return;
2260 }
2261 CGF.Builder.CreateStore(Val: value, Addr: destField);
2262 CGF.Builder.CreateStore(Val: null, Addr: srcField);
2263 }
2264
2265 void emitDispose(CodeGenFunction &CGF, Address field) override {
2266 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2267 }
2268
2269 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2270 // 1 is distinguishable from all pointers and byref flags
2271 id.AddInteger(I: 1);
2272 }
2273};
2274
2275/// Emits the copy/dispose helpers for an ARC __block __strong
2276/// variable that's of block-pointer type.
2277class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {
2278public:
2279 ARCStrongBlockByrefHelpers(CharUnits alignment)
2280 : BlockByrefHelpers(alignment) {}
2281
2282 void emitCopy(CodeGenFunction &CGF, Address destField,
2283 Address srcField) override {
2284 // Do the copy with objc_retainBlock; that's all that
2285 // _Block_object_assign would do anyway, and we'd have to pass the
2286 // right arguments to make sure it doesn't get no-op'ed.
2287 llvm::Value *oldValue = CGF.Builder.CreateLoad(Addr: srcField);
2288 llvm::Value *copy = CGF.EmitARCRetainBlock(value: oldValue, /*mandatory*/ true);
2289 CGF.Builder.CreateStore(Val: copy, Addr: destField);
2290 }
2291
2292 void emitDispose(CodeGenFunction &CGF, Address field) override {
2293 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2294 }
2295
2296 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2297 // 2 is distinguishable from all pointers and byref flags
2298 id.AddInteger(I: 2);
2299 }
2300};
2301
2302/// Emits the copy/dispose helpers for a __block variable with a
2303/// nontrivial copy constructor or destructor.
2304class CXXByrefHelpers final : public BlockByrefHelpers {
2305 QualType VarType;
2306 const Expr *CopyExpr;
2307
2308public:
2309 CXXByrefHelpers(CharUnits alignment, QualType type,
2310 const Expr *copyExpr)
2311 : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
2312
2313 bool needsCopy() const override { return CopyExpr != nullptr; }
2314 void emitCopy(CodeGenFunction &CGF, Address destField,
2315 Address srcField) override {
2316 if (!CopyExpr) return;
2317 CGF.EmitSynthesizedCXXCopyCtor(Dest: destField, Src: srcField, Exp: CopyExpr);
2318 }
2319
2320 void emitDispose(CodeGenFunction &CGF, Address field) override {
2321 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2322 CGF.PushDestructorCleanup(T: VarType, Addr: field);
2323 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2324 }
2325
2326 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2327 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2328 }
2329};
2330
2331/// Emits the copy/dispose helpers for a __block variable with
2332/// address-discriminated pointer authentication.
2333class AddressDiscriminatedByrefHelpers final : public BlockByrefHelpers {
2334 QualType VarType;
2335
2336public:
2337 AddressDiscriminatedByrefHelpers(CharUnits Alignment, QualType Type)
2338 : BlockByrefHelpers(Alignment), VarType(Type) {
2339 assert(Type.hasAddressDiscriminatedPointerAuth());
2340 }
2341
2342 void emitCopy(CodeGenFunction &CGF, Address DestField,
2343 Address SrcField) override {
2344 CGF.EmitPointerAuthCopy(Qualifier: VarType.getPointerAuth(), Type: VarType, DestField,
2345 SrcField);
2346 }
2347
2348 bool needsDispose() const override { return false; }
2349 void emitDispose(CodeGenFunction &CGF, Address Field) override {
2350 llvm_unreachable("should never be called");
2351 }
2352
2353 void profileImpl(llvm::FoldingSetNodeID &ID) const override {
2354 ID.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2355 }
2356};
2357
2358/// Emits the copy/dispose helpers for a __block variable that is a non-trivial
2359/// C struct.
2360class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers {
2361 QualType VarType;
2362
2363public:
2364 NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type)
2365 : BlockByrefHelpers(alignment), VarType(type) {}
2366
2367 void emitCopy(CodeGenFunction &CGF, Address destField,
2368 Address srcField) override {
2369 CGF.callCStructMoveConstructor(Dst: CGF.MakeAddrLValue(Addr: destField, T: VarType),
2370 Src: CGF.MakeAddrLValue(Addr: srcField, T: VarType));
2371 }
2372
2373 bool needsDispose() const override {
2374 return VarType.isDestructedType();
2375 }
2376
2377 void emitDispose(CodeGenFunction &CGF, Address field) override {
2378 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2379 CGF.pushDestroy(dtorKind: VarType.isDestructedType(), addr: field, type: VarType);
2380 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2381 }
2382
2383 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2384 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2385 }
2386};
2387} // end anonymous namespace
2388
2389static llvm::Constant *
2390generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
2391 BlockByrefHelpers &generator) {
2392 ASTContext &Context = CGF.getContext();
2393
2394 QualType ReturnTy = Context.VoidTy;
2395
2396 auto *Dst = ImplicitParamDecl::Create(C&: Context, T: Context.VoidPtrTy,
2397 ParamKind: ImplicitParamKind::Other);
2398 auto *Src = ImplicitParamDecl::Create(C&: Context, T: Context.VoidPtrTy,
2399 ParamKind: ImplicitParamKind::Other);
2400
2401 FunctionArgList args{Dst, Src};
2402 const CGFunctionInfo &FI =
2403 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2404
2405 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2406
2407 // FIXME: We'd like to put these into a mergable by content, with
2408 // internal linkage.
2409 llvm::Function *Fn =
2410 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2411 N: "__Block_byref_object_copy_", M: &CGF.CGM.getModule());
2412
2413 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2414
2415 CGF.StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2416 // Create a scope with an artificial location for the body of this function.
2417 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2418
2419 if (generator.needsCopy()) {
2420 // dst->x
2421 Address destField = CGF.GetAddrOfLocalVar(VD: Dst);
2422 destField = Address(CGF.Builder.CreateLoad(Addr: destField), byrefInfo.Type,
2423 byrefInfo.ByrefAlignment);
2424 destField =
2425 CGF.emitBlockByrefAddress(baseAddr: destField, info: byrefInfo, followForward: false, name: "dest-object");
2426
2427 // src->x
2428 Address srcField = CGF.GetAddrOfLocalVar(VD: Src);
2429 srcField = Address(CGF.Builder.CreateLoad(Addr: srcField), byrefInfo.Type,
2430 byrefInfo.ByrefAlignment);
2431 srcField =
2432 CGF.emitBlockByrefAddress(baseAddr: srcField, info: byrefInfo, followForward: false, name: "src-object");
2433
2434 generator.emitCopy(CGF, dest: destField, src: srcField);
2435 }
2436
2437 CGF.FinishFunction();
2438
2439 return Fn;
2440}
2441
2442/// Build the copy helper for a __block variable.
2443static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
2444 const BlockByrefInfo &byrefInfo,
2445 BlockByrefHelpers &generator) {
2446 CodeGenFunction CGF(CGM);
2447 return generateByrefCopyHelper(CGF, byrefInfo, generator);
2448}
2449
2450/// Generate code for a __block variable's dispose helper.
2451static llvm::Constant *
2452generateByrefDisposeHelper(CodeGenFunction &CGF,
2453 const BlockByrefInfo &byrefInfo,
2454 BlockByrefHelpers &generator) {
2455 ASTContext &Context = CGF.getContext();
2456 QualType R = Context.VoidTy;
2457
2458 auto *Src = ImplicitParamDecl::Create(C&: CGF.getContext(), T: Context.VoidPtrTy,
2459 ParamKind: ImplicitParamKind::Other);
2460
2461 FunctionArgList args{Src};
2462 const CGFunctionInfo &FI =
2463 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: R, args);
2464
2465 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2466
2467 // FIXME: We'd like to put these into a mergable by content, with
2468 // internal linkage.
2469 llvm::Function *Fn =
2470 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2471 N: "__Block_byref_object_dispose_",
2472 M: &CGF.CGM.getModule());
2473
2474 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2475
2476 CGF.StartFunction(GD: GlobalDecl(), RetTy: R, Fn, FnInfo: FI, Args: args);
2477 // Create a scope with an artificial location for the body of this function.
2478 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2479
2480 if (generator.needsDispose()) {
2481 Address addr = CGF.GetAddrOfLocalVar(VD: Src);
2482 addr = Address(CGF.Builder.CreateLoad(Addr: addr), byrefInfo.Type,
2483 byrefInfo.ByrefAlignment);
2484 addr = CGF.emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, followForward: false, name: "object");
2485
2486 generator.emitDispose(CGF, field: addr);
2487 }
2488
2489 CGF.FinishFunction();
2490
2491 return Fn;
2492}
2493
2494/// Build the dispose helper for a __block variable.
2495static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
2496 const BlockByrefInfo &byrefInfo,
2497 BlockByrefHelpers &generator) {
2498 CodeGenFunction CGF(CGM);
2499 return generateByrefDisposeHelper(CGF, byrefInfo, generator);
2500}
2501
2502/// Lazily build the copy and dispose helpers for a __block variable
2503/// with the given information.
2504template <class T>
2505static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,
2506 T &&generator) {
2507 llvm::FoldingSetNodeID id;
2508 generator.Profile(id);
2509
2510 llvm::FoldingSetInsertToken InsertToken;
2511 BlockByrefHelpers *node = CGM.ByrefHelpersCache.lookup(ID: id, Token&: InsertToken);
2512 if (node) return static_cast<T*>(node);
2513
2514 generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);
2515 generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);
2516
2517 T *copy = new (CGM.getContext()) T(std::forward<T>(generator));
2518 CGM.ByrefHelpersCache.insert(copy, InsertToken);
2519 return copy;
2520}
2521
2522/// Build the copy and dispose helpers for the given __block variable
2523/// emission. Places the helpers in the global cache. Returns null
2524/// if no helpers are required.
2525BlockByrefHelpers *
2526CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
2527 const AutoVarEmission &emission) {
2528 const VarDecl &var = *emission.Variable;
2529 assert(var.isEscapingByref() &&
2530 "only escaping __block variables need byref helpers");
2531
2532 QualType type = var.getType();
2533
2534 auto &byrefInfo = getBlockByrefInfo(var: &var);
2535
2536 // The alignment we care about for the purposes of uniquing byref
2537 // helpers is the alignment of the actual byref value field.
2538 CharUnits valueAlignment =
2539 byrefInfo.ByrefAlignment.alignmentAtOffset(offset: byrefInfo.FieldOffset);
2540
2541 if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
2542 const Expr *copyExpr =
2543 CGM.getContext().getBlockVarCopyInit(VD: &var).getCopyExpr();
2544 if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
2545
2546 return ::buildByrefHelpers(
2547 CGM, byrefInfo, generator: CXXByrefHelpers(valueAlignment, type, copyExpr));
2548 }
2549 if (type.hasAddressDiscriminatedPointerAuth()) {
2550 return ::buildByrefHelpers(
2551 CGM, byrefInfo, generator: AddressDiscriminatedByrefHelpers(valueAlignment, type));
2552 }
2553 // If type is a non-trivial C struct type that is non-trivial to
2554 // destructly move or destroy, build the copy and dispose helpers.
2555 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct ||
2556 type.isDestructedType() == QualType::DK_nontrivial_c_struct)
2557 return ::buildByrefHelpers(
2558 CGM, byrefInfo, generator: NonTrivialCStructByrefHelpers(valueAlignment, type));
2559
2560 // Otherwise, if we don't have a retainable type, there's nothing to do.
2561 // that the runtime does extra copies.
2562 if (!type->isObjCRetainableType()) return nullptr;
2563
2564 Qualifiers qs = type.getQualifiers();
2565
2566 // If we have lifetime, that dominates.
2567 if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
2568 switch (lifetime) {
2569 case Qualifiers::OCL_None: llvm_unreachable("impossible");
2570
2571 // These are just bits as far as the runtime is concerned.
2572 case Qualifiers::OCL_ExplicitNone:
2573 case Qualifiers::OCL_Autoreleasing:
2574 return nullptr;
2575
2576 // Tell the runtime that this is ARC __weak, called by the
2577 // byref routines.
2578 case Qualifiers::OCL_Weak:
2579 return ::buildByrefHelpers(CGM, byrefInfo,
2580 generator: ARCWeakByrefHelpers(valueAlignment));
2581
2582 // ARC __strong __block variables need to be retained.
2583 case Qualifiers::OCL_Strong:
2584 // Block pointers need to be copied, and there's no direct
2585 // transfer possible.
2586 if (type->isBlockPointerType()) {
2587 return ::buildByrefHelpers(CGM, byrefInfo,
2588 generator: ARCStrongBlockByrefHelpers(valueAlignment));
2589
2590 // Otherwise, we transfer ownership of the retain from the stack
2591 // to the heap.
2592 } else {
2593 return ::buildByrefHelpers(CGM, byrefInfo,
2594 generator: ARCStrongByrefHelpers(valueAlignment));
2595 }
2596 }
2597 llvm_unreachable("fell out of lifetime switch!");
2598 }
2599
2600 BlockFieldFlags flags;
2601 if (type->isBlockPointerType()) {
2602 flags |= BLOCK_FIELD_IS_BLOCK;
2603 } else if (CGM.getContext().isObjCNSObjectType(Ty: type) ||
2604 type->isObjCObjectPointerType()) {
2605 flags |= BLOCK_FIELD_IS_OBJECT;
2606 } else {
2607 return nullptr;
2608 }
2609
2610 if (type.isObjCGCWeak())
2611 flags |= BLOCK_FIELD_IS_WEAK;
2612
2613 return ::buildByrefHelpers(CGM, byrefInfo,
2614 generator: ObjectByrefHelpers(valueAlignment, flags));
2615}
2616
2617Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2618 const VarDecl *var,
2619 bool followForward) {
2620 auto &info = getBlockByrefInfo(var);
2621 return emitBlockByrefAddress(baseAddr, info, followForward, name: var->getName());
2622}
2623
2624Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2625 const BlockByrefInfo &info,
2626 bool followForward,
2627 const llvm::Twine &name) {
2628 // Chase the forwarding address if requested.
2629 if (followForward) {
2630 Address forwardingAddr = Builder.CreateStructGEP(Addr: baseAddr, Index: 1, Name: "forwarding");
2631 baseAddr = Address(Builder.CreateLoad(Addr: forwardingAddr), info.Type,
2632 info.ByrefAlignment);
2633 }
2634
2635 return Builder.CreateStructGEP(Addr: baseAddr, Index: info.FieldIndex, Name: name);
2636}
2637
2638/// BuildByrefInfo - This routine changes a __block variable declared as T x
2639/// into:
2640///
2641/// struct {
2642/// void *__isa;
2643/// void *__forwarding;
2644/// int32_t __flags;
2645/// int32_t __size;
2646/// void *__copy_helper; // only if needed
2647/// void *__destroy_helper; // only if needed
2648/// void *__byref_variable_layout;// only if needed
2649/// char padding[X]; // only if needed
2650/// T x;
2651/// } x
2652///
2653const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
2654 auto it = BlockByrefInfos.find(Val: D);
2655 if (it != BlockByrefInfos.end())
2656 return it->second;
2657
2658 QualType Ty = D->getType();
2659
2660 CharUnits size;
2661 SmallVector<llvm::Type *, 8> types;
2662
2663 // void *__isa;
2664 types.push_back(Elt: VoidPtrTy);
2665 size += getPointerSize();
2666
2667 // void *__forwarding;
2668 types.push_back(Elt: VoidPtrTy);
2669 size += getPointerSize();
2670
2671 // int32_t __flags;
2672 types.push_back(Elt: Int32Ty);
2673 size += CharUnits::fromQuantity(Quantity: 4);
2674
2675 // int32_t __size;
2676 types.push_back(Elt: Int32Ty);
2677 size += CharUnits::fromQuantity(Quantity: 4);
2678
2679 // Note that this must match *exactly* the logic in buildByrefHelpers.
2680 bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
2681 if (hasCopyAndDispose) {
2682 /// void *__copy_helper;
2683 types.push_back(Elt: VoidPtrTy);
2684 size += getPointerSize();
2685
2686 /// void *__destroy_helper;
2687 types.push_back(Elt: VoidPtrTy);
2688 size += getPointerSize();
2689 }
2690
2691 bool HasByrefExtendedLayout = false;
2692 Qualifiers::ObjCLifetime Lifetime = Qualifiers::OCL_None;
2693 if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&
2694 HasByrefExtendedLayout) {
2695 /// void *__byref_variable_layout;
2696 types.push_back(Elt: VoidPtrTy);
2697 size += CharUnits::fromQuantity(Quantity: PointerSizeInBytes);
2698 }
2699
2700 // T x;
2701 llvm::Type *varTy = ConvertTypeForMem(T: Ty);
2702
2703 bool packed = false;
2704 CharUnits varAlign = getContext().getDeclAlign(D);
2705 CharUnits varOffset = size.alignTo(Align: varAlign);
2706
2707 // We may have to insert padding.
2708 if (varOffset != size) {
2709 llvm::Type *paddingTy =
2710 llvm::ArrayType::get(ElementType: Int8Ty, NumElements: (varOffset - size).getQuantity());
2711
2712 types.push_back(Elt: paddingTy);
2713 size = varOffset;
2714
2715 // Conversely, we might have to prevent LLVM from inserting padding.
2716 } else if (CGM.getDataLayout().getABITypeAlign(Ty: varTy) >
2717 uint64_t(varAlign.getQuantity())) {
2718 packed = true;
2719 }
2720 types.push_back(Elt: varTy);
2721
2722 llvm::StructType *byrefType = llvm::StructType::create(
2723 Context&: getLLVMContext(), Elements: types, Name: "struct.__block_byref_" + D->getNameAsString(),
2724 isPacked: packed);
2725
2726 BlockByrefInfo info;
2727 info.Type = byrefType;
2728 info.FieldIndex = types.size() - 1;
2729 info.FieldOffset = varOffset;
2730 info.ByrefAlignment = std::max(a: varAlign, b: getPointerAlign());
2731
2732 auto pair = BlockByrefInfos.insert(KV: {D, info});
2733 assert(pair.second && "info was inserted recursively?");
2734 return pair.first->second;
2735}
2736
2737/// Initialize the structural components of a __block variable, i.e.
2738/// everything but the actual object.
2739void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
2740 // Find the address of the local.
2741 Address addr = emission.Addr;
2742
2743 // That's an alloca of the byref structure type.
2744 llvm::StructType *byrefType = cast<llvm::StructType>(Val: addr.getElementType());
2745
2746 unsigned nextHeaderIndex = 0;
2747 CharUnits nextHeaderOffset;
2748 auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
2749 const Twine &name, bool isFunction = false) {
2750 auto fieldAddr = Builder.CreateStructGEP(Addr: addr, Index: nextHeaderIndex, Name: name);
2751 if (isFunction) {
2752 if (auto &Schema = CGM.getCodeGenOpts()
2753 .PointerAuth.BlockByrefHelperFunctionPointers) {
2754 auto PointerAuth = EmitPointerAuthInfo(
2755 Schema, StorageAddress: fieldAddr.emitRawPointer(CGF&: *this), SchemaDecl: GlobalDecl(), SchemaType: QualType());
2756 value = EmitPointerAuthSign(Info: PointerAuth, Pointer: value);
2757 }
2758 }
2759 Builder.CreateStore(Val: value, Addr: fieldAddr);
2760
2761 nextHeaderIndex++;
2762 nextHeaderOffset += fieldSize;
2763 };
2764
2765 // Build the byref helpers if necessary. This is null if we don't need any.
2766 BlockByrefHelpers *helpers = buildByrefHelpers(byrefType&: *byrefType, emission);
2767
2768 const VarDecl &D = *emission.Variable;
2769 QualType type = D.getType();
2770
2771 bool HasByrefExtendedLayout = false;
2772 Qualifiers::ObjCLifetime ByrefLifetime = Qualifiers::OCL_None;
2773 bool ByRefHasLifetime =
2774 getContext().getByrefLifetime(Ty: type, Lifetime&: ByrefLifetime, HasByrefExtendedLayout);
2775
2776 llvm::Value *V;
2777
2778 // Initialize the 'isa', which is just 0 or 1.
2779 int isa = 0;
2780 if (type.isObjCGCWeak())
2781 isa = 1;
2782 V = Builder.CreateIntToPtr(V: Builder.getInt32(C: isa), DestTy: Int8PtrTy, Name: "isa");
2783 storeHeaderField(V, getPointerSize(), "byref.isa");
2784
2785 // Store the address of the variable into its own forwarding pointer.
2786 storeHeaderField(addr.emitRawPointer(CGF&: *this), getPointerSize(),
2787 "byref.forwarding");
2788
2789 // Blocks ABI:
2790 // c) the flags field is set to either 0 if no helper functions are
2791 // needed or BLOCK_BYREF_HAS_COPY_DISPOSE if they are,
2792 BlockFlags flags;
2793 if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE;
2794 if (ByRefHasLifetime) {
2795 if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED;
2796 else switch (ByrefLifetime) {
2797 case Qualifiers::OCL_Strong:
2798 flags |= BLOCK_BYREF_LAYOUT_STRONG;
2799 break;
2800 case Qualifiers::OCL_Weak:
2801 flags |= BLOCK_BYREF_LAYOUT_WEAK;
2802 break;
2803 case Qualifiers::OCL_ExplicitNone:
2804 flags |= BLOCK_BYREF_LAYOUT_UNRETAINED;
2805 break;
2806 case Qualifiers::OCL_None:
2807 if (!type->isObjCObjectPointerType() && !type->isBlockPointerType())
2808 flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT;
2809 break;
2810 default:
2811 break;
2812 }
2813 if (CGM.getLangOpts().ObjCGCBitmapPrint) {
2814 printf(format: "\n Inline flag for BYREF variable layout (%d):", flags.getBitMask());
2815 if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE)
2816 printf(format: " BLOCK_BYREF_HAS_COPY_DISPOSE");
2817 if (flags & BLOCK_BYREF_LAYOUT_MASK) {
2818 BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK);
2819 if (ThisFlag == BLOCK_BYREF_LAYOUT_EXTENDED)
2820 printf(format: " BLOCK_BYREF_LAYOUT_EXTENDED");
2821 if (ThisFlag == BLOCK_BYREF_LAYOUT_STRONG)
2822 printf(format: " BLOCK_BYREF_LAYOUT_STRONG");
2823 if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK)
2824 printf(format: " BLOCK_BYREF_LAYOUT_WEAK");
2825 if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED)
2826 printf(format: " BLOCK_BYREF_LAYOUT_UNRETAINED");
2827 if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT)
2828 printf(format: " BLOCK_BYREF_LAYOUT_NON_OBJECT");
2829 }
2830 printf(format: "\n");
2831 }
2832 }
2833 storeHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
2834 getIntSize(), "byref.flags");
2835
2836 CharUnits byrefSize = CGM.GetTargetTypeStoreSize(Ty: byrefType);
2837 V = llvm::ConstantInt::get(Ty: IntTy, V: byrefSize.getQuantity());
2838 storeHeaderField(V, getIntSize(), "byref.size");
2839
2840 if (helpers) {
2841 storeHeaderField(helpers->CopyHelper, getPointerSize(), "byref.copyHelper",
2842 /*isFunction=*/true);
2843 storeHeaderField(helpers->DisposeHelper, getPointerSize(),
2844 "byref.disposeHelper", /*isFunction=*/true);
2845 }
2846
2847 if (ByRefHasLifetime && HasByrefExtendedLayout) {
2848 auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, T: type);
2849 storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");
2850 }
2851}
2852
2853void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,
2854 bool CanThrow) {
2855 llvm::FunctionCallee F = CGM.getBlockObjectDispose();
2856 llvm::Value *args[] = {V,
2857 llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())};
2858
2859 if (CanThrow)
2860 EmitRuntimeCallOrInvoke(callee: F, args);
2861 else
2862 EmitNounwindRuntimeCall(callee: F, args);
2863}
2864
2865void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,
2866 BlockFieldFlags Flags,
2867 bool LoadBlockVarAddr, bool CanThrow) {
2868 EHStack.pushCleanup<CallBlockRelease>(Kind, A: Addr, A: Flags, A: LoadBlockVarAddr,
2869 A: CanThrow);
2870}
2871
2872/// Adjust the declaration of something from the blocks API.
2873static void configureBlocksRuntimeObject(CodeGenModule &CGM,
2874 llvm::Constant *C) {
2875 auto *GV = cast<llvm::GlobalValue>(Val: C->stripPointerCasts());
2876
2877 if (!CGM.getCodeGenOpts().StaticClosure &&
2878 CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
2879 const IdentifierInfo &II = CGM.getContext().Idents.get(Name: C->getName());
2880 TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
2881 DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
2882
2883 assert((isa<llvm::Function>(C->stripPointerCasts()) ||
2884 isa<llvm::GlobalVariable>(C->stripPointerCasts())) &&
2885 "expected Function or GlobalVariable");
2886
2887 const NamedDecl *ND = nullptr;
2888 for (const auto *Result : DC->lookup(Name: &II))
2889 if ((ND = dyn_cast<FunctionDecl>(Val: Result)) ||
2890 (ND = dyn_cast<VarDecl>(Val: Result)))
2891 break;
2892
2893 if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {
2894 GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2895 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2896 } else {
2897 GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2898 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2899 }
2900 }
2901
2902 if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() &&
2903 GV->hasExternalLinkage())
2904 GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2905
2906 CGM.setDSOLocal(GV);
2907}
2908
2909llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() {
2910 if (BlockObjectDispose)
2911 return BlockObjectDispose;
2912
2913 QualType args[] = {Context.VoidPtrTy, Context.IntTy};
2914 BlockObjectDispose =
2915 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_dispose");
2916 configureBlocksRuntimeObject(
2917 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectDispose.getCallee()));
2918 return BlockObjectDispose;
2919}
2920
2921llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() {
2922 if (BlockObjectAssign)
2923 return BlockObjectAssign;
2924
2925 QualType args[] = {Context.VoidPtrTy, Context.VoidPtrTy, Context.IntTy};
2926 BlockObjectAssign =
2927 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_assign");
2928 configureBlocksRuntimeObject(
2929 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectAssign.getCallee()));
2930 return BlockObjectAssign;
2931}
2932
2933llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
2934 if (NSConcreteGlobalBlock)
2935 return NSConcreteGlobalBlock;
2936
2937 NSConcreteGlobalBlock = GetOrCreateLLVMGlobal(
2938 MangledName: "_NSConcreteGlobalBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2939 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteGlobalBlock);
2940 return NSConcreteGlobalBlock;
2941}
2942
2943llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
2944 if (NSConcreteStackBlock)
2945 return NSConcreteStackBlock;
2946
2947 NSConcreteStackBlock = GetOrCreateLLVMGlobal(
2948 MangledName: "_NSConcreteStackBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2949 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteStackBlock);
2950 return NSConcreteStackBlock;
2951}
2952