1//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit blocks.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGBlocks.h"
14#include "CGCXXABI.h"
15#include "CGDebugInfo.h"
16#include "CGObjCRuntime.h"
17#include "CGOpenCLRuntime.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "CodeGenPGO.h"
21#include "ConstantEmitter.h"
22#include "TargetInfo.h"
23#include "clang/AST/Attr.h"
24#include "clang/AST/DeclObjC.h"
25#include "clang/CodeGen/ConstantInitBuilder.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Module.h"
28#include "llvm/Support/ScopedPrinter.h"
29#include <algorithm>
30#include <cstdio>
31
32using namespace clang;
33using namespace CodeGen;
34
35CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
36 : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
37 NoEscape(false), HasCXXObject(false), UsesStret(false),
38 HasCapturedVariableLayout(false), CapturesNonExternalType(false),
39 LocalAddress(RawAddress::invalid()), StructureType(nullptr),
40 Block(block) {
41
42 // Skip asm prefix, if any. 'name' is usually taken directly from
43 // the mangled name of the enclosing function.
44 name.consume_front(Prefix: "\01");
45}
46
47// Anchor the vtable to this translation unit.
48BlockByrefHelpers::~BlockByrefHelpers() {}
49
50/// Build the given block as a global block.
51static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
52 const CGBlockInfo &blockInfo,
53 llvm::Constant *blockFn);
54
55/// Build the helper function to copy a block.
56static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,
57 const CGBlockInfo &blockInfo) {
58 return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);
59}
60
61/// Build the helper function to dispose of a block.
62static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
63 const CGBlockInfo &blockInfo) {
64 return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
65}
66
67namespace {
68
69enum class CaptureStrKind {
70 // String for the copy helper.
71 CopyHelper,
72 // String for the dispose helper.
73 DisposeHelper,
74 // Merge the strings for the copy helper and dispose helper.
75 Merged
76};
77
78} // end anonymous namespace
79
80static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
81 CaptureStrKind StrKind,
82 CharUnits BlockAlignment,
83 CodeGenModule &CGM);
84
85static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo,
86 CodeGenModule &CGM) {
87 std::string Name = "__block_descriptor_";
88 Name += llvm::to_string(Value: BlockInfo.BlockSize.getQuantity()) + "_";
89
90 if (BlockInfo.NeedsCopyDispose) {
91 if (CGM.getLangOpts().Exceptions)
92 Name += "e";
93 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
94 Name += "a";
95 Name += llvm::to_string(Value: BlockInfo.BlockAlign.getQuantity()) + "_";
96
97 for (auto &Cap : BlockInfo.SortedCaptures) {
98 if (Cap.isConstantOrTrivial())
99 continue;
100
101 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
102
103 if (Cap.CopyKind == Cap.DisposeKind) {
104 // If CopyKind and DisposeKind are the same, merge the capture
105 // information.
106 assert(Cap.CopyKind != BlockCaptureEntityKind::None &&
107 "shouldn't see BlockCaptureManagedEntity that is None");
108 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::Merged,
109 BlockAlignment: BlockInfo.BlockAlign, CGM);
110 } else {
111 // If CopyKind and DisposeKind are not the same, which can happen when
112 // either Kind is None or the captured object is a __strong block,
113 // concatenate the copy and dispose strings.
114 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::CopyHelper,
115 BlockAlignment: BlockInfo.BlockAlign, CGM);
116 Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::DisposeHelper,
117 BlockAlignment: BlockInfo.BlockAlign, CGM);
118 }
119 }
120 Name += "_";
121 }
122
123 std::string TypeAtEncoding;
124
125 if (!CGM.getCodeGenOpts().DisableBlockSignatureString) {
126 TypeAtEncoding =
127 CGM.getContext().getObjCEncodingForBlock(blockExpr: BlockInfo.getBlockExpr());
128 /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms
129 /// as a separator between symbol name and symbol version.
130 llvm::replace(Range&: TypeAtEncoding, OldValue: '@', NewValue: '\1');
131 }
132 Name += "e" + llvm::to_string(Value: TypeAtEncoding.size()) + "_" + TypeAtEncoding;
133 Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, blockInfo: BlockInfo);
134 return Name;
135}
136
137/// buildBlockDescriptor - Build the block descriptor meta-data for a block.
138/// buildBlockDescriptor is accessed from 5th field of the Block_literal
139/// meta-data and contains stationary information about the block literal.
140/// Its definition will have 4 (or optionally 6) words.
141/// \code
142/// struct Block_descriptor {
143/// unsigned long reserved;
144/// unsigned long size; // size of Block_literal metadata in bytes.
145/// void *copy_func_helper_decl; // optional copy helper.
146/// void *destroy_func_decl; // optional destructor helper.
147/// void *block_method_encoding_address; // @encode for block literal signature.
148/// void *block_layout_info; // encoding of captured block variables.
149/// };
150/// \endcode
151static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
152 const CGBlockInfo &blockInfo) {
153 ASTContext &C = CGM.getContext();
154
155 llvm::IntegerType *ulong =
156 cast<llvm::IntegerType>(Val: CGM.getTypes().ConvertType(T: C.UnsignedLongTy));
157 llvm::PointerType *i8p = nullptr;
158 if (CGM.getLangOpts().OpenCL)
159 i8p = llvm::PointerType::get(
160 C&: CGM.getLLVMContext(), AddressSpace: C.getTargetAddressSpace(AS: LangAS::opencl_constant));
161 else
162 i8p = CGM.VoidPtrTy;
163
164 std::string descName;
165
166 // If an equivalent block descriptor global variable exists, return it.
167 if (C.getLangOpts().ObjC &&
168 CGM.getLangOpts().getGC() == LangOptions::NonGC) {
169 descName = getBlockDescriptorName(BlockInfo: blockInfo, CGM);
170 if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(Name: descName))
171 return desc;
172 }
173
174 // If there isn't an equivalent block descriptor global variable, create a new
175 // one.
176 ConstantInitBuilder builder(CGM);
177 auto elements = builder.beginStruct();
178
179 // reserved
180 elements.addInt(intTy: ulong, value: 0);
181
182 // Size
183 // FIXME: What is the right way to say this doesn't fit? We should give
184 // a user diagnostic in that case. Better fix would be to change the
185 // API to size_t.
186 elements.addInt(intTy: ulong, value: blockInfo.BlockSize.getQuantity());
187
188 // Optional copy/dispose helpers.
189 bool hasInternalHelper = false;
190 if (blockInfo.NeedsCopyDispose) {
191 // copy_func_helper_decl
192 llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);
193 elements.add(value: copyHelper);
194
195 // destroy_func_decl
196 llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
197 elements.add(value: disposeHelper);
198
199 if (cast<llvm::Function>(Val: copyHelper->stripPointerCasts())
200 ->hasInternalLinkage() ||
201 cast<llvm::Function>(Val: disposeHelper->stripPointerCasts())
202 ->hasInternalLinkage())
203 hasInternalHelper = true;
204 }
205
206 // Signature. Mandatory ObjC-style method descriptor @encode sequence.
207 if (CGM.getCodeGenOpts().DisableBlockSignatureString) {
208 elements.addNullPointer(ptrTy: i8p);
209 } else {
210 std::string typeAtEncoding =
211 CGM.getContext().getObjCEncodingForBlock(blockExpr: blockInfo.getBlockExpr());
212 elements.add(value: CGM.GetAddrOfConstantCString(Str: typeAtEncoding).getPointer());
213 }
214
215 // GC layout.
216 if (C.getLangOpts().ObjC) {
217 if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
218 elements.add(value: CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
219 else
220 elements.add(value: CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));
221 }
222 else
223 elements.addNullPointer(ptrTy: i8p);
224
225 unsigned AddrSpace = 0;
226 if (C.getLangOpts().OpenCL)
227 AddrSpace = C.getTargetAddressSpace(AS: LangAS::opencl_constant);
228
229 llvm::GlobalValue::LinkageTypes linkage;
230 if (descName.empty()) {
231 linkage = llvm::GlobalValue::InternalLinkage;
232 descName = "__block_descriptor_tmp";
233 } else if (hasInternalHelper) {
234 // If either the copy helper or the dispose helper has internal linkage,
235 // the block descriptor must have internal linkage too.
236 linkage = llvm::GlobalValue::InternalLinkage;
237 } else {
238 linkage = llvm::GlobalValue::LinkOnceODRLinkage;
239 }
240
241 llvm::GlobalVariable *global =
242 elements.finishAndCreateGlobal(args&: descName, args: CGM.getPointerAlign(),
243 /*constant*/ args: true, args&: linkage, args&: AddrSpace);
244
245 if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) {
246 if (CGM.supportsCOMDAT())
247 global->setComdat(CGM.getModule().getOrInsertComdat(Name: descName));
248 global->setVisibility(llvm::GlobalValue::HiddenVisibility);
249 global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
250 }
251
252 return global;
253}
254
255/*
256 Purely notional variadic template describing the layout of a block.
257
258 template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>
259 struct Block_literal {
260 /// Initialized to one of:
261 /// extern void *_NSConcreteStackBlock[];
262 /// extern void *_NSConcreteGlobalBlock[];
263 ///
264 /// In theory, we could start one off malloc'ed by setting
265 /// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using
266 /// this isa:
267 /// extern void *_NSConcreteMallocBlock[];
268 struct objc_class *isa;
269
270 /// These are the flags (with corresponding bit number) that the
271 /// compiler is actually supposed to know about.
272 /// 23. BLOCK_IS_NOESCAPE - indicates that the block is non-escaping
273 /// 25. BLOCK_HAS_COPY_DISPOSE - indicates that the block
274 /// descriptor provides copy and dispose helper functions
275 /// 26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured
276 /// object with a nontrivial destructor or copy constructor
277 /// 28. BLOCK_IS_GLOBAL - indicates that the block is allocated
278 /// as global memory
279 /// 29. BLOCK_USE_STRET - indicates that the block function
280 /// uses stret, which objc_msgSend needs to know about
281 /// 30. BLOCK_HAS_SIGNATURE - indicates that the block has an
282 /// @encoded signature string
283 /// And we're not supposed to manipulate these:
284 /// 24. BLOCK_NEEDS_FREE - indicates that the block has been moved
285 /// to malloc'ed memory
286 /// 27. BLOCK_IS_GC - indicates that the block has been moved to
287 /// to GC-allocated memory
288 /// Additionally, the bottom 16 bits are a reference count which
289 /// should be zero on the stack.
290 int flags;
291
292 /// Reserved; should be zero-initialized.
293 int reserved;
294
295 /// Function pointer generated from block literal.
296 _ResultType (*invoke)(Block_literal *, _ParamTypes...);
297
298 /// Block description metadata generated from block literal.
299 struct Block_descriptor *block_descriptor;
300
301 /// Captured values follow.
302 _CapturesTypes captures...;
303 };
304 */
305
306namespace {
307 /// A chunk of data that we actually have to capture in the block.
308 struct BlockLayoutChunk {
309 CharUnits Alignment;
310 CharUnits Size;
311 const BlockDecl::Capture *Capture; // null for 'this'
312 llvm::Type *Type;
313 QualType FieldType;
314 BlockCaptureEntityKind CopyKind, DisposeKind;
315 BlockFieldFlags CopyFlags, DisposeFlags;
316
317 BlockLayoutChunk(CharUnits align, CharUnits size,
318 const BlockDecl::Capture *capture, llvm::Type *type,
319 QualType fieldType, BlockCaptureEntityKind CopyKind,
320 BlockFieldFlags CopyFlags,
321 BlockCaptureEntityKind DisposeKind,
322 BlockFieldFlags DisposeFlags)
323 : Alignment(align), Size(size), Capture(capture), Type(type),
324 FieldType(fieldType), CopyKind(CopyKind), DisposeKind(DisposeKind),
325 CopyFlags(CopyFlags), DisposeFlags(DisposeFlags) {}
326
327 /// Tell the block info that this chunk has the given field index.
328 void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {
329 if (!Capture) {
330 info.CXXThisIndex = index;
331 info.CXXThisOffset = offset;
332 } else {
333 info.SortedCaptures.push_back(Elt: CGBlockInfo::Capture::makeIndex(
334 index, offset, FieldType, CopyKind, CopyFlags, DisposeKind,
335 DisposeFlags, Cap: Capture));
336 }
337 }
338
339 bool isTrivial() const {
340 return CopyKind == BlockCaptureEntityKind::None &&
341 DisposeKind == BlockCaptureEntityKind::None;
342 }
343 };
344
345 /// Order by 1) all __strong together 2) next, all block together 3) next,
346 /// all byref together 4) next, all __weak together. Preserve descending
347 /// alignment in all situations.
348 bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
349 if (left.Alignment != right.Alignment)
350 return left.Alignment > right.Alignment;
351
352 auto getPrefOrder = [](const BlockLayoutChunk &chunk) {
353 switch (chunk.CopyKind) {
354 case BlockCaptureEntityKind::ARCStrong:
355 return 0;
356 case BlockCaptureEntityKind::BlockObject:
357 switch (chunk.CopyFlags.getBitMask()) {
358 case BLOCK_FIELD_IS_OBJECT:
359 return 0;
360 case BLOCK_FIELD_IS_BLOCK:
361 return 1;
362 case BLOCK_FIELD_IS_BYREF:
363 return 2;
364 default:
365 break;
366 }
367 break;
368 case BlockCaptureEntityKind::ARCWeak:
369 return 3;
370 default:
371 break;
372 }
373 return 4;
374 };
375
376 return getPrefOrder(left) < getPrefOrder(right);
377 }
378} // end anonymous namespace
379
380static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
381computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
382 const LangOptions &LangOpts);
383
384static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
385computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
386 const LangOptions &LangOpts);
387
388static void addBlockLayout(CharUnits align, CharUnits size,
389 const BlockDecl::Capture *capture, llvm::Type *type,
390 QualType fieldType,
391 SmallVectorImpl<BlockLayoutChunk> &Layout,
392 CGBlockInfo &Info, CodeGenModule &CGM) {
393 if (!capture) {
394 // 'this' capture.
395 Layout.push_back(Elt: BlockLayoutChunk(
396 align, size, capture, type, fieldType, BlockCaptureEntityKind::None,
397 BlockFieldFlags(), BlockCaptureEntityKind::None, BlockFieldFlags()));
398 return;
399 }
400
401 const LangOptions &LangOpts = CGM.getLangOpts();
402 BlockCaptureEntityKind CopyKind, DisposeKind;
403 BlockFieldFlags CopyFlags, DisposeFlags;
404
405 std::tie(args&: CopyKind, args&: CopyFlags) =
406 computeCopyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
407 std::tie(args&: DisposeKind, args&: DisposeFlags) =
408 computeDestroyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
409 Layout.push_back(Elt: BlockLayoutChunk(align, size, capture, type, fieldType,
410 CopyKind, CopyFlags, DisposeKind,
411 DisposeFlags));
412
413 if (Info.NoEscape)
414 return;
415
416 if (!Layout.back().isTrivial())
417 Info.NeedsCopyDispose = true;
418}
419
420/// Determines if the given type is safe for constant capture in C++.
421static bool isSafeForCXXConstantCapture(QualType type) {
422 const RecordType *recordType =
423 type->getBaseElementTypeUnsafe()->getAs<RecordType>();
424
425 // Only records can be unsafe.
426 if (!recordType) return true;
427
428 const auto *record = cast<CXXRecordDecl>(Val: recordType->getDecl());
429
430 // Maintain semantics for classes with non-trivial dtors or copy ctors.
431 if (!record->hasTrivialDestructor()) return false;
432 if (record->hasNonTrivialCopyConstructor()) return false;
433
434 // Otherwise, we just have to make sure there aren't any mutable
435 // fields that might have changed since initialization.
436 return !record->hasMutableFields();
437}
438
439/// It is illegal to modify a const object after initialization.
440/// Therefore, if a const object has a constant initializer, we don't
441/// actually need to keep storage for it in the block; we'll just
442/// rematerialize it at the start of the block function. This is
443/// acceptable because we make no promises about address stability of
444/// captured variables.
445static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
446 CodeGenFunction *CGF,
447 const VarDecl *var) {
448 // Return if this is a function parameter. We shouldn't try to
449 // rematerialize default arguments of function parameters.
450 if (isa<ParmVarDecl>(Val: var))
451 return nullptr;
452
453 QualType type = var->getType();
454
455 // We can only do this if the variable is const.
456 if (!type.isConstQualified()) return nullptr;
457
458 // Furthermore, in C++ we have to worry about mutable fields:
459 // C++ [dcl.type.cv]p4:
460 // Except that any class member declared mutable can be
461 // modified, any attempt to modify a const object during its
462 // lifetime results in undefined behavior.
463 if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type))
464 return nullptr;
465
466 // If the variable doesn't have any initializer (shouldn't this be
467 // invalid?), it's not clear what we should do. Maybe capture as
468 // zero?
469 const Expr *init = var->getInit();
470 if (!init) return nullptr;
471
472 return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(D: *var);
473}
474
475/// Get the low bit of a nonzero character count. This is the
476/// alignment of the nth byte if the 0th byte is universally aligned.
477static CharUnits getLowBit(CharUnits v) {
478 return CharUnits::fromQuantity(Quantity: v.getQuantity() & (~v.getQuantity() + 1));
479}
480
481static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
482 SmallVectorImpl<llvm::Type*> &elementTypes) {
483
484 assert(elementTypes.empty());
485 if (CGM.getLangOpts().OpenCL) {
486 // The header is basically 'struct { int; int; generic void *;
487 // custom_fields; }'. Assert that struct is packed.
488 auto GenPtrAlign = CharUnits::fromQuantity(
489 Quantity: CGM.getTarget().getPointerAlign(AddrSpace: LangAS::opencl_generic) / 8);
490 auto GenPtrSize = CharUnits::fromQuantity(
491 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: LangAS::opencl_generic) / 8);
492 assert(CGM.getIntSize() <= GenPtrSize);
493 assert(CGM.getIntAlign() <= GenPtrAlign);
494 assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign));
495 elementTypes.push_back(Elt: CGM.IntTy); /* total size */
496 elementTypes.push_back(Elt: CGM.IntTy); /* align */
497 elementTypes.push_back(
498 Elt: CGM.getOpenCLRuntime()
499 .getGenericVoidPointerType()); /* invoke function */
500 unsigned Offset =
501 2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity();
502 unsigned BlockAlign = GenPtrAlign.getQuantity();
503 if (auto *Helper =
504 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
505 for (auto *I : Helper->getCustomFieldTypes()) /* custom fields */ {
506 // TargetOpenCLBlockHelp needs to make sure the struct is packed.
507 // If necessary, add padding fields to the custom fields.
508 unsigned Align = CGM.getDataLayout().getABITypeAlign(Ty: I).value();
509 if (BlockAlign < Align)
510 BlockAlign = Align;
511 assert(Offset % Align == 0);
512 Offset += CGM.getDataLayout().getTypeAllocSize(Ty: I);
513 elementTypes.push_back(Elt: I);
514 }
515 }
516 info.BlockAlign = CharUnits::fromQuantity(Quantity: BlockAlign);
517 info.BlockSize = CharUnits::fromQuantity(Quantity: Offset);
518 } else {
519 // The header is basically 'struct { void *; int; int; void *; void *; }'.
520 // Assert that the struct is packed.
521 assert(CGM.getIntSize() <= CGM.getPointerSize());
522 assert(CGM.getIntAlign() <= CGM.getPointerAlign());
523 assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));
524 info.BlockAlign = CGM.getPointerAlign();
525 info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();
526 elementTypes.push_back(Elt: CGM.VoidPtrTy);
527 elementTypes.push_back(Elt: CGM.IntTy);
528 elementTypes.push_back(Elt: CGM.IntTy);
529 elementTypes.push_back(Elt: CGM.VoidPtrTy);
530 elementTypes.push_back(Elt: CGM.getBlockDescriptorType());
531 }
532}
533
534static QualType getCaptureFieldType(const CodeGenFunction &CGF,
535 const BlockDecl::Capture &CI) {
536 const VarDecl *VD = CI.getVariable();
537
538 // If the variable is captured by an enclosing block or lambda expression,
539 // use the type of the capture field.
540 if (CGF.BlockInfo && CI.isNested())
541 return CGF.BlockInfo->getCapture(var: VD).fieldType();
542 if (auto *FD = CGF.LambdaCaptureFields.lookup(Val: VD))
543 return FD->getType();
544 // If the captured variable is a non-escaping __block variable, the field
545 // type is the reference type. If the variable is a __block variable that
546 // already has a reference type, the field type is the variable's type.
547 return VD->isNonEscapingByref() ?
548 CGF.getContext().getLValueReferenceType(T: VD->getType()) : VD->getType();
549}
550
551/// Compute the layout of the given block. Attempts to lay the block
552/// out with minimal space requirements.
553static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
554 CGBlockInfo &info) {
555 ASTContext &C = CGM.getContext();
556 const BlockDecl *block = info.getBlockDecl();
557
558 SmallVector<llvm::Type*, 8> elementTypes;
559 initializeForBlockHeader(CGM, info, elementTypes);
560 bool hasNonConstantCustomFields = false;
561 if (auto *OpenCLHelper =
562 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper())
563 hasNonConstantCustomFields =
564 !OpenCLHelper->areAllCustomFieldValuesConstant(Info: info);
565 if (!block->hasCaptures() && !hasNonConstantCustomFields) {
566 info.StructureType =
567 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
568 info.CanBeGlobal = true;
569 return;
570 }
571 else if (C.getLangOpts().ObjC &&
572 CGM.getLangOpts().getGC() == LangOptions::NonGC)
573 info.HasCapturedVariableLayout = true;
574
575 if (block->doesNotEscape())
576 info.NoEscape = true;
577
578 // Collect the layout chunks.
579 SmallVector<BlockLayoutChunk, 16> layout;
580 layout.reserve(N: block->capturesCXXThis() +
581 (block->capture_end() - block->capture_begin()));
582
583 CharUnits maxFieldAlign;
584
585 // First, 'this'.
586 if (block->capturesCXXThis()) {
587 assert(CGF && isa_and_nonnull<CXXMethodDecl>(CGF->CurFuncDecl) &&
588 "Can't capture 'this' outside a method");
589 QualType thisType = cast<CXXMethodDecl>(Val: CGF->CurFuncDecl)->getThisType();
590
591 // Theoretically, this could be in a different address space, so
592 // don't assume standard pointer size/align.
593 llvm::Type *llvmType = CGM.getTypes().ConvertType(T: thisType);
594 auto TInfo = CGM.getContext().getTypeInfoInChars(T: thisType);
595 maxFieldAlign = std::max(a: maxFieldAlign, b: TInfo.Align);
596
597 addBlockLayout(align: TInfo.Align, size: TInfo.Width, capture: nullptr, type: llvmType, fieldType: thisType,
598 Layout&: layout, Info&: info, CGM);
599 }
600
601 // Next, all the block captures.
602 for (const auto &CI : block->captures()) {
603 const VarDecl *variable = CI.getVariable();
604
605 if (CI.isEscapingByref()) {
606 // Just use void* instead of a pointer to the byref type.
607 CharUnits align = CGM.getPointerAlign();
608 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
609
610 // Since a __block variable cannot be captured by lambdas, its type and
611 // the capture field type should always match.
612 assert(CGF && getCaptureFieldType(*CGF, CI) == variable->getType() &&
613 "capture type differs from the variable type");
614 addBlockLayout(align, size: CGM.getPointerSize(), capture: &CI, type: CGM.VoidPtrTy,
615 fieldType: variable->getType(), Layout&: layout, Info&: info, CGM);
616 continue;
617 }
618
619 // Otherwise, build a layout chunk with the size and alignment of
620 // the declaration.
621 if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, var: variable)) {
622 info.SortedCaptures.push_back(
623 Elt: CGBlockInfo::Capture::makeConstant(value: constant, Cap: &CI));
624 continue;
625 }
626
627 QualType VT = getCaptureFieldType(CGF: *CGF, CI);
628
629 if (CGM.getLangOpts().CPlusPlus)
630 if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl())
631 if (CI.hasCopyExpr() || !record->hasTrivialDestructor()) {
632 info.HasCXXObject = true;
633 if (!record->isExternallyVisible())
634 info.CapturesNonExternalType = true;
635 }
636
637 CharUnits size = C.getTypeSizeInChars(T: VT);
638 CharUnits align = C.getDeclAlign(D: variable);
639
640 maxFieldAlign = std::max(a: maxFieldAlign, b: align);
641
642 llvm::Type *llvmType =
643 CGM.getTypes().ConvertTypeForMem(T: VT);
644
645 addBlockLayout(align, size, capture: &CI, type: llvmType, fieldType: VT, Layout&: layout, Info&: info, CGM);
646 }
647
648 // If that was everything, we're done here.
649 if (layout.empty()) {
650 info.StructureType =
651 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
652 info.CanBeGlobal = true;
653 info.buildCaptureMap();
654 return;
655 }
656
657 // Sort the layout by alignment. We have to use a stable sort here
658 // to get reproducible results. There should probably be an
659 // llvm::array_pod_stable_sort.
660 llvm::stable_sort(Range&: layout);
661
662 // Needed for blocks layout info.
663 info.BlockHeaderForcedGapOffset = info.BlockSize;
664 info.BlockHeaderForcedGapSize = CharUnits::Zero();
665
666 CharUnits &blockSize = info.BlockSize;
667 info.BlockAlign = std::max(a: maxFieldAlign, b: info.BlockAlign);
668
669 // Assuming that the first byte in the header is maximally aligned,
670 // get the alignment of the first byte following the header.
671 CharUnits endAlign = getLowBit(v: blockSize);
672
673 // If the end of the header isn't satisfactorily aligned for the
674 // maximum thing, look for things that are okay with the header-end
675 // alignment, and keep appending them until we get something that's
676 // aligned right. This algorithm is only guaranteed optimal if
677 // that condition is satisfied at some point; otherwise we can get
678 // things like:
679 // header // next byte has alignment 4
680 // something_with_size_5; // next byte has alignment 1
681 // something_with_alignment_8;
682 // which has 7 bytes of padding, as opposed to the naive solution
683 // which might have less (?).
684 if (endAlign < maxFieldAlign) {
685 SmallVectorImpl<BlockLayoutChunk>::iterator
686 li = layout.begin() + 1, le = layout.end();
687
688 // Look for something that the header end is already
689 // satisfactorily aligned for.
690 for (; li != le && endAlign < li->Alignment; ++li)
691 ;
692
693 // If we found something that's naturally aligned for the end of
694 // the header, keep adding things...
695 if (li != le) {
696 SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
697 for (; li != le; ++li) {
698 assert(endAlign >= li->Alignment);
699
700 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
701 elementTypes.push_back(Elt: li->Type);
702 blockSize += li->Size;
703 endAlign = getLowBit(v: blockSize);
704
705 // ...until we get to the alignment of the maximum field.
706 if (endAlign >= maxFieldAlign) {
707 ++li;
708 break;
709 }
710 }
711 // Don't re-append everything we just appended.
712 layout.erase(CS: first, CE: li);
713 }
714 }
715
716 assert(endAlign == getLowBit(blockSize));
717
718 // At this point, we just have to add padding if the end align still
719 // isn't aligned right.
720 if (endAlign < maxFieldAlign) {
721 CharUnits newBlockSize = blockSize.alignTo(Align: maxFieldAlign);
722 CharUnits padding = newBlockSize - blockSize;
723
724 // If we haven't yet added any fields, remember that there was an
725 // initial gap; this need to go into the block layout bit map.
726 if (blockSize == info.BlockHeaderForcedGapOffset) {
727 info.BlockHeaderForcedGapSize = padding;
728 }
729
730 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
731 NumElements: padding.getQuantity()));
732 blockSize = newBlockSize;
733 endAlign = getLowBit(v: blockSize); // might be > maxFieldAlign
734 }
735
736 assert(endAlign >= maxFieldAlign);
737 assert(endAlign == getLowBit(blockSize));
738 // Slam everything else on now. This works because they have
739 // strictly decreasing alignment and we expect that size is always a
740 // multiple of alignment.
741 for (SmallVectorImpl<BlockLayoutChunk>::iterator
742 li = layout.begin(), le = layout.end(); li != le; ++li) {
743 if (endAlign < li->Alignment) {
744 // size may not be multiple of alignment. This can only happen with
745 // an over-aligned variable. We will be adding a padding field to
746 // make the size be multiple of alignment.
747 CharUnits padding = li->Alignment - endAlign;
748 elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
749 NumElements: padding.getQuantity()));
750 blockSize += padding;
751 endAlign = getLowBit(v: blockSize);
752 }
753 assert(endAlign >= li->Alignment);
754 li->setIndex(info, index: elementTypes.size(), offset: blockSize);
755 elementTypes.push_back(Elt: li->Type);
756 blockSize += li->Size;
757 endAlign = getLowBit(v: blockSize);
758 }
759
760 info.buildCaptureMap();
761 info.StructureType =
762 llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
763}
764
765/// Emit a block literal expression in the current function.
766llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
767 // If the block has no captures, we won't have a pre-computed
768 // layout for it.
769 if (!blockExpr->getBlockDecl()->hasCaptures())
770 // The block literal is emitted as a global variable, and the block invoke
771 // function has to be extracted from its initializer.
772 if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(BE: blockExpr))
773 return Block;
774
775 CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());
776 computeBlockInfo(CGM, CGF: this, info&: blockInfo);
777 blockInfo.BlockExpression = blockExpr;
778 if (!blockInfo.CanBeGlobal)
779 blockInfo.LocalAddress = CreateTempAlloca(Ty: blockInfo.StructureType,
780 align: blockInfo.BlockAlign, Name: "block");
781 return EmitBlockLiteral(Info: blockInfo);
782}
783
784llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
785 bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;
786 auto GenVoidPtrTy =
787 IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
788 LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
789 auto GenVoidPtrSize = CharUnits::fromQuantity(
790 Quantity: CGM.getTarget().getPointerWidth(AddrSpace: GenVoidPtrAddr) / 8);
791 // Using the computed layout, generate the actual block function.
792 bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();
793 CodeGenFunction BlockCGF{CGM, true};
794 BlockCGF.SanOpts = SanOpts;
795 auto *InvokeFn = BlockCGF.GenerateBlockFunction(
796 GD: CurGD, Info: blockInfo, ldm: LocalDeclMap, IsLambdaConversionToBlock: isLambdaConv, BuildGlobalBlock: blockInfo.CanBeGlobal);
797 auto *blockFn = llvm::ConstantExpr::getPointerCast(C: InvokeFn, Ty: GenVoidPtrTy);
798
799 // If there is nothing to capture, we can emit this as a global block.
800 if (blockInfo.CanBeGlobal)
801 return CGM.getAddrOfGlobalBlockIfEmitted(BE: blockInfo.BlockExpression);
802
803 // Otherwise, we have to emit this as a local block.
804
805 RawAddress blockAddr = blockInfo.LocalAddress;
806 assert(blockAddr.isValid() && "block has no address!");
807
808 llvm::Constant *isa;
809 llvm::Constant *descriptor;
810 BlockFlags flags;
811 if (!IsOpenCL) {
812 // If the block is non-escaping, set field 'isa 'to NSConcreteGlobalBlock
813 // and set the BLOCK_IS_GLOBAL bit of field 'flags'. Copying a non-escaping
814 // block just returns the original block and releasing it is a no-op.
815 llvm::Constant *blockISA = blockInfo.NoEscape
816 ? CGM.getNSConcreteGlobalBlock()
817 : CGM.getNSConcreteStackBlock();
818 isa = blockISA;
819
820 // Build the block descriptor.
821 descriptor = buildBlockDescriptor(CGM, blockInfo);
822
823 // Compute the initial on-stack block flags.
824 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
825 flags = BLOCK_HAS_SIGNATURE;
826 if (blockInfo.HasCapturedVariableLayout)
827 flags |= BLOCK_HAS_EXTENDED_LAYOUT;
828 if (blockInfo.NeedsCopyDispose)
829 flags |= BLOCK_HAS_COPY_DISPOSE;
830 if (blockInfo.HasCXXObject)
831 flags |= BLOCK_HAS_CXX_OBJ;
832 if (blockInfo.UsesStret)
833 flags |= BLOCK_USE_STRET;
834 if (blockInfo.NoEscape)
835 flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;
836 }
837
838 auto projectField = [&](unsigned index, const Twine &name) -> Address {
839 return Builder.CreateStructGEP(Addr: blockAddr, Index: index, Name: name);
840 };
841 auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) {
842 Builder.CreateStore(Val: value, Addr: projectField(index, name));
843 };
844
845 // Initialize the block header.
846 {
847 // We assume all the header fields are densely packed.
848 unsigned index = 0;
849 CharUnits offset;
850 auto addHeaderField = [&](llvm::Value *value, CharUnits size,
851 const Twine &name) {
852 storeField(value, index, name);
853 offset += size;
854 index++;
855 };
856
857 if (!IsOpenCL) {
858 addHeaderField(isa, getPointerSize(), "block.isa");
859 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
860 getIntSize(), "block.flags");
861 addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: 0), getIntSize(),
862 "block.reserved");
863 } else {
864 addHeaderField(
865 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockSize.getQuantity()),
866 getIntSize(), "block.size");
867 addHeaderField(
868 llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockAlign.getQuantity()),
869 getIntSize(), "block.align");
870 }
871 addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
872 if (!IsOpenCL)
873 addHeaderField(descriptor, getPointerSize(), "block.descriptor");
874 else if (auto *Helper =
875 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
876 for (auto I : Helper->getCustomFieldValues(CGF&: *this, Info: blockInfo)) {
877 addHeaderField(
878 I.first,
879 CharUnits::fromQuantity(
880 Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: I.first->getType())),
881 I.second);
882 }
883 }
884 }
885
886 // Finally, capture all the values into the block.
887 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
888
889 // First, 'this'.
890 if (blockDecl->capturesCXXThis()) {
891 Address addr =
892 projectField(blockInfo.CXXThisIndex, "block.captured-this.addr");
893 Builder.CreateStore(Val: LoadCXXThis(), Addr: addr);
894 }
895
896 // Next, captured variables.
897 for (const auto &CI : blockDecl->captures()) {
898 const VarDecl *variable = CI.getVariable();
899 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
900
901 // Ignore constant captures.
902 if (capture.isConstant()) continue;
903
904 QualType type = capture.fieldType();
905
906 // This will be a [[type]]*, except that a byref entry will just be
907 // an i8**.
908 Address blockField = projectField(capture.getIndex(), "block.captured");
909
910 // Compute the address of the thing we're going to move into the
911 // block literal.
912 Address src = Address::invalid();
913
914 if (blockDecl->isConversionFromLambda()) {
915 // The lambda capture in a lambda's conversion-to-block-pointer is
916 // special; we'll simply emit it directly.
917 src = Address::invalid();
918 } else if (CI.isEscapingByref()) {
919 if (BlockInfo && CI.isNested()) {
920 // We need to use the capture from the enclosing block.
921 const CGBlockInfo::Capture &enclosingCapture =
922 BlockInfo->getCapture(var: variable);
923
924 // This is a [[type]]*, except that a byref entry will just be an i8**.
925 src = Builder.CreateStructGEP(Addr: LoadBlockStruct(),
926 Index: enclosingCapture.getIndex(),
927 Name: "block.capture.addr");
928 } else {
929 auto I = LocalDeclMap.find(Val: variable);
930 assert(I != LocalDeclMap.end());
931 src = I->second;
932 }
933 } else {
934 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
935 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
936 type.getNonReferenceType(), VK_LValue,
937 SourceLocation());
938 src = EmitDeclRefLValue(E: &declRef).getAddress();
939 };
940
941 // For byrefs, we just write the pointer to the byref struct into
942 // the block field. There's no need to chase the forwarding
943 // pointer at this point, since we're building something that will
944 // live a shorter life than the stack byref anyway.
945 if (CI.isEscapingByref()) {
946 // Get a void* that points to the byref struct.
947 llvm::Value *byrefPointer;
948 if (CI.isNested())
949 byrefPointer = Builder.CreateLoad(Addr: src, Name: "byref.capture");
950 else
951 byrefPointer = src.emitRawPointer(CGF&: *this);
952
953 // Write that void* into the capture field.
954 Builder.CreateStore(Val: byrefPointer, Addr: blockField);
955
956 // If we have a copy constructor, evaluate that into the block field.
957 } else if (const Expr *copyExpr = CI.getCopyExpr()) {
958 if (blockDecl->isConversionFromLambda()) {
959 // If we have a lambda conversion, emit the expression
960 // directly into the block instead.
961 AggValueSlot Slot =
962 AggValueSlot::forAddr(addr: blockField, quals: Qualifiers(),
963 isDestructed: AggValueSlot::IsDestructed,
964 needsGC: AggValueSlot::DoesNotNeedGCBarriers,
965 isAliased: AggValueSlot::IsNotAliased,
966 mayOverlap: AggValueSlot::DoesNotOverlap);
967 EmitAggExpr(E: copyExpr, AS: Slot);
968 } else {
969 EmitSynthesizedCXXCopyCtor(Dest: blockField, Src: src, Exp: copyExpr);
970 }
971
972 // If it's a reference variable, copy the reference into the block field.
973 } else if (type->getAs<ReferenceType>()) {
974 Builder.CreateStore(Val: src.emitRawPointer(CGF&: *this), Addr: blockField);
975
976 // If type is const-qualified, copy the value into the block field.
977 } else if (type.isConstQualified() &&
978 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
979 CGM.getCodeGenOpts().OptimizationLevel != 0) {
980 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "captured");
981 Builder.CreateStore(Val: value, Addr: blockField);
982
983 // If this is an ARC __strong block-pointer variable, don't do a
984 // block copy.
985 //
986 // TODO: this can be generalized into the normal initialization logic:
987 // we should never need to do a block-copy when initializing a local
988 // variable, because the local variable's lifetime should be strictly
989 // contained within the stack block's.
990 } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&
991 type->isBlockPointerType()) {
992 // Load the block and do a simple retain.
993 llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "block.captured_block");
994 value = EmitARCRetainNonBlock(value);
995
996 // Do a primitive store to the block field.
997 Builder.CreateStore(Val: value, Addr: blockField);
998
999 // Otherwise, fake up a POD copy into the block field.
1000 } else {
1001 // Fake up a new variable so that EmitScalarInit doesn't think
1002 // we're referring to the variable in its own initializer.
1003 ImplicitParamDecl BlockFieldPseudoVar(getContext(), type,
1004 ImplicitParamKind::Other);
1005
1006 // We use one of these or the other depending on whether the
1007 // reference is nested.
1008 DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
1009 /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
1010 type, VK_LValue, SourceLocation());
1011
1012 ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
1013 &declRef, VK_PRValue, FPOptionsOverride());
1014 // FIXME: Pass a specific location for the expr init so that the store is
1015 // attributed to a reasonable location - otherwise it may be attributed to
1016 // locations of subexpressions in the initialization.
1017 EmitExprAsInit(init: &l2r, D: &BlockFieldPseudoVar,
1018 lvalue: MakeAddrLValue(Addr: blockField, T: type, Source: AlignmentSource::Decl),
1019 /*captured by init*/ capturedByInit: false);
1020 }
1021
1022 // Push a cleanup for the capture if necessary.
1023 if (!blockInfo.NoEscape && !blockInfo.NeedsCopyDispose)
1024 continue;
1025
1026 // Ignore __block captures; there's nothing special in the on-stack block
1027 // that we need to do for them.
1028 if (CI.isByRef())
1029 continue;
1030
1031 // Ignore objects that aren't destructed.
1032 QualType::DestructionKind dtorKind = type.isDestructedType();
1033 if (dtorKind == QualType::DK_none)
1034 continue;
1035
1036 CodeGenFunction::Destroyer *destroyer;
1037
1038 // Block captures count as local values and have imprecise semantics.
1039 // They also can't be arrays, so need to worry about that.
1040 //
1041 // For const-qualified captures, emit clang.arc.use to ensure the captured
1042 // object doesn't get released while we are still depending on its validity
1043 // within the block.
1044 if (type.isConstQualified() &&
1045 type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1046 CGM.getCodeGenOpts().OptimizationLevel != 0) {
1047 assert(CGM.getLangOpts().ObjCAutoRefCount &&
1048 "expected ObjC ARC to be enabled");
1049 destroyer = emitARCIntrinsicUse;
1050 } else if (dtorKind == QualType::DK_objc_strong_lifetime) {
1051 destroyer = destroyARCStrongImprecise;
1052 } else {
1053 destroyer = getDestroyer(destructionKind: dtorKind);
1054 }
1055
1056 CleanupKind cleanupKind = NormalCleanup;
1057 bool useArrayEHCleanup = needsEHCleanup(kind: dtorKind);
1058 if (useArrayEHCleanup)
1059 cleanupKind = NormalAndEHCleanup;
1060
1061 // Extend the lifetime of the capture to the end of the scope enclosing the
1062 // block expression except when the block decl is in the list of RetExpr's
1063 // cleanup objects, in which case its lifetime ends after the full
1064 // expression.
1065 auto IsBlockDeclInRetExpr = [&]() {
1066 auto *EWC = llvm::dyn_cast_or_null<ExprWithCleanups>(Val: RetExpr);
1067 if (EWC)
1068 for (auto &C : EWC->getObjects())
1069 if (auto *BD = C.dyn_cast<BlockDecl *>())
1070 if (BD == blockDecl)
1071 return true;
1072 return false;
1073 };
1074
1075 if (IsBlockDeclInRetExpr())
1076 pushDestroy(kind: cleanupKind, addr: blockField, type, destroyer, useEHCleanupForArray: useArrayEHCleanup);
1077 else
1078 pushLifetimeExtendedDestroy(kind: cleanupKind, addr: blockField, type, destroyer,
1079 useEHCleanupForArray: useArrayEHCleanup);
1080 }
1081
1082 // Cast to the converted block-pointer type, which happens (somewhat
1083 // unfortunately) to be a pointer to function type.
1084 llvm::Value *result = Builder.CreatePointerCast(
1085 V: blockAddr.getPointer(), DestTy: ConvertType(T: blockInfo.getBlockExpr()->getType()));
1086
1087 if (IsOpenCL) {
1088 CGM.getOpenCLRuntime().recordBlockInfo(E: blockInfo.BlockExpression, InvokeF: InvokeFn,
1089 Block: result, BlockTy: blockInfo.StructureType);
1090 }
1091
1092 return result;
1093}
1094
1095
1096llvm::Type *CodeGenModule::getBlockDescriptorType() {
1097 if (BlockDescriptorType)
1098 return BlockDescriptorType;
1099
1100 unsigned AddrSpace = 0;
1101 if (getLangOpts().OpenCL)
1102 AddrSpace = getContext().getTargetAddressSpace(AS: LangAS::opencl_constant);
1103 BlockDescriptorType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: AddrSpace);
1104 return BlockDescriptorType;
1105}
1106
1107llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
1108 if (GenericBlockLiteralType)
1109 return GenericBlockLiteralType;
1110
1111 llvm::Type *BlockDescPtrTy = getBlockDescriptorType();
1112
1113 if (getLangOpts().OpenCL) {
1114 // struct __opencl_block_literal_generic {
1115 // int __size;
1116 // int __align;
1117 // __generic void *__invoke;
1118 // /* custom fields */
1119 // };
1120 SmallVector<llvm::Type *, 8> StructFields(
1121 {IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});
1122 if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1123 llvm::append_range(C&: StructFields, R: Helper->getCustomFieldTypes());
1124 }
1125 GenericBlockLiteralType = llvm::StructType::create(
1126 Elements: StructFields, Name: "struct.__opencl_block_literal_generic");
1127 } else {
1128 // struct __block_literal_generic {
1129 // void *__isa;
1130 // int __flags;
1131 // int __reserved;
1132 // void (*__invoke)(void *);
1133 // struct __block_descriptor *__descriptor;
1134 // };
1135 GenericBlockLiteralType =
1136 llvm::StructType::create(Name: "struct.__block_literal_generic", elt1: VoidPtrTy,
1137 elts: IntTy, elts: IntTy, elts: VoidPtrTy, elts: BlockDescPtrTy);
1138 }
1139
1140 return GenericBlockLiteralType;
1141}
1142
1143RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
1144 ReturnValueSlot ReturnValue,
1145 llvm::CallBase **CallOrInvoke) {
1146 const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();
1147 llvm::Value *BlockPtr = EmitScalarExpr(E: E->getCallee());
1148 llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();
1149 llvm::Value *Func = nullptr;
1150 QualType FnType = BPT->getPointeeType();
1151 ASTContext &Ctx = getContext();
1152 CallArgList Args;
1153
1154 if (getLangOpts().OpenCL) {
1155 // For OpenCL, BlockPtr is already casted to generic block literal.
1156
1157 // First argument of a block call is a generic block literal casted to
1158 // generic void pointer, i.e. i8 addrspace(4)*
1159 llvm::Type *GenericVoidPtrTy =
1160 CGM.getOpenCLRuntime().getGenericVoidPointerType();
1161 llvm::Value *BlockDescriptor = Builder.CreatePointerCast(
1162 V: BlockPtr, DestTy: GenericVoidPtrTy);
1163 QualType VoidPtrQualTy = Ctx.getPointerType(
1164 T: Ctx.getAddrSpaceQualType(T: Ctx.VoidTy, AddressSpace: LangAS::opencl_generic));
1165 Args.add(rvalue: RValue::get(V: BlockDescriptor), type: VoidPtrQualTy);
1166 // And the rest of the arguments.
1167 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1168
1169 // We *can* call the block directly unless it is a function argument.
1170 if (!isa<ParmVarDecl>(Val: E->getCalleeDecl()))
1171 Func = CGM.getOpenCLRuntime().getInvokeFunction(E: E->getCallee());
1172 else {
1173 llvm::Value *FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 2);
1174 Func = Builder.CreateAlignedLoad(Ty: GenericVoidPtrTy, Addr: FuncPtr,
1175 Align: getPointerAlign());
1176 }
1177 } else {
1178 // Bitcast the block literal to a generic block literal.
1179 BlockPtr =
1180 Builder.CreatePointerCast(V: BlockPtr, DestTy: UnqualPtrTy, Name: "block.literal");
1181 // Get pointer to the block invoke function
1182 llvm::Value *FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 3);
1183
1184 // First argument is a block literal casted to a void pointer
1185 BlockPtr = Builder.CreatePointerCast(V: BlockPtr, DestTy: VoidPtrTy);
1186 Args.add(rvalue: RValue::get(V: BlockPtr), type: Ctx.VoidPtrTy);
1187 // And the rest of the arguments.
1188 EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1189
1190 // Load the function.
1191 Func = Builder.CreateAlignedLoad(Ty: VoidPtrTy, Addr: FuncPtr, Align: getPointerAlign());
1192 }
1193
1194 const FunctionType *FuncTy = FnType->castAs<FunctionType>();
1195 const CGFunctionInfo &FnInfo =
1196 CGM.getTypes().arrangeBlockFunctionCall(args: Args, type: FuncTy);
1197
1198 // Prepare the callee.
1199 CGCallee Callee(CGCalleeInfo(), Func);
1200
1201 // And call the block.
1202 return EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke);
1203}
1204
1205Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {
1206 assert(BlockInfo && "evaluating block ref without block information?");
1207 const CGBlockInfo::Capture &capture = BlockInfo->getCapture(var: variable);
1208
1209 // Handle constant captures.
1210 if (capture.isConstant()) return LocalDeclMap.find(Val: variable)->second;
1211
1212 Address addr = Builder.CreateStructGEP(Addr: LoadBlockStruct(), Index: capture.getIndex(),
1213 Name: "block.capture.addr");
1214
1215 if (variable->isEscapingByref()) {
1216 // addr should be a void** right now. Load, then cast the result
1217 // to byref*.
1218
1219 auto &byrefInfo = getBlockByrefInfo(var: variable);
1220 addr = Address(Builder.CreateLoad(Addr: addr), byrefInfo.Type,
1221 byrefInfo.ByrefAlignment);
1222
1223 addr = emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, /*follow*/ followForward: true,
1224 name: variable->getName());
1225 }
1226
1227 assert((!variable->isNonEscapingByref() ||
1228 capture.fieldType()->isReferenceType()) &&
1229 "the capture field of a non-escaping variable should have a "
1230 "reference type");
1231 if (capture.fieldType()->isReferenceType())
1232 addr = EmitLoadOfReference(RefLVal: MakeAddrLValue(Addr: addr, T: capture.fieldType()));
1233
1234 return addr;
1235}
1236
1237void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE,
1238 llvm::Constant *Addr) {
1239 bool Ok = EmittedGlobalBlocks.insert(KV: std::make_pair(x&: BE, y&: Addr)).second;
1240 (void)Ok;
1241 assert(Ok && "Trying to replace an already-existing global block!");
1242}
1243
1244llvm::Constant *
1245CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE,
1246 StringRef Name) {
1247 if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE))
1248 return Block;
1249
1250 CGBlockInfo blockInfo(BE->getBlockDecl(), Name);
1251 blockInfo.BlockExpression = BE;
1252
1253 // Compute information about the layout, etc., of this block.
1254 computeBlockInfo(CGM&: *this, CGF: nullptr, info&: blockInfo);
1255
1256 // Using that metadata, generate the actual block function.
1257 {
1258 CodeGenFunction::DeclMapTy LocalDeclMap;
1259 CodeGenFunction(*this).GenerateBlockFunction(
1260 GD: GlobalDecl(), Info: blockInfo, ldm: LocalDeclMap,
1261 /*IsLambdaConversionToBlock*/ false, /*BuildGlobalBlock*/ true);
1262 }
1263
1264 return getAddrOfGlobalBlockIfEmitted(BE);
1265}
1266
1267static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
1268 const CGBlockInfo &blockInfo,
1269 llvm::Constant *blockFn) {
1270 assert(blockInfo.CanBeGlobal);
1271 // Callers should detect this case on their own: calling this function
1272 // generally requires computing layout information, which is a waste of time
1273 // if we've already emitted this block.
1274 assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) &&
1275 "Refusing to re-emit a global block.");
1276
1277 // Generate the constants for the block literal initializer.
1278 ConstantInitBuilder builder(CGM);
1279 auto fields = builder.beginStruct();
1280
1281 bool IsOpenCL = CGM.getLangOpts().OpenCL;
1282 bool IsWindows = CGM.getTarget().getTriple().isOSWindows();
1283 if (!IsOpenCL) {
1284 // isa
1285 if (IsWindows)
1286 fields.addNullPointer(ptrTy: CGM.Int8PtrPtrTy);
1287 else
1288 fields.add(value: CGM.getNSConcreteGlobalBlock());
1289
1290 // __flags
1291 BlockFlags flags = BLOCK_IS_GLOBAL;
1292 if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
1293 flags |= BLOCK_HAS_SIGNATURE;
1294 if (blockInfo.UsesStret)
1295 flags |= BLOCK_USE_STRET;
1296
1297 fields.addInt(intTy: CGM.IntTy, value: flags.getBitMask());
1298
1299 // Reserved
1300 fields.addInt(intTy: CGM.IntTy, value: 0);
1301 } else {
1302 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockSize.getQuantity());
1303 fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockAlign.getQuantity());
1304 }
1305
1306 // Function
1307 fields.add(value: blockFn);
1308
1309 if (!IsOpenCL) {
1310 // Descriptor
1311 fields.add(value: buildBlockDescriptor(CGM, blockInfo));
1312 } else if (auto *Helper =
1313 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1314 for (auto *I : Helper->getCustomFieldValues(CGM, Info: blockInfo)) {
1315 fields.add(value: I);
1316 }
1317 }
1318
1319 unsigned AddrSpace = 0;
1320 if (CGM.getContext().getLangOpts().OpenCL)
1321 AddrSpace = CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
1322
1323 llvm::GlobalVariable *literal = fields.finishAndCreateGlobal(
1324 args: "__block_literal_global", args: blockInfo.BlockAlign,
1325 /*constant*/ args: !IsWindows, args: llvm::GlobalVariable::InternalLinkage, args&: AddrSpace);
1326
1327 literal->addAttribute(Kind: "objc_arc_inert");
1328
1329 // Windows does not allow globals to be initialised to point to globals in
1330 // different DLLs. Any such variables must run code to initialise them.
1331 if (IsWindows) {
1332 auto *Init = llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy,
1333 isVarArg: {}), Linkage: llvm::GlobalValue::InternalLinkage, N: ".block_isa_init",
1334 M: &CGM.getModule());
1335 llvm::IRBuilder<> b(llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "entry",
1336 Parent: Init));
1337 b.CreateAlignedStore(Val: CGM.getNSConcreteGlobalBlock(),
1338 Ptr: b.CreateStructGEP(Ty: literal->getValueType(), Ptr: literal, Idx: 0),
1339 Align: CGM.getPointerAlign().getAsAlign());
1340 b.CreateRetVoid();
1341 // We can't use the normal LLVM global initialisation array, because we
1342 // need to specify that this runs early in library initialisation.
1343 auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
1344 /*isConstant*/true, llvm::GlobalValue::InternalLinkage,
1345 Init, ".block_isa_init_ptr");
1346 InitVar->setSection(".CRT$XCLa");
1347 CGM.addUsedGlobal(GV: InitVar);
1348 }
1349
1350 // Return a constant of the appropriately-casted type.
1351 llvm::Type *RequiredType =
1352 CGM.getTypes().ConvertType(T: blockInfo.getBlockExpr()->getType());
1353 llvm::Constant *Result =
1354 llvm::ConstantExpr::getPointerCast(C: literal, Ty: RequiredType);
1355 CGM.setAddrOfGlobalBlock(BE: blockInfo.BlockExpression, Addr: Result);
1356 if (CGM.getContext().getLangOpts().OpenCL)
1357 CGM.getOpenCLRuntime().recordBlockInfo(
1358 E: blockInfo.BlockExpression,
1359 InvokeF: cast<llvm::Function>(Val: blockFn->stripPointerCasts()), Block: Result,
1360 BlockTy: literal->getValueType());
1361 return Result;
1362}
1363
1364void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
1365 unsigned argNum,
1366 llvm::Value *arg) {
1367 assert(BlockInfo && "not emitting prologue of block invocation function?!");
1368
1369 // Allocate a stack slot like for any local variable to guarantee optimal
1370 // debug info at -O0. The mem2reg pass will eliminate it when optimizing.
1371 RawAddress alloc = CreateMemTemp(T: D->getType(), Name: D->getName() + ".addr");
1372 Builder.CreateStore(Val: arg, Addr: alloc);
1373 if (CGDebugInfo *DI = getDebugInfo()) {
1374 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1375 DI->setLocation(D->getLocation());
1376 DI->EmitDeclareOfBlockLiteralArgVariable(
1377 block: *BlockInfo, Name: D->getName(), ArgNo: argNum,
1378 LocalAddr: cast<llvm::AllocaInst>(Val: alloc.getPointer()->stripPointerCasts()),
1379 Builder);
1380 }
1381 }
1382
1383 SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc();
1384 ApplyDebugLocation Scope(*this, StartLoc);
1385
1386 // Instead of messing around with LocalDeclMap, just set the value
1387 // directly as BlockPointer.
1388 BlockPointer = Builder.CreatePointerCast(
1389 V: arg,
1390 DestTy: llvm::PointerType::get(
1391 C&: getLLVMContext(),
1392 AddressSpace: getContext().getLangOpts().OpenCL
1393 ? getContext().getTargetAddressSpace(AS: LangAS::opencl_generic)
1394 : 0),
1395 Name: "block");
1396}
1397
1398Address CodeGenFunction::LoadBlockStruct() {
1399 assert(BlockInfo && "not in a block invocation function!");
1400 assert(BlockPointer && "no block pointer set!");
1401 return Address(BlockPointer, BlockInfo->StructureType, BlockInfo->BlockAlign);
1402}
1403
1404llvm::Function *CodeGenFunction::GenerateBlockFunction(
1405 GlobalDecl GD, const CGBlockInfo &blockInfo, const DeclMapTy &ldm,
1406 bool IsLambdaConversionToBlock, bool BuildGlobalBlock) {
1407 const BlockDecl *blockDecl = blockInfo.getBlockDecl();
1408
1409 CurGD = GD;
1410
1411 CurEHLocation = blockInfo.getBlockExpr()->getEndLoc();
1412
1413 BlockInfo = &blockInfo;
1414
1415 // Arrange for local static and local extern declarations to appear
1416 // to be local to this function as well, in case they're directly
1417 // referenced in a block.
1418 for (const auto &KV : ldm) {
1419 const auto *var = dyn_cast<VarDecl>(Val: KV.first);
1420 if (var && !var->hasLocalStorage())
1421 setAddrOfLocalVar(VD: var, Addr: KV.second);
1422 }
1423
1424 // Begin building the function declaration.
1425
1426 // Build the argument list.
1427 FunctionArgList args;
1428
1429 // The first argument is the block pointer. Just take it as a void*
1430 // and cast it later.
1431 QualType selfTy = getContext().VoidPtrTy;
1432
1433 // For OpenCL passed block pointer can be private AS local variable or
1434 // global AS program scope variable (for the case with and without captures).
1435 // Generic AS is used therefore to be able to accommodate both private and
1436 // generic AS in one implementation.
1437 if (getLangOpts().OpenCL)
1438 selfTy = getContext().getPointerType(T: getContext().getAddrSpaceQualType(
1439 T: getContext().VoidTy, AddressSpace: LangAS::opencl_generic));
1440
1441 const IdentifierInfo *II = &CGM.getContext().Idents.get(Name: ".block_descriptor");
1442
1443 ImplicitParamDecl SelfDecl(getContext(), const_cast<BlockDecl *>(blockDecl),
1444 SourceLocation(), II, selfTy,
1445 ImplicitParamKind::ObjCSelf);
1446 args.push_back(Elt: &SelfDecl);
1447
1448 // Now add the rest of the parameters.
1449 args.append(in_start: blockDecl->param_begin(), in_end: blockDecl->param_end());
1450
1451 // Create the function declaration.
1452 const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();
1453 const CGFunctionInfo &fnInfo =
1454 CGM.getTypes().arrangeBlockFunctionDeclaration(type: fnType, args);
1455 if (CGM.ReturnSlotInterferesWithArgs(FI: fnInfo))
1456 blockInfo.UsesStret = true;
1457
1458 llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(Info: fnInfo);
1459
1460 StringRef name = CGM.getBlockMangledName(GD, BD: blockDecl);
1461 llvm::Function *fn = llvm::Function::Create(
1462 Ty: fnLLVMType, Linkage: llvm::GlobalValue::InternalLinkage, N: name, M: &CGM.getModule());
1463 CGM.SetInternalFunctionAttributes(GD: blockDecl, F: fn, FI: fnInfo);
1464
1465 if (BuildGlobalBlock) {
1466 auto GenVoidPtrTy = getContext().getLangOpts().OpenCL
1467 ? CGM.getOpenCLRuntime().getGenericVoidPointerType()
1468 : VoidPtrTy;
1469 buildGlobalBlock(CGM, blockInfo,
1470 blockFn: llvm::ConstantExpr::getPointerCast(C: fn, Ty: GenVoidPtrTy));
1471 }
1472
1473 // Begin generating the function.
1474 StartFunction(GD: blockDecl, RetTy: fnType->getReturnType(), Fn: fn, FnInfo: fnInfo, Args: args,
1475 Loc: blockDecl->getLocation(),
1476 StartLoc: blockInfo.getBlockExpr()->getBody()->getBeginLoc());
1477
1478 // Okay. Undo some of what StartFunction did.
1479
1480 // At -O0 we generate an explicit alloca for the BlockPointer, so the RA
1481 // won't delete the dbg.declare intrinsics for captured variables.
1482 llvm::Value *BlockPointerDbgLoc = BlockPointer;
1483 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
1484 // Allocate a stack slot for it, so we can point the debugger to it
1485 Address Alloca = CreateTempAlloca(Ty: BlockPointer->getType(),
1486 align: getPointerAlign(),
1487 Name: "block.addr");
1488 // Set the DebugLocation to empty, so the store is recognized as a
1489 // frame setup instruction by llvm::DwarfDebug::beginFunction().
1490 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1491 Builder.CreateStore(Val: BlockPointer, Addr: Alloca);
1492 BlockPointerDbgLoc = Alloca.emitRawPointer(CGF&: *this);
1493 }
1494
1495 // If we have a C++ 'this' reference, go ahead and force it into
1496 // existence now.
1497 if (blockDecl->capturesCXXThis()) {
1498 Address addr = Builder.CreateStructGEP(
1499 Addr: LoadBlockStruct(), Index: blockInfo.CXXThisIndex, Name: "block.captured-this");
1500 CXXThisValue = Builder.CreateLoad(Addr: addr, Name: "this");
1501 }
1502
1503 // Also force all the constant captures.
1504 for (const auto &CI : blockDecl->captures()) {
1505 const VarDecl *variable = CI.getVariable();
1506 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1507 if (!capture.isConstant()) continue;
1508
1509 CharUnits align = getContext().getDeclAlign(D: variable);
1510 Address alloca =
1511 CreateMemTemp(T: variable->getType(), Align: align, Name: "block.captured-const");
1512
1513 Builder.CreateStore(Val: capture.getConstant(), Addr: alloca);
1514
1515 setAddrOfLocalVar(VD: variable, Addr: alloca);
1516 }
1517
1518 // Save a spot to insert the debug information for all the DeclRefExprs.
1519 llvm::BasicBlock *entry = Builder.GetInsertBlock();
1520 llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();
1521 --entry_ptr;
1522
1523 if (IsLambdaConversionToBlock)
1524 EmitLambdaBlockInvokeBody();
1525 else {
1526 PGO->assignRegionCounters(GD: GlobalDecl(blockDecl), Fn: fn);
1527 incrementProfileCounter(S: blockDecl->getBody());
1528 EmitStmt(S: blockDecl->getBody());
1529 }
1530
1531 // Remember where we were...
1532 llvm::BasicBlock *resume = Builder.GetInsertBlock();
1533
1534 // Go back to the entry.
1535 if (entry_ptr->getNextNonDebugInstruction())
1536 entry_ptr = entry_ptr->getNextNonDebugInstruction()->getIterator();
1537 else
1538 entry_ptr = entry->end();
1539 Builder.SetInsertPoint(TheBB: entry, IP: entry_ptr);
1540
1541 // Emit debug information for all the DeclRefExprs.
1542 // FIXME: also for 'this'
1543 if (CGDebugInfo *DI = getDebugInfo()) {
1544 for (const auto &CI : blockDecl->captures()) {
1545 const VarDecl *variable = CI.getVariable();
1546 DI->EmitLocation(Builder, Loc: variable->getLocation());
1547
1548 if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1549 const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1550 if (capture.isConstant()) {
1551 auto addr = LocalDeclMap.find(Val: variable)->second;
1552 (void)DI->EmitDeclareOfAutoVariable(
1553 Decl: variable, AI: addr.emitRawPointer(CGF&: *this), Builder);
1554 continue;
1555 }
1556
1557 DI->EmitDeclareOfBlockDeclRefVariable(
1558 variable, storage: BlockPointerDbgLoc, Builder, blockInfo,
1559 InsertPoint: entry_ptr == entry->end() ? nullptr : &*entry_ptr);
1560 }
1561 }
1562 // Recover location if it was changed in the above loop.
1563 DI->EmitLocation(Builder,
1564 Loc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1565 }
1566
1567 // And resume where we left off.
1568 if (resume == nullptr)
1569 Builder.ClearInsertionPoint();
1570 else
1571 Builder.SetInsertPoint(resume);
1572
1573 FinishFunction(EndLoc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1574
1575 return fn;
1576}
1577
1578static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
1579computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
1580 const LangOptions &LangOpts) {
1581 if (CI.getCopyExpr()) {
1582 assert(!CI.isByRef());
1583 // don't bother computing flags
1584 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
1585 }
1586 BlockFieldFlags Flags;
1587 if (CI.isEscapingByref()) {
1588 Flags = BLOCK_FIELD_IS_BYREF;
1589 if (T.isObjCGCWeak())
1590 Flags |= BLOCK_FIELD_IS_WEAK;
1591 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1592 }
1593
1594 if (T.hasAddressDiscriminatedPointerAuth())
1595 return std::make_pair(
1596 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth, y&: Flags);
1597
1598 Flags = BLOCK_FIELD_IS_OBJECT;
1599 bool isBlockPointer = T->isBlockPointerType();
1600 if (isBlockPointer)
1601 Flags = BLOCK_FIELD_IS_BLOCK;
1602
1603 switch (T.isNonTrivialToPrimitiveCopy()) {
1604 case QualType::PCK_Struct:
1605 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
1606 y: BlockFieldFlags());
1607 case QualType::PCK_ARCWeak:
1608 // We need to register __weak direct captures with the runtime.
1609 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak, y&: Flags);
1610 case QualType::PCK_ARCStrong:
1611 // We need to retain the copied value for __strong direct captures.
1612 // If it's a block pointer, we have to copy the block and assign that to
1613 // the destination pointer, so we might as well use _Block_object_assign.
1614 // Otherwise we can avoid that.
1615 return std::make_pair(x: !isBlockPointer ? BlockCaptureEntityKind::ARCStrong
1616 : BlockCaptureEntityKind::BlockObject,
1617 y&: Flags);
1618 case QualType::PCK_PtrAuth:
1619 return std::make_pair(
1620 x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth,
1621 y: BlockFieldFlags());
1622 case QualType::PCK_Trivial:
1623 case QualType::PCK_VolatileTrivial: {
1624 if (!T->isObjCRetainableType())
1625 // For all other types, the memcpy is fine.
1626 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1627
1628 // Honor the inert __unsafe_unretained qualifier, which doesn't actually
1629 // make it into the type system.
1630 if (T->isObjCInertUnsafeUnretainedType())
1631 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1632
1633 // Special rules for ARC captures:
1634 Qualifiers QS = T.getQualifiers();
1635
1636 // Non-ARC captures of retainable pointers are strong and
1637 // therefore require a call to _Block_object_assign.
1638 if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount)
1639 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1640
1641 // Otherwise the memcpy is fine.
1642 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1643 }
1644 }
1645 llvm_unreachable("after exhaustive PrimitiveCopyKind switch");
1646}
1647
1648namespace {
1649/// Release a __block variable.
1650struct CallBlockRelease final : EHScopeStack::Cleanup {
1651 Address Addr;
1652 BlockFieldFlags FieldFlags;
1653 bool LoadBlockVarAddr, CanThrow;
1654
1655 CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,
1656 bool CT)
1657 : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),
1658 CanThrow(CT) {}
1659
1660 void Emit(CodeGenFunction &CGF, Flags flags) override {
1661 llvm::Value *BlockVarAddr;
1662 if (LoadBlockVarAddr) {
1663 BlockVarAddr = CGF.Builder.CreateLoad(Addr);
1664 } else {
1665 BlockVarAddr = Addr.emitRawPointer(CGF);
1666 }
1667
1668 CGF.BuildBlockRelease(DeclPtr: BlockVarAddr, flags: FieldFlags, CanThrow);
1669 }
1670};
1671} // end anonymous namespace
1672
1673/// Check if \p T is a C++ class that has a destructor that can throw.
1674bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {
1675 if (const auto *RD = T->getAsCXXRecordDecl())
1676 if (const CXXDestructorDecl *DD = RD->getDestructor())
1677 return DD->getType()->castAs<FunctionProtoType>()->canThrow();
1678 return false;
1679}
1680
1681// Return a string that has the information about a capture.
1682static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
1683 CaptureStrKind StrKind,
1684 CharUnits BlockAlignment,
1685 CodeGenModule &CGM) {
1686 std::string Str;
1687 ASTContext &Ctx = CGM.getContext();
1688 const BlockDecl::Capture &CI = *Cap.Cap;
1689 QualType CaptureTy = CI.getVariable()->getType();
1690
1691 BlockCaptureEntityKind Kind;
1692 BlockFieldFlags Flags;
1693
1694 // CaptureStrKind::Merged should be passed only when the operations and the
1695 // flags are the same for copy and dispose.
1696 assert((StrKind != CaptureStrKind::Merged ||
1697 (Cap.CopyKind == Cap.DisposeKind &&
1698 Cap.CopyFlags == Cap.DisposeFlags)) &&
1699 "different operations and flags");
1700
1701 if (StrKind == CaptureStrKind::DisposeHelper) {
1702 Kind = Cap.DisposeKind;
1703 Flags = Cap.DisposeFlags;
1704 } else {
1705 Kind = Cap.CopyKind;
1706 Flags = Cap.CopyFlags;
1707 }
1708
1709 switch (Kind) {
1710 case BlockCaptureEntityKind::CXXRecord: {
1711 Str += "c";
1712 SmallString<256> TyStr;
1713 llvm::raw_svector_ostream Out(TyStr);
1714 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: CaptureTy, Out);
1715 Str += llvm::to_string(Value: TyStr.size()) + TyStr.c_str();
1716 break;
1717 }
1718 case BlockCaptureEntityKind::ARCWeak:
1719 Str += "w";
1720 break;
1721 case BlockCaptureEntityKind::ARCStrong:
1722 Str += "s";
1723 break;
1724 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1725 auto PtrAuth = CaptureTy.getPointerAuth();
1726 assert(PtrAuth && PtrAuth.isAddressDiscriminated());
1727 Str += "p" + llvm::to_string(Value: PtrAuth.getKey()) + "d" +
1728 llvm::to_string(Value: PtrAuth.getExtraDiscriminator());
1729 break;
1730 }
1731 case BlockCaptureEntityKind::BlockObject: {
1732 const VarDecl *Var = CI.getVariable();
1733 unsigned F = Flags.getBitMask();
1734 if (F & BLOCK_FIELD_IS_BYREF) {
1735 Str += "r";
1736 if (F & BLOCK_FIELD_IS_WEAK)
1737 Str += "w";
1738 else {
1739 // If CaptureStrKind::Merged is passed, check both the copy expression
1740 // and the destructor.
1741 if (StrKind != CaptureStrKind::DisposeHelper) {
1742 if (Ctx.getBlockVarCopyInit(VD: Var).canThrow())
1743 Str += "c";
1744 }
1745 if (StrKind != CaptureStrKind::CopyHelper) {
1746 if (CodeGenFunction::cxxDestructorCanThrow(T: CaptureTy))
1747 Str += "d";
1748 }
1749 }
1750 } else {
1751 assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");
1752 if (F == BLOCK_FIELD_IS_BLOCK)
1753 Str += "b";
1754 else
1755 Str += "o";
1756 }
1757 break;
1758 }
1759 case BlockCaptureEntityKind::NonTrivialCStruct: {
1760 bool IsVolatile = CaptureTy.isVolatileQualified();
1761 CharUnits Alignment = BlockAlignment.alignmentAtOffset(offset: Cap.getOffset());
1762
1763 Str += "n";
1764 std::string FuncStr;
1765 if (StrKind == CaptureStrKind::DisposeHelper)
1766 FuncStr = CodeGenFunction::getNonTrivialDestructorStr(
1767 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1768 else
1769 // If CaptureStrKind::Merged is passed, use the copy constructor string.
1770 // It has all the information that the destructor string has.
1771 FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr(
1772 QT: CaptureTy, Alignment, IsVolatile, Ctx);
1773 // The underscore is necessary here because non-trivial copy constructor
1774 // and destructor strings can start with a number.
1775 Str += llvm::to_string(Value: FuncStr.size()) + "_" + FuncStr;
1776 break;
1777 }
1778 case BlockCaptureEntityKind::None:
1779 break;
1780 }
1781
1782 return Str;
1783}
1784
1785static std::string getCopyDestroyHelperFuncName(
1786 const SmallVectorImpl<CGBlockInfo::Capture> &Captures,
1787 CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) {
1788 assert((StrKind == CaptureStrKind::CopyHelper ||
1789 StrKind == CaptureStrKind::DisposeHelper) &&
1790 "unexpected CaptureStrKind");
1791 std::string Name = StrKind == CaptureStrKind::CopyHelper
1792 ? "__copy_helper_block_"
1793 : "__destroy_helper_block_";
1794 if (CGM.getLangOpts().Exceptions)
1795 Name += "e";
1796 if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
1797 Name += "a";
1798 Name += llvm::to_string(Value: BlockAlignment.getQuantity()) + "_";
1799
1800 for (auto &Cap : Captures) {
1801 if (Cap.isConstantOrTrivial())
1802 continue;
1803 Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
1804 Name += getBlockCaptureStr(Cap, StrKind, BlockAlignment, CGM);
1805 }
1806
1807 return Name;
1808}
1809
1810static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
1811 Address Field, QualType CaptureType,
1812 BlockFieldFlags Flags, bool ForCopyHelper,
1813 VarDecl *Var, CodeGenFunction &CGF) {
1814 bool EHOnly = ForCopyHelper;
1815
1816 switch (CaptureKind) {
1817 case BlockCaptureEntityKind::CXXRecord:
1818 case BlockCaptureEntityKind::ARCWeak:
1819 case BlockCaptureEntityKind::NonTrivialCStruct:
1820 case BlockCaptureEntityKind::ARCStrong: {
1821 if (CaptureType.isDestructedType() &&
1822 (!EHOnly || CGF.needsEHCleanup(kind: CaptureType.isDestructedType()))) {
1823 CodeGenFunction::Destroyer *Destroyer =
1824 CaptureKind == BlockCaptureEntityKind::ARCStrong
1825 ? CodeGenFunction::destroyARCStrongImprecise
1826 : CGF.getDestroyer(destructionKind: CaptureType.isDestructedType());
1827 CleanupKind Kind =
1828 EHOnly ? EHCleanup
1829 : CGF.getCleanupKind(kind: CaptureType.isDestructedType());
1830 CGF.pushDestroy(kind: Kind, addr: Field, type: CaptureType, destroyer: Destroyer, useEHCleanupForArray: Kind & EHCleanup);
1831 }
1832 break;
1833 }
1834 case BlockCaptureEntityKind::BlockObject: {
1835 if (!EHOnly || CGF.getLangOpts().Exceptions) {
1836 CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;
1837 // Calls to _Block_object_dispose along the EH path in the copy helper
1838 // function don't throw as newly-copied __block variables always have a
1839 // reference count of 2.
1840 bool CanThrow =
1841 !ForCopyHelper && CGF.cxxDestructorCanThrow(T: CaptureType);
1842 CGF.enterByrefCleanup(Kind, Addr: Field, Flags, /*LoadBlockVarAddr*/ true,
1843 CanThrow);
1844 }
1845 break;
1846 }
1847 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth:
1848 case BlockCaptureEntityKind::None:
1849 break;
1850 }
1851}
1852
1853static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,
1854 llvm::Function *Fn,
1855 const CGFunctionInfo &FI,
1856 CodeGenModule &CGM) {
1857 if (CapturesNonExternalType) {
1858 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
1859 } else {
1860 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1861 Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1862 CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
1863 CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
1864 }
1865}
1866/// Generate the copy-helper function for a block closure object:
1867/// static void block_copy_helper(block_t *dst, block_t *src);
1868/// The runtime will have previously initialized 'dst' by doing a
1869/// bit-copy of 'src'.
1870///
1871/// Note that this copies an entire block closure object to the heap;
1872/// it should not be confused with a 'byref copy helper', which moves
1873/// the contents of an individual __block variable to the heap.
1874llvm::Constant *
1875CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
1876 std::string FuncName = getCopyDestroyHelperFuncName(
1877 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
1878 StrKind: CaptureStrKind::CopyHelper, CGM);
1879
1880 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
1881 return Func;
1882
1883 ASTContext &C = getContext();
1884
1885 QualType ReturnTy = C.VoidTy;
1886
1887 FunctionArgList args;
1888 ImplicitParamDecl DstDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
1889 args.push_back(Elt: &DstDecl);
1890 ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
1891 args.push_back(Elt: &SrcDecl);
1892
1893 const CGFunctionInfo &FI =
1894 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
1895
1896 // FIXME: it would be nice if these were mergeable with things with
1897 // identical semantics.
1898 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
1899
1900 llvm::Function *Fn =
1901 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
1902 N: FuncName, M: &CGM.getModule());
1903 if (CGM.supportsCOMDAT())
1904 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
1905
1906 SmallVector<QualType, 2> ArgTys;
1907 ArgTys.push_back(Elt: C.VoidPtrTy);
1908 ArgTys.push_back(Elt: C.VoidPtrTy);
1909
1910 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
1911 CGM);
1912 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
1913 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1914
1915 Address src = GetAddrOfLocalVar(VD: &SrcDecl);
1916 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
1917 blockInfo.BlockAlign);
1918
1919 Address dst = GetAddrOfLocalVar(VD: &DstDecl);
1920 dst = Address(Builder.CreateLoad(Addr: dst), blockInfo.StructureType,
1921 blockInfo.BlockAlign);
1922
1923 for (auto &capture : blockInfo.SortedCaptures) {
1924 if (capture.isConstantOrTrivial())
1925 continue;
1926
1927 const BlockDecl::Capture &CI = *capture.Cap;
1928 QualType captureType = CI.getVariable()->getType();
1929 BlockFieldFlags flags = capture.CopyFlags;
1930
1931 unsigned index = capture.getIndex();
1932 Address srcField = Builder.CreateStructGEP(Addr: src, Index: index);
1933 Address dstField = Builder.CreateStructGEP(Addr: dst, Index: index);
1934
1935 switch (capture.CopyKind) {
1936 case BlockCaptureEntityKind::CXXRecord:
1937 // If there's an explicit copy expression, we do that.
1938 assert(CI.getCopyExpr() && "copy expression for variable is missing");
1939 EmitSynthesizedCXXCopyCtor(Dest: dstField, Src: srcField, Exp: CI.getCopyExpr());
1940 break;
1941 case BlockCaptureEntityKind::ARCWeak:
1942 EmitARCCopyWeak(dst: dstField, src: srcField);
1943 break;
1944 case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1945 QualType Type = CI.getVariable()->getType();
1946 PointerAuthQualifier PointerAuth = Type.getPointerAuth();
1947 assert(PointerAuth && PointerAuth.isAddressDiscriminated());
1948 EmitPointerAuthCopy(Qualifier: PointerAuth, Type, DestField: dstField, SrcField: srcField);
1949 // We don't need to push cleanups for ptrauth types.
1950 continue;
1951 }
1952 case BlockCaptureEntityKind::NonTrivialCStruct: {
1953 // If this is a C struct that requires non-trivial copy construction,
1954 // emit a call to its copy constructor.
1955 QualType varType = CI.getVariable()->getType();
1956 callCStructCopyConstructor(Dst: MakeAddrLValue(Addr: dstField, T: varType),
1957 Src: MakeAddrLValue(Addr: srcField, T: varType));
1958 break;
1959 }
1960 case BlockCaptureEntityKind::ARCStrong: {
1961 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
1962 // At -O0, store null into the destination field (so that the
1963 // storeStrong doesn't over-release) and then call storeStrong.
1964 // This is a workaround to not having an initStrong call.
1965 if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
1966 auto *ty = cast<llvm::PointerType>(Val: srcValue->getType());
1967 llvm::Value *null = llvm::ConstantPointerNull::get(T: ty);
1968 Builder.CreateStore(Val: null, Addr: dstField);
1969 EmitARCStoreStrongCall(addr: dstField, value: srcValue, resultIgnored: true);
1970
1971 // With optimization enabled, take advantage of the fact that
1972 // the blocks runtime guarantees a memcpy of the block data, and
1973 // just emit a retain of the src field.
1974 } else {
1975 EmitARCRetainNonBlock(value: srcValue);
1976
1977 // Unless EH cleanup is required, we don't need this anymore, so kill
1978 // it. It's not quite worth the annoyance to avoid creating it in the
1979 // first place.
1980 if (!needsEHCleanup(kind: captureType.isDestructedType()))
1981 if (auto *I =
1982 cast_or_null<llvm::Instruction>(Val: dstField.getBasePointer()))
1983 I->eraseFromParent();
1984 }
1985 break;
1986 }
1987 case BlockCaptureEntityKind::BlockObject: {
1988 llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
1989 llvm::Value *dstAddr = dstField.emitRawPointer(CGF&: *this);
1990 llvm::Value *args[] = {
1991 dstAddr, srcValue, llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())
1992 };
1993
1994 if (CI.isByRef() && C.getBlockVarCopyInit(VD: CI.getVariable()).canThrow())
1995 EmitRuntimeCallOrInvoke(callee: CGM.getBlockObjectAssign(), args);
1996 else
1997 EmitNounwindRuntimeCall(callee: CGM.getBlockObjectAssign(), args);
1998 break;
1999 }
2000 case BlockCaptureEntityKind::None:
2001 continue;
2002 }
2003
2004 // Ensure that we destroy the copied object if an exception is thrown later
2005 // in the helper function.
2006 pushCaptureCleanup(CaptureKind: capture.CopyKind, Field: dstField, CaptureType: captureType, Flags: flags,
2007 /*ForCopyHelper*/ true, Var: CI.getVariable(), CGF&: *this);
2008 }
2009
2010 FinishFunction();
2011
2012 return Fn;
2013}
2014
2015static BlockFieldFlags
2016getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI,
2017 QualType T) {
2018 BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT;
2019 if (T->isBlockPointerType())
2020 Flags = BLOCK_FIELD_IS_BLOCK;
2021 return Flags;
2022}
2023
2024static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
2025computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
2026 const LangOptions &LangOpts) {
2027 if (CI.isEscapingByref()) {
2028 BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
2029 if (T.isObjCGCWeak())
2030 Flags |= BLOCK_FIELD_IS_WEAK;
2031 return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
2032 }
2033
2034 switch (T.isDestructedType()) {
2035 case QualType::DK_cxx_destructor:
2036 return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
2037 case QualType::DK_objc_strong_lifetime:
2038 // Use objc_storeStrong for __strong direct captures; the
2039 // dynamic tools really like it when we do this.
2040 return std::make_pair(x: BlockCaptureEntityKind::ARCStrong,
2041 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2042 case QualType::DK_objc_weak_lifetime:
2043 // Support __weak direct captures.
2044 return std::make_pair(x: BlockCaptureEntityKind::ARCWeak,
2045 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2046 case QualType::DK_nontrivial_c_struct:
2047 return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
2048 y: BlockFieldFlags());
2049 case QualType::DK_none: {
2050 // Non-ARC captures are strong, and we need to use _Block_object_dispose.
2051 // But honor the inert __unsafe_unretained qualifier, which doesn't actually
2052 // make it into the type system.
2053 if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() &&
2054 !LangOpts.ObjCAutoRefCount && !T->isObjCInertUnsafeUnretainedType())
2055 return std::make_pair(x: BlockCaptureEntityKind::BlockObject,
2056 y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2057 // Otherwise, we have nothing to do.
2058 return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
2059 }
2060 }
2061 llvm_unreachable("after exhaustive DestructionKind switch");
2062}
2063
2064/// Generate the destroy-helper function for a block closure object:
2065/// static void block_destroy_helper(block_t *theBlock);
2066///
2067/// Note that this destroys a heap-allocated block closure object;
2068/// it should not be confused with a 'byref destroy helper', which
2069/// destroys the heap-allocated contents of an individual __block
2070/// variable.
2071llvm::Constant *
2072CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
2073 std::string FuncName = getCopyDestroyHelperFuncName(
2074 Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
2075 StrKind: CaptureStrKind::DisposeHelper, CGM);
2076
2077 if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
2078 return Func;
2079
2080 ASTContext &C = getContext();
2081
2082 QualType ReturnTy = C.VoidTy;
2083
2084 FunctionArgList args;
2085 ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
2086 args.push_back(Elt: &SrcDecl);
2087
2088 const CGFunctionInfo &FI =
2089 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2090
2091 // FIXME: We'd like to put these into a mergable by content, with
2092 // internal linkage.
2093 llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
2094
2095 llvm::Function *Fn =
2096 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
2097 N: FuncName, M: &CGM.getModule());
2098 if (CGM.supportsCOMDAT())
2099 Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
2100
2101 SmallVector<QualType, 1> ArgTys;
2102 ArgTys.push_back(Elt: C.VoidPtrTy);
2103
2104 setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
2105 CGM);
2106 StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2107 markAsIgnoreThreadCheckingAtRuntime(Fn);
2108
2109 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2110
2111 Address src = GetAddrOfLocalVar(VD: &SrcDecl);
2112 src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
2113 blockInfo.BlockAlign);
2114
2115 CodeGenFunction::RunCleanupsScope cleanups(*this);
2116
2117 for (auto &capture : blockInfo.SortedCaptures) {
2118 if (capture.isConstantOrTrivial())
2119 continue;
2120
2121 const BlockDecl::Capture &CI = *capture.Cap;
2122 BlockFieldFlags flags = capture.DisposeFlags;
2123
2124 Address srcField = Builder.CreateStructGEP(Addr: src, Index: capture.getIndex());
2125
2126 pushCaptureCleanup(CaptureKind: capture.DisposeKind, Field: srcField,
2127 CaptureType: CI.getVariable()->getType(), Flags: flags,
2128 /*ForCopyHelper*/ false, Var: CI.getVariable(), CGF&: *this);
2129 }
2130
2131 cleanups.ForceCleanup();
2132
2133 FinishFunction();
2134
2135 return Fn;
2136}
2137
2138namespace {
2139
2140/// Emits the copy/dispose helper functions for a __block object of id type.
2141class ObjectByrefHelpers final : public BlockByrefHelpers {
2142 BlockFieldFlags Flags;
2143
2144public:
2145 ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
2146 : BlockByrefHelpers(alignment), Flags(flags) {}
2147
2148 void emitCopy(CodeGenFunction &CGF, Address destField,
2149 Address srcField) override {
2150 destField = destField.withElementType(ElemTy: CGF.Int8Ty);
2151
2152 srcField = srcField.withElementType(ElemTy: CGF.Int8PtrTy);
2153 llvm::Value *srcValue = CGF.Builder.CreateLoad(Addr: srcField);
2154
2155 unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
2156
2157 llvm::Value *flagsVal = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: flags);
2158 llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
2159
2160 llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};
2161 CGF.EmitNounwindRuntimeCall(callee: fn, args);
2162 }
2163
2164 void emitDispose(CodeGenFunction &CGF, Address field) override {
2165 field = field.withElementType(ElemTy: CGF.Int8PtrTy);
2166 llvm::Value *value = CGF.Builder.CreateLoad(Addr: field);
2167
2168 CGF.BuildBlockRelease(DeclPtr: value, flags: Flags | BLOCK_BYREF_CALLER, CanThrow: false);
2169 }
2170
2171 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2172 id.AddInteger(I: Flags.getBitMask());
2173 }
2174};
2175
2176/// Emits the copy/dispose helpers for an ARC __block __weak variable.
2177class ARCWeakByrefHelpers final : public BlockByrefHelpers {
2178public:
2179 ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2180
2181 void emitCopy(CodeGenFunction &CGF, Address destField,
2182 Address srcField) override {
2183 CGF.EmitARCMoveWeak(dst: destField, src: srcField);
2184 }
2185
2186 void emitDispose(CodeGenFunction &CGF, Address field) override {
2187 CGF.EmitARCDestroyWeak(addr: field);
2188 }
2189
2190 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2191 // 0 is distinguishable from all pointers and byref flags
2192 id.AddInteger(I: 0);
2193 }
2194};
2195
2196/// Emits the copy/dispose helpers for an ARC __block __strong variable
2197/// that's not of block-pointer type.
2198class ARCStrongByrefHelpers final : public BlockByrefHelpers {
2199public:
2200 ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2201
2202 void emitCopy(CodeGenFunction &CGF, Address destField,
2203 Address srcField) override {
2204 // Do a "move" by copying the value and then zeroing out the old
2205 // variable.
2206
2207 llvm::Value *value = CGF.Builder.CreateLoad(Addr: srcField);
2208
2209 llvm::Value *null =
2210 llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: value->getType()));
2211
2212 if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {
2213 CGF.Builder.CreateStore(Val: null, Addr: destField);
2214 CGF.EmitARCStoreStrongCall(addr: destField, value, /*ignored*/ resultIgnored: true);
2215 CGF.EmitARCStoreStrongCall(addr: srcField, value: null, /*ignored*/ resultIgnored: true);
2216 return;
2217 }
2218 CGF.Builder.CreateStore(Val: value, Addr: destField);
2219 CGF.Builder.CreateStore(Val: null, Addr: srcField);
2220 }
2221
2222 void emitDispose(CodeGenFunction &CGF, Address field) override {
2223 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2224 }
2225
2226 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2227 // 1 is distinguishable from all pointers and byref flags
2228 id.AddInteger(I: 1);
2229 }
2230};
2231
2232/// Emits the copy/dispose helpers for an ARC __block __strong
2233/// variable that's of block-pointer type.
2234class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {
2235public:
2236 ARCStrongBlockByrefHelpers(CharUnits alignment)
2237 : BlockByrefHelpers(alignment) {}
2238
2239 void emitCopy(CodeGenFunction &CGF, Address destField,
2240 Address srcField) override {
2241 // Do the copy with objc_retainBlock; that's all that
2242 // _Block_object_assign would do anyway, and we'd have to pass the
2243 // right arguments to make sure it doesn't get no-op'ed.
2244 llvm::Value *oldValue = CGF.Builder.CreateLoad(Addr: srcField);
2245 llvm::Value *copy = CGF.EmitARCRetainBlock(value: oldValue, /*mandatory*/ true);
2246 CGF.Builder.CreateStore(Val: copy, Addr: destField);
2247 }
2248
2249 void emitDispose(CodeGenFunction &CGF, Address field) override {
2250 CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2251 }
2252
2253 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2254 // 2 is distinguishable from all pointers and byref flags
2255 id.AddInteger(I: 2);
2256 }
2257};
2258
2259/// Emits the copy/dispose helpers for a __block variable with a
2260/// nontrivial copy constructor or destructor.
2261class CXXByrefHelpers final : public BlockByrefHelpers {
2262 QualType VarType;
2263 const Expr *CopyExpr;
2264
2265public:
2266 CXXByrefHelpers(CharUnits alignment, QualType type,
2267 const Expr *copyExpr)
2268 : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
2269
2270 bool needsCopy() const override { return CopyExpr != nullptr; }
2271 void emitCopy(CodeGenFunction &CGF, Address destField,
2272 Address srcField) override {
2273 if (!CopyExpr) return;
2274 CGF.EmitSynthesizedCXXCopyCtor(Dest: destField, Src: srcField, Exp: CopyExpr);
2275 }
2276
2277 void emitDispose(CodeGenFunction &CGF, Address field) override {
2278 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2279 CGF.PushDestructorCleanup(T: VarType, Addr: field);
2280 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2281 }
2282
2283 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2284 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2285 }
2286};
2287
2288/// Emits the copy/dispose helpers for a __block variable with
2289/// address-discriminated pointer authentication.
2290class AddressDiscriminatedByrefHelpers final : public BlockByrefHelpers {
2291 QualType VarType;
2292
2293public:
2294 AddressDiscriminatedByrefHelpers(CharUnits Alignment, QualType Type)
2295 : BlockByrefHelpers(Alignment), VarType(Type) {
2296 assert(Type.hasAddressDiscriminatedPointerAuth());
2297 }
2298
2299 void emitCopy(CodeGenFunction &CGF, Address DestField,
2300 Address SrcField) override {
2301 CGF.EmitPointerAuthCopy(Qualifier: VarType.getPointerAuth(), Type: VarType, DestField,
2302 SrcField);
2303 }
2304
2305 bool needsDispose() const override { return false; }
2306 void emitDispose(CodeGenFunction &CGF, Address Field) override {
2307 llvm_unreachable("should never be called");
2308 }
2309
2310 void profileImpl(llvm::FoldingSetNodeID &ID) const override {
2311 ID.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2312 }
2313};
2314
2315/// Emits the copy/dispose helpers for a __block variable that is a non-trivial
2316/// C struct.
2317class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers {
2318 QualType VarType;
2319
2320public:
2321 NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type)
2322 : BlockByrefHelpers(alignment), VarType(type) {}
2323
2324 void emitCopy(CodeGenFunction &CGF, Address destField,
2325 Address srcField) override {
2326 CGF.callCStructMoveConstructor(Dst: CGF.MakeAddrLValue(Addr: destField, T: VarType),
2327 Src: CGF.MakeAddrLValue(Addr: srcField, T: VarType));
2328 }
2329
2330 bool needsDispose() const override {
2331 return VarType.isDestructedType();
2332 }
2333
2334 void emitDispose(CodeGenFunction &CGF, Address field) override {
2335 EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2336 CGF.pushDestroy(dtorKind: VarType.isDestructedType(), addr: field, type: VarType);
2337 CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2338 }
2339
2340 void profileImpl(llvm::FoldingSetNodeID &id) const override {
2341 id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2342 }
2343};
2344} // end anonymous namespace
2345
2346static llvm::Constant *
2347generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
2348 BlockByrefHelpers &generator) {
2349 ASTContext &Context = CGF.getContext();
2350
2351 QualType ReturnTy = Context.VoidTy;
2352
2353 FunctionArgList args;
2354 ImplicitParamDecl Dst(Context, Context.VoidPtrTy, ImplicitParamKind::Other);
2355 args.push_back(Elt: &Dst);
2356
2357 ImplicitParamDecl Src(Context, Context.VoidPtrTy, ImplicitParamKind::Other);
2358 args.push_back(Elt: &Src);
2359
2360 const CGFunctionInfo &FI =
2361 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2362
2363 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2364
2365 // FIXME: We'd like to put these into a mergable by content, with
2366 // internal linkage.
2367 llvm::Function *Fn =
2368 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2369 N: "__Block_byref_object_copy_", M: &CGF.CGM.getModule());
2370
2371 SmallVector<QualType, 2> ArgTys;
2372 ArgTys.push_back(Elt: Context.VoidPtrTy);
2373 ArgTys.push_back(Elt: Context.VoidPtrTy);
2374
2375 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2376
2377 CGF.StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2378 // Create a scope with an artificial location for the body of this function.
2379 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2380
2381 if (generator.needsCopy()) {
2382 // dst->x
2383 Address destField = CGF.GetAddrOfLocalVar(VD: &Dst);
2384 destField = Address(CGF.Builder.CreateLoad(Addr: destField), byrefInfo.Type,
2385 byrefInfo.ByrefAlignment);
2386 destField =
2387 CGF.emitBlockByrefAddress(baseAddr: destField, info: byrefInfo, followForward: false, name: "dest-object");
2388
2389 // src->x
2390 Address srcField = CGF.GetAddrOfLocalVar(VD: &Src);
2391 srcField = Address(CGF.Builder.CreateLoad(Addr: srcField), byrefInfo.Type,
2392 byrefInfo.ByrefAlignment);
2393 srcField =
2394 CGF.emitBlockByrefAddress(baseAddr: srcField, info: byrefInfo, followForward: false, name: "src-object");
2395
2396 generator.emitCopy(CGF, dest: destField, src: srcField);
2397 }
2398
2399 CGF.FinishFunction();
2400
2401 return Fn;
2402}
2403
2404/// Build the copy helper for a __block variable.
2405static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
2406 const BlockByrefInfo &byrefInfo,
2407 BlockByrefHelpers &generator) {
2408 CodeGenFunction CGF(CGM);
2409 return generateByrefCopyHelper(CGF, byrefInfo, generator);
2410}
2411
2412/// Generate code for a __block variable's dispose helper.
2413static llvm::Constant *
2414generateByrefDisposeHelper(CodeGenFunction &CGF,
2415 const BlockByrefInfo &byrefInfo,
2416 BlockByrefHelpers &generator) {
2417 ASTContext &Context = CGF.getContext();
2418 QualType R = Context.VoidTy;
2419
2420 FunctionArgList args;
2421 ImplicitParamDecl Src(CGF.getContext(), Context.VoidPtrTy,
2422 ImplicitParamKind::Other);
2423 args.push_back(Elt: &Src);
2424
2425 const CGFunctionInfo &FI =
2426 CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: R, args);
2427
2428 llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2429
2430 // FIXME: We'd like to put these into a mergable by content, with
2431 // internal linkage.
2432 llvm::Function *Fn =
2433 llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2434 N: "__Block_byref_object_dispose_",
2435 M: &CGF.CGM.getModule());
2436
2437 SmallVector<QualType, 1> ArgTys;
2438 ArgTys.push_back(Elt: Context.VoidPtrTy);
2439
2440 CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2441
2442 CGF.StartFunction(GD: GlobalDecl(), RetTy: R, Fn, FnInfo: FI, Args: args);
2443 // Create a scope with an artificial location for the body of this function.
2444 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2445
2446 if (generator.needsDispose()) {
2447 Address addr = CGF.GetAddrOfLocalVar(VD: &Src);
2448 addr = Address(CGF.Builder.CreateLoad(Addr: addr), byrefInfo.Type,
2449 byrefInfo.ByrefAlignment);
2450 addr = CGF.emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, followForward: false, name: "object");
2451
2452 generator.emitDispose(CGF, field: addr);
2453 }
2454
2455 CGF.FinishFunction();
2456
2457 return Fn;
2458}
2459
2460/// Build the dispose helper for a __block variable.
2461static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
2462 const BlockByrefInfo &byrefInfo,
2463 BlockByrefHelpers &generator) {
2464 CodeGenFunction CGF(CGM);
2465 return generateByrefDisposeHelper(CGF, byrefInfo, generator);
2466}
2467
2468/// Lazily build the copy and dispose helpers for a __block variable
2469/// with the given information.
2470template <class T>
2471static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,
2472 T &&generator) {
2473 llvm::FoldingSetNodeID id;
2474 generator.Profile(id);
2475
2476 void *insertPos;
2477 BlockByrefHelpers *node
2478 = CGM.ByrefHelpersCache.FindNodeOrInsertPos(ID: id, InsertPos&: insertPos);
2479 if (node) return static_cast<T*>(node);
2480
2481 generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);
2482 generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);
2483
2484 T *copy = new (CGM.getContext()) T(std::forward<T>(generator));
2485 CGM.ByrefHelpersCache.InsertNode(copy, insertPos);
2486 return copy;
2487}
2488
2489/// Build the copy and dispose helpers for the given __block variable
2490/// emission. Places the helpers in the global cache. Returns null
2491/// if no helpers are required.
2492BlockByrefHelpers *
2493CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
2494 const AutoVarEmission &emission) {
2495 const VarDecl &var = *emission.Variable;
2496 assert(var.isEscapingByref() &&
2497 "only escaping __block variables need byref helpers");
2498
2499 QualType type = var.getType();
2500
2501 auto &byrefInfo = getBlockByrefInfo(var: &var);
2502
2503 // The alignment we care about for the purposes of uniquing byref
2504 // helpers is the alignment of the actual byref value field.
2505 CharUnits valueAlignment =
2506 byrefInfo.ByrefAlignment.alignmentAtOffset(offset: byrefInfo.FieldOffset);
2507
2508 if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
2509 const Expr *copyExpr =
2510 CGM.getContext().getBlockVarCopyInit(VD: &var).getCopyExpr();
2511 if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
2512
2513 return ::buildByrefHelpers(
2514 CGM, byrefInfo, generator: CXXByrefHelpers(valueAlignment, type, copyExpr));
2515 }
2516 if (type.hasAddressDiscriminatedPointerAuth()) {
2517 return ::buildByrefHelpers(
2518 CGM, byrefInfo, generator: AddressDiscriminatedByrefHelpers(valueAlignment, type));
2519 }
2520 // If type is a non-trivial C struct type that is non-trivial to
2521 // destructly move or destroy, build the copy and dispose helpers.
2522 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct ||
2523 type.isDestructedType() == QualType::DK_nontrivial_c_struct)
2524 return ::buildByrefHelpers(
2525 CGM, byrefInfo, generator: NonTrivialCStructByrefHelpers(valueAlignment, type));
2526
2527 // Otherwise, if we don't have a retainable type, there's nothing to do.
2528 // that the runtime does extra copies.
2529 if (!type->isObjCRetainableType()) return nullptr;
2530
2531 Qualifiers qs = type.getQualifiers();
2532
2533 // If we have lifetime, that dominates.
2534 if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
2535 switch (lifetime) {
2536 case Qualifiers::OCL_None: llvm_unreachable("impossible");
2537
2538 // These are just bits as far as the runtime is concerned.
2539 case Qualifiers::OCL_ExplicitNone:
2540 case Qualifiers::OCL_Autoreleasing:
2541 return nullptr;
2542
2543 // Tell the runtime that this is ARC __weak, called by the
2544 // byref routines.
2545 case Qualifiers::OCL_Weak:
2546 return ::buildByrefHelpers(CGM, byrefInfo,
2547 generator: ARCWeakByrefHelpers(valueAlignment));
2548
2549 // ARC __strong __block variables need to be retained.
2550 case Qualifiers::OCL_Strong:
2551 // Block pointers need to be copied, and there's no direct
2552 // transfer possible.
2553 if (type->isBlockPointerType()) {
2554 return ::buildByrefHelpers(CGM, byrefInfo,
2555 generator: ARCStrongBlockByrefHelpers(valueAlignment));
2556
2557 // Otherwise, we transfer ownership of the retain from the stack
2558 // to the heap.
2559 } else {
2560 return ::buildByrefHelpers(CGM, byrefInfo,
2561 generator: ARCStrongByrefHelpers(valueAlignment));
2562 }
2563 }
2564 llvm_unreachable("fell out of lifetime switch!");
2565 }
2566
2567 BlockFieldFlags flags;
2568 if (type->isBlockPointerType()) {
2569 flags |= BLOCK_FIELD_IS_BLOCK;
2570 } else if (CGM.getContext().isObjCNSObjectType(Ty: type) ||
2571 type->isObjCObjectPointerType()) {
2572 flags |= BLOCK_FIELD_IS_OBJECT;
2573 } else {
2574 return nullptr;
2575 }
2576
2577 if (type.isObjCGCWeak())
2578 flags |= BLOCK_FIELD_IS_WEAK;
2579
2580 return ::buildByrefHelpers(CGM, byrefInfo,
2581 generator: ObjectByrefHelpers(valueAlignment, flags));
2582}
2583
2584Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2585 const VarDecl *var,
2586 bool followForward) {
2587 auto &info = getBlockByrefInfo(var);
2588 return emitBlockByrefAddress(baseAddr, info, followForward, name: var->getName());
2589}
2590
2591Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2592 const BlockByrefInfo &info,
2593 bool followForward,
2594 const llvm::Twine &name) {
2595 // Chase the forwarding address if requested.
2596 if (followForward) {
2597 Address forwardingAddr = Builder.CreateStructGEP(Addr: baseAddr, Index: 1, Name: "forwarding");
2598 baseAddr = Address(Builder.CreateLoad(Addr: forwardingAddr), info.Type,
2599 info.ByrefAlignment);
2600 }
2601
2602 return Builder.CreateStructGEP(Addr: baseAddr, Index: info.FieldIndex, Name: name);
2603}
2604
2605/// BuildByrefInfo - This routine changes a __block variable declared as T x
2606/// into:
2607///
2608/// struct {
2609/// void *__isa;
2610/// void *__forwarding;
2611/// int32_t __flags;
2612/// int32_t __size;
2613/// void *__copy_helper; // only if needed
2614/// void *__destroy_helper; // only if needed
2615/// void *__byref_variable_layout;// only if needed
2616/// char padding[X]; // only if needed
2617/// T x;
2618/// } x
2619///
2620const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
2621 auto it = BlockByrefInfos.find(Val: D);
2622 if (it != BlockByrefInfos.end())
2623 return it->second;
2624
2625 QualType Ty = D->getType();
2626
2627 CharUnits size;
2628 SmallVector<llvm::Type *, 8> types;
2629
2630 // void *__isa;
2631 types.push_back(Elt: VoidPtrTy);
2632 size += getPointerSize();
2633
2634 // void *__forwarding;
2635 types.push_back(Elt: VoidPtrTy);
2636 size += getPointerSize();
2637
2638 // int32_t __flags;
2639 types.push_back(Elt: Int32Ty);
2640 size += CharUnits::fromQuantity(Quantity: 4);
2641
2642 // int32_t __size;
2643 types.push_back(Elt: Int32Ty);
2644 size += CharUnits::fromQuantity(Quantity: 4);
2645
2646 // Note that this must match *exactly* the logic in buildByrefHelpers.
2647 bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
2648 if (hasCopyAndDispose) {
2649 /// void *__copy_helper;
2650 types.push_back(Elt: VoidPtrTy);
2651 size += getPointerSize();
2652
2653 /// void *__destroy_helper;
2654 types.push_back(Elt: VoidPtrTy);
2655 size += getPointerSize();
2656 }
2657
2658 bool HasByrefExtendedLayout = false;
2659 Qualifiers::ObjCLifetime Lifetime = Qualifiers::OCL_None;
2660 if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&
2661 HasByrefExtendedLayout) {
2662 /// void *__byref_variable_layout;
2663 types.push_back(Elt: VoidPtrTy);
2664 size += CharUnits::fromQuantity(Quantity: PointerSizeInBytes);
2665 }
2666
2667 // T x;
2668 llvm::Type *varTy = ConvertTypeForMem(T: Ty);
2669
2670 bool packed = false;
2671 CharUnits varAlign = getContext().getDeclAlign(D);
2672 CharUnits varOffset = size.alignTo(Align: varAlign);
2673
2674 // We may have to insert padding.
2675 if (varOffset != size) {
2676 llvm::Type *paddingTy =
2677 llvm::ArrayType::get(ElementType: Int8Ty, NumElements: (varOffset - size).getQuantity());
2678
2679 types.push_back(Elt: paddingTy);
2680 size = varOffset;
2681
2682 // Conversely, we might have to prevent LLVM from inserting padding.
2683 } else if (CGM.getDataLayout().getABITypeAlign(Ty: varTy) >
2684 uint64_t(varAlign.getQuantity())) {
2685 packed = true;
2686 }
2687 types.push_back(Elt: varTy);
2688
2689 llvm::StructType *byrefType = llvm::StructType::create(
2690 Context&: getLLVMContext(), Elements: types, Name: "struct.__block_byref_" + D->getNameAsString(),
2691 isPacked: packed);
2692
2693 BlockByrefInfo info;
2694 info.Type = byrefType;
2695 info.FieldIndex = types.size() - 1;
2696 info.FieldOffset = varOffset;
2697 info.ByrefAlignment = std::max(a: varAlign, b: getPointerAlign());
2698
2699 auto pair = BlockByrefInfos.insert(KV: {D, info});
2700 assert(pair.second && "info was inserted recursively?");
2701 return pair.first->second;
2702}
2703
2704/// Initialize the structural components of a __block variable, i.e.
2705/// everything but the actual object.
2706void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
2707 // Find the address of the local.
2708 Address addr = emission.Addr;
2709
2710 // That's an alloca of the byref structure type.
2711 llvm::StructType *byrefType = cast<llvm::StructType>(Val: addr.getElementType());
2712
2713 unsigned nextHeaderIndex = 0;
2714 CharUnits nextHeaderOffset;
2715 auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
2716 const Twine &name) {
2717 auto fieldAddr = Builder.CreateStructGEP(Addr: addr, Index: nextHeaderIndex, Name: name);
2718 Builder.CreateStore(Val: value, Addr: fieldAddr);
2719
2720 nextHeaderIndex++;
2721 nextHeaderOffset += fieldSize;
2722 };
2723
2724 // Build the byref helpers if necessary. This is null if we don't need any.
2725 BlockByrefHelpers *helpers = buildByrefHelpers(byrefType&: *byrefType, emission);
2726
2727 const VarDecl &D = *emission.Variable;
2728 QualType type = D.getType();
2729
2730 bool HasByrefExtendedLayout = false;
2731 Qualifiers::ObjCLifetime ByrefLifetime = Qualifiers::OCL_None;
2732 bool ByRefHasLifetime =
2733 getContext().getByrefLifetime(Ty: type, Lifetime&: ByrefLifetime, HasByrefExtendedLayout);
2734
2735 llvm::Value *V;
2736
2737 // Initialize the 'isa', which is just 0 or 1.
2738 int isa = 0;
2739 if (type.isObjCGCWeak())
2740 isa = 1;
2741 V = Builder.CreateIntToPtr(V: Builder.getInt32(C: isa), DestTy: Int8PtrTy, Name: "isa");
2742 storeHeaderField(V, getPointerSize(), "byref.isa");
2743
2744 // Store the address of the variable into its own forwarding pointer.
2745 storeHeaderField(addr.emitRawPointer(CGF&: *this), getPointerSize(),
2746 "byref.forwarding");
2747
2748 // Blocks ABI:
2749 // c) the flags field is set to either 0 if no helper functions are
2750 // needed or BLOCK_BYREF_HAS_COPY_DISPOSE if they are,
2751 BlockFlags flags;
2752 if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE;
2753 if (ByRefHasLifetime) {
2754 if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED;
2755 else switch (ByrefLifetime) {
2756 case Qualifiers::OCL_Strong:
2757 flags |= BLOCK_BYREF_LAYOUT_STRONG;
2758 break;
2759 case Qualifiers::OCL_Weak:
2760 flags |= BLOCK_BYREF_LAYOUT_WEAK;
2761 break;
2762 case Qualifiers::OCL_ExplicitNone:
2763 flags |= BLOCK_BYREF_LAYOUT_UNRETAINED;
2764 break;
2765 case Qualifiers::OCL_None:
2766 if (!type->isObjCObjectPointerType() && !type->isBlockPointerType())
2767 flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT;
2768 break;
2769 default:
2770 break;
2771 }
2772 if (CGM.getLangOpts().ObjCGCBitmapPrint) {
2773 printf(format: "\n Inline flag for BYREF variable layout (%d):", flags.getBitMask());
2774 if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE)
2775 printf(format: " BLOCK_BYREF_HAS_COPY_DISPOSE");
2776 if (flags & BLOCK_BYREF_LAYOUT_MASK) {
2777 BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK);
2778 if (ThisFlag == BLOCK_BYREF_LAYOUT_EXTENDED)
2779 printf(format: " BLOCK_BYREF_LAYOUT_EXTENDED");
2780 if (ThisFlag == BLOCK_BYREF_LAYOUT_STRONG)
2781 printf(format: " BLOCK_BYREF_LAYOUT_STRONG");
2782 if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK)
2783 printf(format: " BLOCK_BYREF_LAYOUT_WEAK");
2784 if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED)
2785 printf(format: " BLOCK_BYREF_LAYOUT_UNRETAINED");
2786 if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT)
2787 printf(format: " BLOCK_BYREF_LAYOUT_NON_OBJECT");
2788 }
2789 printf(format: "\n");
2790 }
2791 }
2792 storeHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
2793 getIntSize(), "byref.flags");
2794
2795 CharUnits byrefSize = CGM.GetTargetTypeStoreSize(Ty: byrefType);
2796 V = llvm::ConstantInt::get(Ty: IntTy, V: byrefSize.getQuantity());
2797 storeHeaderField(V, getIntSize(), "byref.size");
2798
2799 if (helpers) {
2800 storeHeaderField(helpers->CopyHelper, getPointerSize(),
2801 "byref.copyHelper");
2802 storeHeaderField(helpers->DisposeHelper, getPointerSize(),
2803 "byref.disposeHelper");
2804 }
2805
2806 if (ByRefHasLifetime && HasByrefExtendedLayout) {
2807 auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, T: type);
2808 storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");
2809 }
2810}
2811
2812void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,
2813 bool CanThrow) {
2814 llvm::FunctionCallee F = CGM.getBlockObjectDispose();
2815 llvm::Value *args[] = {V,
2816 llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())};
2817
2818 if (CanThrow)
2819 EmitRuntimeCallOrInvoke(callee: F, args);
2820 else
2821 EmitNounwindRuntimeCall(callee: F, args);
2822}
2823
2824void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,
2825 BlockFieldFlags Flags,
2826 bool LoadBlockVarAddr, bool CanThrow) {
2827 EHStack.pushCleanup<CallBlockRelease>(Kind, A: Addr, A: Flags, A: LoadBlockVarAddr,
2828 A: CanThrow);
2829}
2830
2831/// Adjust the declaration of something from the blocks API.
2832static void configureBlocksRuntimeObject(CodeGenModule &CGM,
2833 llvm::Constant *C) {
2834 auto *GV = cast<llvm::GlobalValue>(Val: C->stripPointerCasts());
2835
2836 if (!CGM.getCodeGenOpts().StaticClosure &&
2837 CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
2838 const IdentifierInfo &II = CGM.getContext().Idents.get(Name: C->getName());
2839 TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
2840 DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
2841
2842 assert((isa<llvm::Function>(C->stripPointerCasts()) ||
2843 isa<llvm::GlobalVariable>(C->stripPointerCasts())) &&
2844 "expected Function or GlobalVariable");
2845
2846 const NamedDecl *ND = nullptr;
2847 for (const auto *Result : DC->lookup(Name: &II))
2848 if ((ND = dyn_cast<FunctionDecl>(Val: Result)) ||
2849 (ND = dyn_cast<VarDecl>(Val: Result)))
2850 break;
2851
2852 if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {
2853 GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2854 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2855 } else {
2856 GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2857 GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2858 }
2859 }
2860
2861 if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() &&
2862 GV->hasExternalLinkage())
2863 GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2864
2865 CGM.setDSOLocal(GV);
2866}
2867
2868llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() {
2869 if (BlockObjectDispose)
2870 return BlockObjectDispose;
2871
2872 QualType args[] = {Context.VoidPtrTy, Context.IntTy};
2873 BlockObjectDispose =
2874 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_dispose");
2875 configureBlocksRuntimeObject(
2876 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectDispose.getCallee()));
2877 return BlockObjectDispose;
2878}
2879
2880llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() {
2881 if (BlockObjectAssign)
2882 return BlockObjectAssign;
2883
2884 QualType args[] = {Context.VoidPtrTy, Context.VoidPtrTy, Context.IntTy};
2885 BlockObjectAssign =
2886 CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_assign");
2887 configureBlocksRuntimeObject(
2888 CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectAssign.getCallee()));
2889 return BlockObjectAssign;
2890}
2891
2892llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
2893 if (NSConcreteGlobalBlock)
2894 return NSConcreteGlobalBlock;
2895
2896 NSConcreteGlobalBlock = GetOrCreateLLVMGlobal(
2897 MangledName: "_NSConcreteGlobalBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2898 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteGlobalBlock);
2899 return NSConcreteGlobalBlock;
2900}
2901
2902llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
2903 if (NSConcreteStackBlock)
2904 return NSConcreteStackBlock;
2905
2906 NSConcreteStackBlock = GetOrCreateLLVMGlobal(
2907 MangledName: "_NSConcreteStackBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2908 configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteStackBlock);
2909 return NSConcreteStackBlock;
2910}
2911