1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
10#define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
11
12#include "Address.h"
13#include "CIRGenRecordLayout.h"
14#include "CIRGenTypeCache.h"
15#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
16#include "mlir/IR/Attributes.h"
17#include "mlir/IR/Builders.h"
18#include "mlir/IR/BuiltinAttributes.h"
19#include "mlir/Support/LLVM.h"
20#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
21#include "clang/CIR/MissingFeatures.h"
22
23#include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
24#include "clang/CIR/MissingFeatures.h"
25#include "llvm/ADT/APFloat.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/IR/FPEnv.h"
28
29namespace clang::CIRGen {
30
31class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
32 const CIRGenTypeCache &typeCache;
33
34 llvm::StringMap<unsigned> recordNames;
35 llvm::StringMap<unsigned> globalsVersioning;
36
37public:
38 CIRGenBuilderTy(mlir::MLIRContext &mlirContext, const CIRGenTypeCache &tc)
39 : CIRBaseBuilderTy(mlirContext), typeCache(tc) {}
40
41 /// Get a cir::ConstArrayAttr for a string literal.
42 /// Note: This is different from what is returned by
43 /// mlir::Builder::getStringAttr() which is an mlir::StringAttr.
44 mlir::Attribute getString(llvm::StringRef str, mlir::Type eltTy,
45 std::optional<size_t> size,
46 bool ensureNullTerm = true) {
47 size_t finalSize = size.value_or(u: str.size());
48
49 size_t lastNonZeroPos = str.find_last_not_of(C: '\0');
50 // If the string is full of null bytes, emit a #cir.zero rather than
51 // a #cir.const_array.
52 if (lastNonZeroPos == llvm::StringRef::npos) {
53 auto arrayTy = cir::ArrayType::get(elementType: eltTy, size: finalSize);
54 return cir::ZeroAttr::get(type: arrayTy);
55 }
56
57 // We emit trailing zeros for all trailing zeros, so the null-terminator in
58 // a constant is always in trailing zeros, and the null-terminator is
59 // skipped in the CIR representation.
60 size_t trailingZerosNum = finalSize - lastNonZeroPos - 1;
61 auto truncatedArrayTy =
62 cir::ArrayType::get(elementType: eltTy, size: finalSize - trailingZerosNum);
63 auto strAttr = mlir::StringAttr::get(bytes: str.drop_back(N: trailingZerosNum),
64 type: truncatedArrayTy);
65
66 // Most C strings are null terminated, so if we are ensuring there is one,
67 // grow the array size by 1 to add a trailing zero if necessary. The 'auto'
68 // calculation of trailing zeros (the difference between the provided string
69 // and the type) will ensure we get the count correct.
70 finalSize += (ensureNullTerm && trailingZerosNum == 0);
71
72 auto fullArrayTy = cir::ArrayType::get(elementType: eltTy, size: finalSize);
73 return cir::ConstArrayAttr::get(type: fullArrayTy, elts: strAttr);
74 }
75
76 cir::ConstArrayAttr getConstArray(mlir::Attribute attrs,
77 cir::ArrayType arrayTy) const {
78 return cir::ConstArrayAttr::get(type: arrayTy, elts: attrs);
79 }
80
81 mlir::Attribute getConstRecordOrZeroAttr(mlir::ArrayAttr arrayAttr,
82 cir::RecordType recordTy);
83
84 cir::ConstRecordAttr getAnonConstRecord(mlir::ArrayAttr arrayAttr) {
85 llvm::SmallVector<mlir::Type> members;
86 for (auto &f : arrayAttr) {
87 auto ta = mlir::cast<mlir::TypedAttr>(Val: f);
88 members.push_back(Elt: ta.getType());
89 }
90
91 auto sTy = getAnonRecordTy(members, /*packed=*/packed: false,
92 memberKinds: cir::RecordType::getAllDataKinds(members));
93 return cir::ConstRecordAttr::get(type: sTy, members: arrayAttr);
94 }
95
96 cir::TypeInfoAttr getTypeInfo(mlir::ArrayAttr fieldsAttr) {
97 cir::ConstRecordAttr anonRecord = getAnonConstRecord(arrayAttr: fieldsAttr);
98 return cir::TypeInfoAttr::get(type: anonRecord.getType(), data: fieldsAttr);
99 }
100
101 std::string getUniqueAnonRecordName() { return getUniqueRecordName(baseName: "anon"); }
102
103 std::string getUniqueRecordName(const std::string &baseName) {
104 auto it = recordNames.find(Key: baseName);
105 if (it == recordNames.end()) {
106 recordNames[baseName] = 0;
107 return baseName;
108 }
109
110 return baseName + "." + std::to_string(val: recordNames[baseName]++);
111 }
112
113 cir::LongDoubleType getLongDoubleTy(const llvm::fltSemantics &format) const {
114 if (&format == &llvm::APFloat::IEEEdouble())
115 return cir::LongDoubleType::get(context: getContext(), underlying: typeCache.doubleTy);
116 if (&format == &llvm::APFloat::x87DoubleExtended())
117 return cir::LongDoubleType::get(context: getContext(), underlying: typeCache.fP80Ty);
118 if (&format == &llvm::APFloat::IEEEquad())
119 return cir::LongDoubleType::get(context: getContext(), underlying: typeCache.fP128Ty);
120 if (&format == &llvm::APFloat::PPCDoubleDouble())
121 llvm_unreachable("NYI: PPC double-double format for long double");
122 llvm_unreachable("Unsupported format for long double");
123 }
124
125 mlir::Type getPtrToVPtrType() {
126 return getPointerTo(ty: cir::VPtrType::get(ctx: getContext()));
127 }
128
129 cir::FuncType getFuncType(llvm::ArrayRef<mlir::Type> params, mlir::Type retTy,
130 bool isVarArg = false) {
131 return cir::FuncType::get(inputs: params, returnType: retTy, isVarArg);
132 }
133
134 /// Get a CIR record kind from a AST declaration tag.
135 /// Returns true if the tag kind represents a C++ class (as opposed to a
136 /// plain struct).
137 static bool tagKindIsClass(const clang::TagTypeKind kind) {
138 return kind == clang::TagTypeKind::Class;
139 }
140
141 /// Returns true if the tag kind is a union.
142 static bool tagKindIsUnion(const clang::TagTypeKind kind) {
143 return kind == clang::TagTypeKind::Union;
144 }
145
146 /// Get a CIR named record type.
147 ///
148 /// If a record already exists and is complete, but the client tries to fetch
149 /// it with a different set of attributes, this method will crash.
150 cir::RecordType getCompleteNamedRecordType(
151 llvm::ArrayRef<mlir::Type> members, bool packed, llvm::StringRef name,
152 llvm::ArrayRef<cir::RecordMemberKind> memberKinds) {
153 const auto nameAttr = getStringAttr(bytes: name);
154 assert(!cir::MissingFeatures::astRecordDeclAttr());
155
156 // Always a struct at this call site, never a class or a union.
157 auto type = cir::StructType::get(context: getContext(), members, name: nameAttr, packed,
158 /*is_class=*/is_class: false, member_kinds: memberKinds);
159
160 // If we found an existing type, verify that either it is incomplete or
161 // it matches the requested attributes.
162 assert(!type.isIncomplete() ||
163 (type.getMembers() == members && type.getPacked() == packed));
164
165 // Complete an incomplete record or ensure the existing complete record
166 // matches the requested attributes.
167 type.complete(members, packed, memberKinds);
168
169 return type;
170 }
171
172 /// Get an incomplete CIR record type. If we have a complete record
173 /// declaration, we may create an incomplete type and then add the
174 /// members, so \p rd here may be complete.
175 cir::RecordType getIncompleteRecordTy(llvm::StringRef name,
176 const clang::RecordDecl *rd) {
177 const mlir::StringAttr nameAttr = getStringAttr(bytes: name);
178 if (rd && tagKindIsUnion(kind: rd->getTagKind()))
179 return cir::UnionType::get(context: getContext(), name: nameAttr);
180 bool is_class = rd && tagKindIsClass(kind: rd->getTagKind());
181 return cir::StructType::get(context: getContext(), name: nameAttr, is_class);
182 }
183
184 //
185 // Operation creation helpers
186 // --------------------------
187 //
188 using CIRBaseBuilderTy::createCopy;
189 cir::CopyOp createCopy(Address dst, Address src, bool isVolatile = false,
190 bool skipTailPadding = false) {
191 cir::CopyOp op = createCopy(dst: dst.getPointer(), src: src.getPointer(), isVolatile,
192 skipTailPadding);
193 op.setDstAlignment(dst.getAlignment().getQuantity());
194 op.setSrcAlignment(src.getAlignment().getQuantity());
195 return op;
196 }
197
198 cir::MemCpyOp createMemCpy(mlir::Location loc, Address dst, Address src,
199 mlir::Value len) {
200 return cir::MemCpyOp::create(
201 builder&: *this, location: loc, dst: dst.getPointer(), src: src.getPointer(), len,
202 dst_alignment: getI64IntegerAttr(value: dst.getAlignment().getQuantity()),
203 src_alignment: getI64IntegerAttr(value: src.getAlignment().getQuantity()));
204 }
205
206 cir::MemMoveOp createMemMove(mlir::Location loc, Address dst, Address src,
207 mlir::Value len) {
208 return cir::MemMoveOp::create(
209 builder&: *this, location: loc, dst: dst.getPointer(), src: src.getPointer(), len,
210 dst_alignment: getI64IntegerAttr(value: dst.getAlignment().getQuantity()),
211 src_alignment: getI64IntegerAttr(value: src.getAlignment().getQuantity()));
212 }
213
214 cir::MemSetOp createMemSet(mlir::Location loc, mlir::Value dst,
215 mlir::Value val, mlir::Value len) {
216 assert(val.getType() == getUInt8Ty());
217 return cir::MemSetOp::create(builder&: *this, location: loc, dst, alignment: {}, val, len);
218 }
219
220 cir::MemSetOp createMemSet(mlir::Location loc, Address dst, mlir::Value val,
221 mlir::Value len) {
222 mlir::IntegerAttr align = getAlignmentAttr(alignment: dst.getAlignment());
223 assert(val.getType() == getUInt8Ty());
224 return cir::MemSetOp::create(builder&: *this, location: loc, dst: dst.getPointer(), alignment: align, val, len);
225 }
226 // ---------------------------
227
228 cir::DataMemberAttr getDataMemberAttr(cir::DataMemberType ty,
229 llvm::ArrayRef<int32_t> path) {
230 return cir::DataMemberAttr::get(type: ty, path);
231 }
232
233 cir::DataMemberAttr getNullDataMemberAttr(cir::DataMemberType ty) {
234 return cir::DataMemberAttr::get(type: ty);
235 }
236
237 // Return true if the value is a null constant such as null pointer, (+0.0)
238 // for floating-point or zero initializer
239 bool isNullValue(mlir::Attribute attr) const {
240 if (mlir::isa<cir::ZeroAttr>(Val: attr))
241 return true;
242
243 if (const auto ptrVal = mlir::dyn_cast<cir::ConstPtrAttr>(Val&: attr))
244 return ptrVal.isNullValue();
245
246 if (const auto intVal = mlir::dyn_cast<cir::IntAttr>(Val&: attr))
247 return intVal.isNullValue();
248
249 if (const auto boolVal = mlir::dyn_cast<cir::BoolAttr>(Val&: attr))
250 return !boolVal.getValue();
251
252 if (auto fpAttr = mlir::dyn_cast<cir::FPAttr>(Val&: attr)) {
253 auto fpVal = fpAttr.getValue();
254 bool ignored;
255 llvm::APFloat fv(+0.0);
256 fv.convert(ToSemantics: fpVal.getSemantics(), RM: llvm::APFloat::rmNearestTiesToEven,
257 losesInfo: &ignored);
258 return fv.bitwiseIsEqual(RHS: fpVal);
259 }
260 if (const auto recordVal = mlir::dyn_cast<cir::ConstRecordAttr>(Val&: attr)) {
261 for (const auto elt : recordVal.getMembers()) {
262 // FIXME(cir): the record's ID should not be considered a member.
263 if (mlir::isa<mlir::StringAttr>(Val: elt))
264 continue;
265 if (!isNullValue(attr: elt))
266 return false;
267 }
268 return true;
269 }
270
271 if (const auto arrayVal = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: attr)) {
272 if (mlir::isa<mlir::StringAttr>(Val: arrayVal.getElts()))
273 return false;
274
275 return llvm::all_of(
276 Range: mlir::cast<mlir::ArrayAttr>(Val: arrayVal.getElts()),
277 P: [&](const mlir::Attribute &elt) { return isNullValue(attr: elt); });
278 }
279 return false;
280 }
281
282 //
283 // Type helpers
284 // ------------
285 //
286 cir::IntType getUIntNTy(int n) {
287 switch (n) {
288 case 8:
289 return getUInt8Ty();
290 case 16:
291 return getUInt16Ty();
292 case 32:
293 return getUInt32Ty();
294 case 64:
295 return getUInt64Ty();
296 default:
297 return cir::IntType::get(context: getContext(), width: n, is_signed: false);
298 }
299 }
300
301 cir::IntType getSIntNTy(int n) {
302 switch (n) {
303 case 8:
304 return getSInt8Ty();
305 case 16:
306 return getSInt16Ty();
307 case 32:
308 return getSInt32Ty();
309 case 64:
310 return getSInt64Ty();
311 default:
312 return cir::IntType::get(context: getContext(), width: n, is_signed: true);
313 }
314 }
315
316 cir::VoidType getVoidTy() { return typeCache.voidTy; }
317
318 cir::IntType getBoolMemoryTy() { return getUInt8Ty(); }
319
320 cir::IntType getSInt8Ty() { return typeCache.sInt8Ty; }
321 cir::IntType getSInt16Ty() { return typeCache.sInt16Ty; }
322 cir::IntType getSInt32Ty() { return typeCache.sInt32Ty; }
323 cir::IntType getSInt64Ty() { return typeCache.sInt64Ty; }
324
325 cir::IntType getUInt8Ty() { return typeCache.uInt8Ty; }
326 cir::IntType getUInt16Ty() { return typeCache.uInt16Ty; }
327 cir::IntType getUInt32Ty() { return typeCache.uInt32Ty; }
328 cir::IntType getUInt64Ty() { return typeCache.uInt64Ty; }
329
330 cir::FP16Type getFp16Ty() { return typeCache.fP16Ty; }
331 cir::BF16Type getBfloat16Ty() { return typeCache.bFloat16Ty; }
332 cir::SingleType getSingleTy() { return typeCache.floatTy; }
333 cir::DoubleType getDoubleTy() { return typeCache.doubleTy; }
334
335 cir::ConstantOp getConstInt(mlir::Location loc, llvm::APSInt intVal);
336
337 cir::ConstantOp getConstInt(mlir::Location loc, llvm::APInt intVal,
338 bool isUnsigned = true);
339
340 cir::ConstantOp getConstInt(mlir::Location loc, mlir::Type t, uint64_t c);
341
342 cir::ConstantOp getConstFP(mlir::Location loc, mlir::Type t,
343 llvm::APFloat fpVal);
344
345 bool isInt8Ty(mlir::Type i) {
346 return i == typeCache.uInt8Ty || i == typeCache.sInt8Ty;
347 }
348 bool isInt16Ty(mlir::Type i) {
349 return i == typeCache.uInt16Ty || i == typeCache.sInt16Ty;
350 }
351 bool isInt32Ty(mlir::Type i) {
352 return i == typeCache.uInt32Ty || i == typeCache.sInt32Ty;
353 }
354 bool isInt64Ty(mlir::Type i) {
355 return i == typeCache.uInt64Ty || i == typeCache.sInt64Ty;
356 }
357 bool isInt(mlir::Type i) { return mlir::isa<cir::IntType>(Val: i); }
358
359 cir::IntType getExtendedIntTy(cir::IntType ty, bool isSigned) {
360 switch (ty.getWidth()) {
361 case 8:
362 return isSigned ? typeCache.sInt16Ty : typeCache.uInt16Ty;
363 case 16:
364 return isSigned ? typeCache.sInt32Ty : typeCache.uInt32Ty;
365 case 32:
366 return isSigned ? typeCache.sInt64Ty : typeCache.uInt64Ty;
367 default:
368 llvm_unreachable("NYI");
369 }
370 }
371
372 cir::IntType getTruncatedIntTy(cir::IntType ty, bool isSigned) {
373 switch (ty.getWidth()) {
374 case 16:
375 return isSigned ? typeCache.sInt8Ty : typeCache.uInt8Ty;
376 case 32:
377 return isSigned ? typeCache.sInt16Ty : typeCache.uInt16Ty;
378 case 64:
379 return isSigned ? typeCache.sInt32Ty : typeCache.uInt32Ty;
380 default:
381 llvm_unreachable("NYI");
382 }
383 }
384
385 cir::VectorType
386 getExtendedOrTruncatedElementVectorType(cir::VectorType vt, bool isExtended,
387 bool isSigned = false) {
388 auto elementTy = mlir::dyn_cast_or_null<cir::IntType>(Val: vt.getElementType());
389 assert(elementTy && "expected int vector");
390 return cir::VectorType::get(element_type: isExtended
391 ? getExtendedIntTy(ty: elementTy, isSigned)
392 : getTruncatedIntTy(ty: elementTy, isSigned),
393 size: vt.getSize());
394 }
395
396 // Fetch the type representing a pointer to unsigned int8 values.
397 cir::PointerType getUInt8PtrTy() { return typeCache.uInt8PtrTy; }
398
399 /// Get a CIR anonymous struct type.
400 cir::StructType
401 getAnonRecordTy(llvm::ArrayRef<mlir::Type> members, bool packed,
402 llvm::ArrayRef<cir::RecordMemberKind> memberKinds) {
403 assert(!cir::MissingFeatures::astRecordDeclAttr());
404 return cir::StructType::get(context: getContext(), members, packed,
405 /*is_class=*/is_class: false, member_kinds: memberKinds);
406 }
407
408 //===--------------------------------------------------------------------===//
409 // Constant creation helpers
410 //===--------------------------------------------------------------------===//
411 cir::ConstantOp getSInt32(int32_t c, mlir::Location loc) {
412 return getConstantInt(loc, ty: getSInt32Ty(), value: c);
413 }
414 cir::ConstantOp getUInt32(uint32_t c, mlir::Location loc) {
415 return getConstantInt(loc, ty: getUInt32Ty(), value: c);
416 }
417 cir::ConstantOp getSInt64(uint64_t c, mlir::Location loc) {
418 return getConstantInt(loc, ty: getSInt64Ty(), value: c);
419 }
420 cir::ConstantOp getUInt64(uint64_t c, mlir::Location loc) {
421 return getConstantInt(loc, ty: getUInt64Ty(), value: c);
422 }
423
424 cir::ConstantOp getZero(mlir::Location loc, mlir::Type ty) {
425 // TODO: dispatch creation for primitive types.
426 assert((mlir::isa<cir::RecordType>(ty) || mlir::isa<cir::ArrayType>(ty) ||
427 mlir::isa<cir::VectorType>(ty)) &&
428 "NYI for other types");
429 return cir::ConstantOp::create(builder&: *this, location: loc, value: cir::ZeroAttr::get(type: ty));
430 }
431
432 //===--------------------------------------------------------------------===//
433 // UnaryOp creation helpers
434 //===--------------------------------------------------------------------===//
435 mlir::Value createNeg(mlir::Location loc, mlir::Value value,
436 bool nsw = false) {
437
438 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: value.getType())) {
439 // Source is a unsigned integer: first cast it to signed.
440 if (intTy.isUnsigned())
441 value = createIntCast(src: value, newTy: getSIntNTy(n: intTy.getWidth()));
442 return createMinus(loc, input: value, nsw);
443 }
444
445 llvm_unreachable("negation for the given type is NYI");
446 }
447
448 //===--------------------------------------------------------------------===//
449 // CastOp creation helpers
450 //===--------------------------------------------------------------------===//
451
452 // TODO: split this to createFPExt/createFPTrunc when we have dedicated cast
453 // operations.
454 mlir::Value createFloatingCast(mlir::Value v, mlir::Type destType) {
455 return cir::CastOp::create(builder&: *this, location: v.getLoc(), result: destType,
456 kind: cir::CastKind::floating, src: v,
457 fenv: getConstrainedFPAttr());
458 }
459
460 mlir::Value createDynCast(mlir::Location loc, mlir::Value src,
461 cir::PointerType destType, bool isRefCast,
462 cir::DynamicCastInfoAttr info) {
463 auto castKind =
464 isRefCast ? cir::DynamicCastKind::Ref : cir::DynamicCastKind::Ptr;
465 return cir::DynamicCastOp::create(builder&: *this, location: loc, result: destType, kind: castKind, src,
466 info);
467 }
468
469 mlir::Value createDynCastToVoid(mlir::Location loc, mlir::Value src) {
470 auto srcPtrTy = mlir::cast<cir::PointerType>(Val: src.getType());
471 cir::PointerType destTy = getVoidPtrTy(as: srcPtrTy.getAddrSpace());
472 return cir::DynamicCastOp::create(builder&: *this, location: loc, result: destTy,
473 kind: cir::DynamicCastKind::Ptr, src,
474 info: cir::DynamicCastInfoAttr{});
475 }
476
477 //===--------------------------------------------------------------------===//
478 // Address creation helpers
479 //===--------------------------------------------------------------------===//
480 Address createBaseClassAddr(mlir::Location loc, Address addr,
481 mlir::Type destType, unsigned offset,
482 bool assumeNotNull) {
483 if (destType == addr.getElementType())
484 return addr;
485
486 auto ptrTy = getPointerTo(ty: destType, as: addr.getAddressSpace());
487 auto baseAddr =
488 cir::BaseClassAddrOp::create(builder&: *this, location: loc, result: ptrTy, src_addr: addr.getPointer(),
489 offset: mlir::APInt(64, offset), assume_not_null: assumeNotNull);
490 return Address(baseAddr, destType, addr.getAlignment());
491 }
492
493 Address createDerivedClassAddr(mlir::Location loc, Address addr,
494 mlir::Type destType, unsigned offset,
495 bool assumeNotNull) {
496 if (destType == addr.getElementType())
497 return addr;
498
499 cir::PointerType ptrTy = getPointerTo(ty: destType, as: addr.getAddressSpace());
500 auto derivedAddr =
501 cir::DerivedClassAddrOp::create(builder&: *this, location: loc, result: ptrTy, src_addr: addr.getPointer(),
502 offset: mlir::APInt(64, offset), assume_not_null: assumeNotNull);
503 return Address(derivedAddr, destType, addr.getAlignment());
504 }
505
506 //===--------------------------------------------------------------------===//
507 // Virtual Address creation helpers
508 //===--------------------------------------------------------------------===//
509 mlir::Value createVTTAddrPoint(mlir::Location loc, mlir::Type retTy,
510 mlir::Value addr, uint64_t offset) {
511 return cir::VTTAddrPointOp::create(builder&: *this, location: loc, addr: retTy,
512 name: mlir::FlatSymbolRefAttr{}, sym_addr: addr, offset);
513 }
514
515 mlir::Value createVTTAddrPoint(mlir::Location loc, mlir::Type retTy,
516 mlir::FlatSymbolRefAttr sym, uint64_t offset) {
517 return cir::VTTAddrPointOp::create(builder&: *this, location: loc, addr: retTy, name: sym, sym_addr: mlir::Value{},
518 offset);
519 }
520
521 //===--------------------------------------------------------------------===//
522 // Other creation helpers
523 //===--------------------------------------------------------------------===//
524 cir::IsFPClassOp createIsFPClass(mlir::Location loc, mlir::Value src,
525 cir::FPClassTest flags) {
526 // FPClassTest occupies bits 0-9 (fcAllFlags). Sema rejects an
527 // out-of-range __builtin_isfpclass mask, so any extra bit here is an
528 // internal error; assert and mask it off so lowering stays well-formed.
529 uint32_t raw = static_cast<uint32_t>(flags);
530 uint32_t all = static_cast<uint32_t>(cir::FPClassTest::All);
531 assert((raw & ~all) == 0 && "FPClassTest mask has bits outside 0-9");
532 flags = static_cast<cir::FPClassTest>(raw & all);
533 return cir::IsFPClassOp::create(builder&: *this, location: loc, src, flags);
534 }
535
536 /// Cast the element type of the given address to a different type,
537 /// preserving information like the alignment.
538 Address createElementBitCast(mlir::Location loc, Address addr,
539 mlir::Type destType) {
540 if (destType == addr.getElementType())
541 return addr;
542
543 auto srcPtrTy = mlir::cast<cir::PointerType>(Val: addr.getPointer().getType());
544 auto ptrTy = getPointerTo(ty: destType, as: srcPtrTy.getAddrSpace());
545 return Address(createBitcast(loc, src: addr.getPointer(), newTy: ptrTy), destType,
546 addr.getAlignment());
547 }
548
549 cir::LoadOp createLoad(mlir::Location loc, Address addr,
550 bool isVolatile = false, bool isNontemporal = false) {
551 mlir::IntegerAttr align = getAlignmentAttr(alignment: addr.getAlignment());
552 return cir::LoadOp::create(builder&: *this, location: loc, addr: addr.getPointer(), /*isDeref=*/isDeref: false,
553 is_volatile: isVolatile, is_nontemporal: isNontemporal,
554 /*alignment=*/alignment: align,
555 /*sync_scope=*/sync_scope: cir::SyncScopeKindAttr{},
556 /*mem_order=*/mem_order: cir::MemOrderAttr{},
557 /*invariant=*/invariant: false);
558 }
559
560 cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty,
561 mlir::Value ptr, llvm::MaybeAlign align) {
562 if (ty != mlir::cast<cir::PointerType>(Val: ptr.getType()).getPointee())
563 ptr = createPtrBitcast(src: ptr, newPointeeTy: ty);
564 uint64_t alignment = align ? align->value() : 0;
565 mlir::IntegerAttr alignAttr = getAlignmentAttr(alignment);
566 return cir::LoadOp::create(builder&: *this, location: loc, addr: ptr, /*isDeref=*/isDeref: false,
567 /*isVolatile=*/is_volatile: false, /*isNontemporal=*/is_nontemporal: false,
568 alignment: alignAttr,
569 /*sync_scope=*/sync_scope: cir::SyncScopeKindAttr{},
570 /*mem_order=*/mem_order: cir::MemOrderAttr{},
571 /*invariant=*/invariant: false);
572 }
573
574 cir::LoadOp
575 createAlignedLoad(mlir::Location loc, mlir::Type ty, mlir::Value ptr,
576 clang::CharUnits align = clang::CharUnits::One()) {
577 return createAlignedLoad(loc, ty, ptr, align: align.getAsAlign());
578 }
579
580 cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst,
581 bool isVolatile = false, bool isNontemporal = false,
582 mlir::IntegerAttr align = {},
583 cir::SyncScopeKindAttr scope = {},
584 cir::MemOrderAttr order = {}) {
585 if (!align)
586 align = getAlignmentAttr(alignment: dst.getAlignment());
587 return CIRBaseBuilderTy::createStore(loc, val, dst: dst.getPointer(), isVolatile,
588 isNontemporal, align, scope, order);
589 }
590
591 /// Create a cir.complex.real_ptr operation that derives a pointer to the real
592 /// part of the complex value pointed to by the specified pointer value.
593 mlir::Value createComplexRealPtr(mlir::Location loc, mlir::Value value) {
594 auto srcPtrTy = mlir::cast<cir::PointerType>(Val: value.getType());
595 auto srcComplexTy = mlir::cast<cir::ComplexType>(Val: srcPtrTy.getPointee());
596 return cir::ComplexRealPtrOp::create(
597 builder&: *this, location: loc, result: getPointerTo(ty: srcComplexTy.getElementType()), operand: value);
598 }
599
600 Address createComplexRealPtr(mlir::Location loc, Address addr) {
601 return Address{createComplexRealPtr(loc, value: addr.getPointer()),
602 addr.getAlignment()};
603 }
604
605 /// Create a cir.complex.imag_ptr operation that derives a pointer to the
606 /// imaginary part of the complex value pointed to by the specified pointer
607 /// value.
608 mlir::Value createComplexImagPtr(mlir::Location loc, mlir::Value value) {
609 auto srcPtrTy = mlir::cast<cir::PointerType>(Val: value.getType());
610 auto srcComplexTy = mlir::cast<cir::ComplexType>(Val: srcPtrTy.getPointee());
611 return cir::ComplexImagPtrOp::create(
612 builder&: *this, location: loc, result: getPointerTo(ty: srcComplexTy.getElementType()), operand: value);
613 }
614
615 Address createComplexImagPtr(mlir::Location loc, Address addr) {
616 return Address{createComplexImagPtr(loc, value: addr.getPointer()),
617 addr.getAlignment()};
618 }
619
620 using CIRBaseBuilderTy::createGetMember;
621 Address createGetMember(mlir::Location loc, Address base,
622 llvm::StringRef name, unsigned index,
623 const cir::CIRDataLayout &layout) {
624 auto recordTy = mlir::cast<cir::RecordType>(Val: base.getElementType());
625
626 assert(index < recordTy.getMembers().size() &&
627 "member index out of bounds");
628 mlir::Type memberTy = recordTy.getMembers()[index];
629 mlir::Type memberPtrTy = getPointerTo(ty: memberTy, as: base.getAddressSpace());
630
631 auto memberOffset = CharUnits::fromQuantity(
632 Quantity: recordTy.getElementOffset(dataLayout: layout.layout, idx: index));
633
634 mlir::Value memberPtr =
635 createGetMember(loc, resultTy: memberPtrTy, base: base.getBasePointer(), name, index);
636 return Address(memberPtr, memberTy,
637 base.getAlignment().alignmentAtOffset(offset: memberOffset),
638 base.isKnownNonNull());
639 }
640
641 cir::GetRuntimeMemberOp createGetIndirectMember(mlir::Location loc,
642 mlir::Value objectPtr,
643 mlir::Value memberPtr) {
644 auto memberPtrTy = mlir::cast<cir::DataMemberType>(Val: memberPtr.getType());
645 auto objectPtrTy = mlir::cast<cir::PointerType>(Val: objectPtr.getType());
646
647 // The member lives in the same address space as the object.
648 cir::PointerType resultTy =
649 getPointerTo(ty: memberPtrTy.getMemberTy(), as: objectPtrTy.getAddrSpace());
650
651 return cir::GetRuntimeMemberOp::create(builder&: *this, location: loc, result: resultTy, addr: objectPtr,
652 member: memberPtr);
653 }
654
655 /// Create a cir.get_element or cir.ptr_stride operation to get access to an
656 /// array element. \p idx is the index of the element to access,
657 /// \p shouldDecay is true if the result should decay to a pointer to the
658 /// element type. \p flags are set on the cir.get_element emitted when
659 /// \p shouldDecay is true and \p arrayPtr points to the array type
660 /// \p eltTy, and are dropped when a cir.ptr_stride is emitted.
661 mlir::Value
662 getArrayElement(mlir::Location arrayLocBegin, mlir::Location arrayLocEnd,
663 mlir::Value arrayPtr, mlir::Type eltTy, mlir::Value idx,
664 bool shouldDecay,
665 cir::PtrOffsetFlags flags = cir::PtrOffsetFlags::None);
666
667 /// Returns a decayed pointer to the first element of the array
668 /// pointed to by \p arrayPtr.
669 mlir::Value maybeBuildArrayDecay(mlir::Location loc, mlir::Value arrayPtr,
670 mlir::Type eltTy);
671
672 // Convert byte offset to sequence of high-level indices suitable for
673 // GlobalViewAttr. Ideally we shouldn't deal with low-level offsets at all
674 // but currently some parts of Clang AST, which we don't want to touch just
675 // yet, return them.
676 //
677 // Returns false if the offset doesn't designate a subelement of \p ty, which
678 // happens when it lands outside of the object or in the middle of a scalar
679 // member. In that case \p indices is left in an unspecified state and the
680 // caller must describe the address with a byte offset, using a
681 // GlobalOffsetAttr, instead.
682 [[nodiscard]] bool computeGlobalViewIndicesFromFlatOffset(
683 int64_t offset, mlir::Type ty, const cir::CIRDataLayout &layout,
684 llvm::SmallVectorImpl<int64_t> &indices);
685
686 // Convert high-level indices (e.g. from GlobalViewAttr) to byte offset.
687 uint64_t computeOffsetFromGlobalViewIndices(const cir::CIRDataLayout &layout,
688 mlir::Type ty,
689 llvm::ArrayRef<int64_t> indices);
690
691 /// Creates a versioned global variable. If the symbol is already taken, an ID
692 /// will be appended to the symbol. The returned global must always be queried
693 /// for its name so it can be referenced correctly.
694 [[nodiscard]] cir::GlobalOp
695 createVersionedGlobal(mlir::ModuleOp module, mlir::Location loc,
696 mlir::StringRef name, mlir::Type type, bool isConstant,
697 cir::GlobalLinkageKind linkage,
698 mlir::ptr::MemorySpaceAttrInterface addrSpace = {}) {
699 // Create a unique name if the given name is already taken.
700 std::string uniqueName;
701 if (unsigned version = globalsVersioning[name.str()]++)
702 uniqueName = name.str() + "." + std::to_string(val: version);
703 else
704 uniqueName = name.str();
705
706 return createGlobal(mlirModule: module, loc, name: uniqueName, type, isConstant, linkage,
707 addrSpace);
708 }
709
710 cir::StackSaveOp createStackSave(mlir::Location loc, mlir::Type ty) {
711 return cir::StackSaveOp::create(builder&: *this, location: loc, result: ty);
712 }
713
714 cir::StackRestoreOp createStackRestore(mlir::Location loc, mlir::Value v) {
715 return cir::StackRestoreOp::create(builder&: *this, location: loc, ptr: v);
716 }
717
718 cir::CmpThreeWayOp createThreeWayCmpTotalOrdering(
719 mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
720 const llvm::APSInt &ltRes, const llvm::APSInt &eqRes,
721 const llvm::APSInt &gtRes, cir::CmpOrdering ordering) {
722 assert(ltRes.getBitWidth() == eqRes.getBitWidth() &&
723 ltRes.getBitWidth() == gtRes.getBitWidth() &&
724 "the three comparison results must have the same bit width");
725 assert((ordering == cir::CmpOrdering::Strong ||
726 ordering == cir::CmpOrdering::Weak) &&
727 "total ordering must be strong or weak");
728 cir::IntType cmpResultTy = getSIntNTy(n: ltRes.getBitWidth());
729 auto infoAttr = cir::CmpThreeWayInfoAttr::get(
730 context: getContext(), ordering, lt: ltRes.getSExtValue(), eq: eqRes.getSExtValue(),
731 gt: gtRes.getSExtValue());
732 return cir::CmpThreeWayOp::create(builder&: *this, location: loc, result: cmpResultTy, lhs, rhs,
733 info: infoAttr);
734 }
735
736 cir::CmpThreeWayOp createThreeWayCmpPartialOrdering(
737 mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
738 const llvm::APSInt &ltRes, const llvm::APSInt &eqRes,
739 const llvm::APSInt &gtRes, const llvm::APSInt &unorderedRes) {
740 assert(ltRes.getBitWidth() == eqRes.getBitWidth() &&
741 ltRes.getBitWidth() == gtRes.getBitWidth() &&
742 ltRes.getBitWidth() == unorderedRes.getBitWidth() &&
743 "the four comparison results must have the same bit width");
744 cir::IntType cmpResultTy = getSIntNTy(n: ltRes.getBitWidth());
745 auto infoAttr = cir::CmpThreeWayInfoAttr::get(
746 context: getContext(), lt: ltRes.getSExtValue(), eq: eqRes.getSExtValue(),
747 gt: gtRes.getSExtValue(), unordered: unorderedRes.getSExtValue());
748 return cir::CmpThreeWayOp::create(builder&: *this, location: loc, result: cmpResultTy, lhs, rhs,
749 info: infoAttr);
750 }
751
752 mlir::Value createSetBitfield(mlir::Location loc, mlir::Type resultType,
753 Address dstAddr, mlir::Type storageType,
754 mlir::Value src, const CIRGenBitFieldInfo &info,
755 bool isLvalueVolatile, bool useVolatile) {
756 unsigned offset = useVolatile ? info.volatileOffset : info.offset;
757
758 // If using AAPCS and the field is volatile, load with the size of the
759 // declared field
760 storageType =
761 useVolatile ? cir::IntType::get(context: storageType.getContext(),
762 width: info.volatileStorageSize, is_signed: info.isSigned)
763 : storageType;
764 return cir::SetBitfieldOp::create(
765 builder&: *this, location: loc, type: resultType, addr: dstAddr.getPointer(), storage_type: storageType, src,
766 name: info.name, size: info.size, offset, is_signed: info.isSigned, is_volatile: isLvalueVolatile,
767 alignment: dstAddr.getAlignment().getAsAlign().value());
768 }
769
770 mlir::Value createGetBitfield(mlir::Location loc, mlir::Type resultType,
771 Address addr, mlir::Type storageType,
772 const CIRGenBitFieldInfo &info,
773 bool isLvalueVolatile, bool useVolatile) {
774 unsigned offset = useVolatile ? info.volatileOffset : info.offset;
775
776 // If using AAPCS and the field is volatile, load with the size of the
777 // declared field
778 storageType =
779 useVolatile ? cir::IntType::get(context: storageType.getContext(),
780 width: info.volatileStorageSize, is_signed: info.isSigned)
781 : storageType;
782 return cir::GetBitfieldOp::create(builder&: *this, location: loc, type: resultType, addr: addr.getPointer(),
783 storage_type: storageType, name: info.name, size: info.size, offset,
784 is_signed: info.isSigned, is_volatile: isLvalueVolatile,
785 alignment: addr.getAlignment().getAsAlign().value());
786 }
787
788 mlir::Value createMaskedLoad(mlir::Location loc, mlir::Type ty,
789 mlir::Value ptr, llvm::Align alignment,
790 mlir::Value mask, mlir::Value passThru) {
791 assert(mlir::isa<cir::VectorType>(ty) && "Type should be vector");
792 assert(mask && "Mask should not be all-ones (null)");
793
794 if (!passThru)
795 passThru = this->getConstant(loc, attr: cir::PoisonAttr::get(type: ty));
796
797 auto alignAttr =
798 this->getI64IntegerAttr(value: static_cast<int64_t>(alignment.value()));
799
800 return cir::VecMaskedLoadOp::create(builder&: *this, location: loc, result: ty, addr: ptr, mask, pass_thru: passThru,
801 alignment: alignAttr);
802 }
803
804 cir::VecShuffleOp
805 createVecShuffle(mlir::Location loc, mlir::Value vec1, mlir::Value vec2,
806 llvm::ArrayRef<mlir::Attribute> maskAttrs) {
807 auto vecType = mlir::cast<cir::VectorType>(Val: vec1.getType());
808 auto resultTy =
809 cir::VectorType::getFixed(elementType: vecType.getElementType(), size: maskAttrs.size());
810 return cir::VecShuffleOp::create(builder&: *this, location: loc, result: resultTy, vec1, vec2,
811 indices: getArrayAttr(value: maskAttrs));
812 }
813
814 cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1,
815 mlir::Type resultTy,
816 llvm::ArrayRef<int64_t> mask) {
817 cir::ConstantOp poison =
818 getConstant(loc, attr: cir::PoisonAttr::get(type: vec1.getType()));
819 auto maskAttrs = llvm::to_vector_of<mlir::Attribute>(
820 Range: llvm::map_range(C&: mask, F: [&](int32_t idx) {
821 return cir::IntAttr::get(type: getSInt32Ty(), value: idx);
822 }));
823 return cir::VecShuffleOp::create(builder&: *this, location: loc, result: resultTy, vec1, vec2: poison,
824 indices: getArrayAttr(value: maskAttrs));
825 }
826
827 cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1,
828 mlir::Value vec2,
829 llvm::ArrayRef<int64_t> mask) {
830 auto maskAttrs = llvm::to_vector_of<mlir::Attribute>(
831 Range: llvm::map_range(C&: mask, F: [&](int32_t idx) {
832 return cir::IntAttr::get(type: getSInt32Ty(), value: idx);
833 }));
834 return createVecShuffle(loc, vec1, vec2, maskAttrs);
835 }
836
837 cir::VecShuffleOp createVecShuffle(mlir::Location loc, mlir::Value vec1,
838 llvm::ArrayRef<int64_t> mask) {
839 /// Create a unary shuffle. The second vector operand of the IR instruction
840 /// is poison.
841 cir::ConstantOp poison =
842 getConstant(loc, attr: cir::PoisonAttr::get(type: vec1.getType()));
843 return createVecShuffle(loc, vec1, vec2: poison, mask);
844 }
845
846 cir::MatrixExtractOp createMatrixExtract(mlir::Location loc,
847 mlir::Value matrix,
848 mlir::Value rowIdx,
849 mlir::Value columnIdx) {
850 return cir::MatrixExtractOp::create(builder&: *this, location: loc, matrix, rowIdx, columnIdx);
851 }
852
853 cir::MatrixExtractRowOp createMatrixExtractRow(mlir::Location loc,
854 mlir::Value matrix,
855 mlir::Value rowIdx) {
856 return cir::MatrixExtractRowOp::create(builder&: *this, location: loc, matrix, rowIdx);
857 }
858
859 cir::MatrixInsertOp createMatrixInsert(mlir::Location loc, mlir::Value value,
860 mlir::Value matrix, mlir::Value rowIdx,
861 mlir::Value columnIdx) {
862 return cir::MatrixInsertOp::create(builder&: *this, location: loc, value, matrix, rowIdx,
863 columnIdx);
864 }
865
866 cir::MatrixColumnMajorLoadOp createMatrixColumnMajorLoad(mlir::Location loc,
867 mlir::Type resultTy,
868 mlir::Value value,
869 mlir::Value stride,
870 bool isVolatile) {
871 return cir::MatrixColumnMajorLoadOp::create(builder&: *this, location: loc, result: resultTy, value,
872 stride, is_volatile: isVolatile);
873 }
874
875 /// Multiply an MxK matrix by a KxN matrix, producing an MxN matrix.
876 cir::MatrixMultiplyOp createMatrixMultiply(mlir::Location loc,
877 mlir::Value lhs, mlir::Value rhs) {
878 auto lhsTy = mlir::cast<cir::MatrixType>(Val: lhs.getType());
879 auto rhsTy = mlir::cast<cir::MatrixType>(Val: rhs.getType());
880 auto resultTy = cir::MatrixType::get(
881 element_type: lhsTy.getElementType(), num_rows: lhsTy.getNumRows(), num_columns: rhsTy.getNumColumns());
882 return cir::MatrixMultiplyOp::create(builder&: *this, location: loc, result: resultTy, lhs, rhs);
883 }
884
885 cir::MatrixTransposeOp createMatrixTranspose(mlir::Location loc,
886 mlir::Value matrix) {
887 auto inputTy = mlir::cast<cir::MatrixType>(Val: matrix.getType());
888 auto resultTy =
889 cir::MatrixType::get(element_type: inputTy.getElementType(), num_rows: inputTy.getNumColumns(),
890 num_columns: inputTy.getNumRows());
891 return cir::MatrixTransposeOp::create(builder&: *this, location: loc, result: resultTy, value: matrix);
892 }
893
894 cir::MatrixColumnMajorStoreOp createMatrixColumnMajorStore(mlir::Location loc,
895 mlir::Value matrix,
896 mlir::Value data,
897 mlir::Value stride,
898 bool isVolatile) {
899 return cir::MatrixColumnMajorStoreOp::create(builder&: *this, location: loc, matrix, value: data,
900 stride, is_volatile: isVolatile);
901 }
902
903 std::pair<mlir::Value, mlir::Value>
904 splatMatrixOpOperandsIfNecessary(mlir::Location loc, mlir::Value lhs,
905 mlir::Value rhs) {
906 assert(mlir::isa<cir::MatrixType>(lhs.getType()) ||
907 mlir::isa<cir::MatrixType>(rhs.getType()));
908
909 if (!mlir::isa<cir::MatrixType>(Val: lhs.getType()))
910 lhs = cir::VecSplatOp::create(builder&: *this, location: loc, result: rhs.getType(), value: lhs);
911 else if (!mlir::isa<cir::MatrixType>(Val: rhs.getType()))
912 rhs = cir::VecSplatOp::create(builder&: *this, location: loc, result: lhs.getType(), value: rhs);
913
914 return {lhs, rhs};
915 }
916
917 template <typename... Operands>
918 mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str,
919 const mlir::Type &resTy, Operands &&...op) {
920 return cir::LLVMIntrinsicCallOp::create(*this, loc,
921 this->getStringAttr(bytes: str), resTy,
922 std::forward<Operands>(op)...)
923 .getResult();
924 }
925
926 template <typename... Operands>
927 mlir::Value emitIntrinsicCallOp(mlir::Location loc, const llvm::StringRef str,
928 const mlir::Type &resTy,
929 cir::FastMathFlagsAttr fastmath,
930 Operands &&...op) {
931 return cir::LLVMIntrinsicCallOp::create(
932 *this, loc, this->getStringAttr(bytes: str), resTy,
933 std::forward<Operands>(op)..., fastmath)
934 .getResult();
935 }
936
937 /// Converts \p src to \p dstTy, saturating at its bounds. The signedness of
938 /// \p dstTy selects between fptosi.sat and fptoui.sat.
939 mlir::Value createFPToIntSat(mlir::Location loc, mlir::Value src,
940 cir::IntType dstTy) {
941 return emitIntrinsicCallOp(
942 loc, str: dstTy.isSigned() ? "fptosi.sat" : "fptoui.sat", resTy: dstTy, op&: src);
943 }
944};
945
946} // namespace clang::CIRGen
947
948#endif
949