| 1 | //===--- CIRGenAsm.cpp - Inline Assembly Support for CIR CodeGen ---------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains code to emit inline assembly. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "clang/Basic/DiagnosticSema.h" |
| 14 | |
| 15 | #include "CIRGenFunction.h" |
| 16 | #include "clang/CIR/MissingFeatures.h" |
| 17 | |
| 18 | using namespace clang; |
| 19 | using namespace clang::CIRGen; |
| 20 | using namespace cir; |
| 21 | |
| 22 | static AsmFlavor inferFlavor(const CIRGenModule &cgm, const AsmStmt &s) { |
| 23 | AsmFlavor gnuAsmFlavor = |
| 24 | cgm.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT |
| 25 | ? AsmFlavor::x86_att |
| 26 | : AsmFlavor::x86_intel; |
| 27 | |
| 28 | return isa<MSAsmStmt>(Val: &s) ? AsmFlavor::x86_intel : gnuAsmFlavor; |
| 29 | } |
| 30 | |
| 31 | static void collectClobbers(const CIRGenFunction &cgf, const AsmStmt &s, |
| 32 | std::string &constraints, bool &hasUnwindClobber, |
| 33 | bool &readOnly, bool readNone) { |
| 34 | |
| 35 | hasUnwindClobber = false; |
| 36 | const CIRGenModule &cgm = cgf.getCIRGenModule(); |
| 37 | |
| 38 | // Clobbers |
| 39 | for (unsigned i = 0, e = s.getNumClobbers(); i != e; i++) { |
| 40 | std::string clobber = s.getClobber(i); |
| 41 | if (clobber == "memory" ) { |
| 42 | readOnly = readNone = false; |
| 43 | } else if (clobber == "unwind" ) { |
| 44 | hasUnwindClobber = true; |
| 45 | continue; |
| 46 | } else if (clobber != "cc" ) { |
| 47 | clobber = cgf.getTarget().getNormalizedGCCRegisterName(Name: clobber); |
| 48 | if (cgm.getCodeGenOpts().StackClashProtector && |
| 49 | cgf.getTarget().isSPRegName(clobber)) |
| 50 | cgm.getDiags().Report(Loc: s.getAsmLoc(), |
| 51 | DiagID: diag::warn_stack_clash_protection_inline_asm); |
| 52 | } |
| 53 | |
| 54 | if (isa<MSAsmStmt>(Val: &s)) { |
| 55 | if (clobber == "eax" || clobber == "edx" ) { |
| 56 | if (constraints.find(s: "=&A" ) != std::string::npos) |
| 57 | continue; |
| 58 | std::string::size_type position1 = |
| 59 | constraints.find(str: "={" + clobber + "}" ); |
| 60 | if (position1 != std::string::npos) { |
| 61 | constraints.insert(pos: position1 + 1, s: "&" ); |
| 62 | continue; |
| 63 | } |
| 64 | std::string::size_type position2 = constraints.find(s: "=A" ); |
| 65 | if (position2 != std::string::npos) { |
| 66 | constraints.insert(pos: position2 + 1, s: "&" ); |
| 67 | continue; |
| 68 | } |
| 69 | } |
| 70 | } |
| 71 | if (!constraints.empty()) |
| 72 | constraints += ','; |
| 73 | |
| 74 | constraints += "~{" ; |
| 75 | constraints += clobber; |
| 76 | constraints += '}'; |
| 77 | } |
| 78 | |
| 79 | // Add machine specific clobbers |
| 80 | std::string_view machineClobbers = cgf.getTarget().getClobbers(); |
| 81 | if (!machineClobbers.empty()) { |
| 82 | if (!constraints.empty()) |
| 83 | constraints += ','; |
| 84 | constraints += machineClobbers; |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | static void |
| 89 | collectInOutConstraintInfos(const CIRGenFunction &cgf, const AsmStmt &s, |
| 90 | SmallVectorImpl<TargetInfo::ConstraintInfo> &out, |
| 91 | SmallVectorImpl<TargetInfo::ConstraintInfo> &in) { |
| 92 | |
| 93 | for (unsigned i = 0, e = s.getNumOutputs(); i != e; ++i) { |
| 94 | StringRef name; |
| 95 | if (const GCCAsmStmt *gas = dyn_cast<GCCAsmStmt>(Val: &s)) |
| 96 | name = gas->getOutputName(i); |
| 97 | TargetInfo::ConstraintInfo info(s.getOutputConstraint(i), name); |
| 98 | // `validateOutputConstraint` modifies the `info` object by setting the |
| 99 | // read/write, clobber, allows-register, and allows-memory process. |
| 100 | bool isValid = cgf.getTarget().validateOutputConstraint(Info&: info); |
| 101 | (void)isValid; |
| 102 | assert(isValid && "Failed to parse output constraint" ); |
| 103 | out.push_back(Elt: info); |
| 104 | } |
| 105 | |
| 106 | for (unsigned i = 0, e = s.getNumInputs(); i != e; ++i) { |
| 107 | StringRef name; |
| 108 | if (const GCCAsmStmt *gas = dyn_cast<GCCAsmStmt>(Val: &s)) |
| 109 | name = gas->getInputName(i); |
| 110 | TargetInfo::ConstraintInfo info(s.getInputConstraint(i), name); |
| 111 | // `validateInputConstraint` modifies the `info` object by setting the |
| 112 | // read/write, clobber, allows-register, and allows-memory process. |
| 113 | bool isValid = cgf.getTarget().validateInputConstraint(OutputConstraints: out, info); |
| 114 | assert(isValid && "Failed to parse input constraint" ); |
| 115 | (void)isValid; |
| 116 | in.push_back(Elt: info); |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | std::pair<mlir::Value, mlir::Type> CIRGenFunction::emitAsmInputLValue( |
| 121 | const TargetInfo::ConstraintInfo &info, LValue inputValue, |
| 122 | QualType inputType, std::string &constraintString, SourceLocation loc) { |
| 123 | |
| 124 | if (info.allowsRegister() || !info.allowsMemory()) { |
| 125 | if (hasScalarEvaluationKind(type: inputType)) |
| 126 | return {emitLoadOfLValue(lv: inputValue, loc).getValue(), mlir::Type()}; |
| 127 | |
| 128 | mlir::Type ty = convertType(t: inputType); |
| 129 | uint64_t size = cgm.getDataLayout().getTypeSizeInBits(ty); |
| 130 | if ((size <= 64 && llvm::isPowerOf2_64(Value: size)) || |
| 131 | getTargetHooks().isScalarizableAsmOperand(cgf&: *this, ty)) { |
| 132 | ty = cir::IntType::get(context: &getMLIRContext(), width: size, is_signed: false); |
| 133 | |
| 134 | return {builder.createLoad( |
| 135 | loc: getLoc(srcLoc: loc), |
| 136 | addr: inputValue.getAddress().withElementType(builder, ElemTy: ty)), |
| 137 | mlir::Type()}; |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | Address addr = inputValue.getAddress(); |
| 142 | constraintString += '*'; |
| 143 | return {addr.getPointer(), addr.getElementType()}; |
| 144 | } |
| 145 | |
| 146 | std::pair<mlir::Value, mlir::Type> |
| 147 | CIRGenFunction::emitAsmInput(const TargetInfo::ConstraintInfo &info, |
| 148 | const Expr *inputExpr, |
| 149 | std::string &constraintString) { |
| 150 | mlir::Location loc = getLoc(srcLoc: inputExpr->getExprLoc()); |
| 151 | |
| 152 | // If this can't be a register or memory, i.e., has to be a constant |
| 153 | // (immediate or symbolic), try to emit it as such. |
| 154 | if (!info.allowsRegister() && !info.allowsMemory()) { |
| 155 | if (info.requiresImmediateConstant()) { |
| 156 | Expr::EvalResult evalResult; |
| 157 | inputExpr->EvaluateAsRValue(Result&: evalResult, Ctx: getContext(), InConstantContext: true); |
| 158 | |
| 159 | llvm::APSInt intResult; |
| 160 | if (evalResult.Val.toIntegralConstant(Result&: intResult, SrcTy: inputExpr->getType(), |
| 161 | Ctx: getContext())) |
| 162 | return {builder.getConstInt(loc, intVal: intResult), mlir::Type()}; |
| 163 | } |
| 164 | |
| 165 | Expr::EvalResult result; |
| 166 | if (inputExpr->EvaluateAsInt(Result&: result, Ctx: getContext())) |
| 167 | return {builder.getConstInt(loc, intVal: result.Val.getInt()), mlir::Type()}; |
| 168 | } |
| 169 | |
| 170 | if (info.allowsRegister() || !info.allowsMemory()) |
| 171 | if (CIRGenFunction::hasScalarEvaluationKind(type: inputExpr->getType())) |
| 172 | return {emitScalarExpr(e: inputExpr), mlir::Type()}; |
| 173 | if (inputExpr->getStmtClass() == Expr::CXXThisExprClass) |
| 174 | return {emitScalarExpr(e: inputExpr), mlir::Type()}; |
| 175 | inputExpr = inputExpr->IgnoreParenNoopCasts(Ctx: getContext()); |
| 176 | LValue dest = emitLValue(e: inputExpr); |
| 177 | return emitAsmInputLValue(info, inputValue: dest, inputType: inputExpr->getType(), constraintString, |
| 178 | loc: inputExpr->getExprLoc()); |
| 179 | } |
| 180 | |
| 181 | static void emitAsmStores(CIRGenFunction &cgf, const AsmStmt &s, |
| 182 | const llvm::ArrayRef<mlir::Value> regResults, |
| 183 | const llvm::ArrayRef<mlir::Type> resultRegTypes, |
| 184 | const llvm::ArrayRef<mlir::Type> resultTruncRegTypes, |
| 185 | const llvm::ArrayRef<LValue> resultRegDests, |
| 186 | const llvm::ArrayRef<QualType> resultRegQualTys, |
| 187 | const llvm::BitVector &resultTypeRequiresCast, |
| 188 | const llvm::BitVector &resultRegIsFlagReg) { |
| 189 | CIRGenBuilderTy &builder = cgf.getBuilder(); |
| 190 | CIRGenModule &cgm = cgf.cgm; |
| 191 | mlir::MLIRContext *ctx = builder.getContext(); |
| 192 | |
| 193 | assert(regResults.size() == resultRegTypes.size()); |
| 194 | assert(regResults.size() == resultTruncRegTypes.size()); |
| 195 | assert(regResults.size() == resultRegDests.size()); |
| 196 | |
| 197 | // ResultRegDests can be also populated by addReturnRegisterOutputs() above, |
| 198 | // in which case its size may grow. |
| 199 | assert(resultTypeRequiresCast.size() <= resultRegDests.size()); |
| 200 | assert(resultRegIsFlagReg.size() <= resultRegDests.size()); |
| 201 | |
| 202 | for (unsigned i = 0, e = regResults.size(); i != e; ++i) { |
| 203 | mlir::Value tmp = regResults[i]; |
| 204 | mlir::Type truncTy = resultTruncRegTypes[i]; |
| 205 | |
| 206 | if (i < resultRegIsFlagReg.size() && resultRegIsFlagReg[i]) |
| 207 | assert(!cir::MissingFeatures::asmLLVMAssume()); |
| 208 | |
| 209 | // If the result type of the LLVM IR asm doesn't match the result type of |
| 210 | // the expression, do the conversion. |
| 211 | if (resultRegTypes[i] != truncTy) { |
| 212 | |
| 213 | // Truncate the integer result to the right size, note that TruncTy can be |
| 214 | // a pointer. |
| 215 | if (mlir::isa<cir::FPTypeInterface>(Val: truncTy)) { |
| 216 | tmp = builder.createFloatingCast(v: tmp, destType: truncTy); |
| 217 | } else if (isa<cir::PointerType>(Val: truncTy) && |
| 218 | isa<cir::IntType>(Val: tmp.getType())) { |
| 219 | uint64_t resSize = cgm.getDataLayout().getTypeSizeInBits(ty: truncTy); |
| 220 | tmp = builder.createIntCast( |
| 221 | src: tmp, newTy: cir::IntType::get(context: ctx, width: (unsigned)resSize, is_signed: false)); |
| 222 | tmp = builder.createIntToPtr(src: tmp, newTy: truncTy); |
| 223 | } else if (isa<cir::PointerType>(Val: tmp.getType()) && |
| 224 | isa<cir::IntType>(Val: truncTy)) { |
| 225 | uint64_t tmpSize = cgm.getDataLayout().getTypeSizeInBits(ty: tmp.getType()); |
| 226 | tmp = builder.createPtrToInt( |
| 227 | src: tmp, newTy: cir::IntType::get(context: ctx, width: (unsigned)tmpSize, is_signed: false)); |
| 228 | tmp = builder.createIntCast(src: tmp, newTy: truncTy); |
| 229 | } else if (isa<cir::IntType>(Val: truncTy)) { |
| 230 | tmp = builder.createIntCast(src: tmp, newTy: truncTy); |
| 231 | } else if (isa<cir::VectorType>(Val: truncTy)) { |
| 232 | assert(!cir::MissingFeatures::asmVectorType()); |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | LValue dest = resultRegDests[i]; |
| 237 | // ResultTypeRequiresCast elements correspond to the first |
| 238 | // ResultTypeRequiresCast.size() elements of RegResults. |
| 239 | if ((i < resultTypeRequiresCast.size()) && resultTypeRequiresCast[i]) { |
| 240 | unsigned size = cgf.getContext().getTypeSize(T: resultRegQualTys[i]); |
| 241 | Address addr = |
| 242 | dest.getAddress().withElementType(builder, ElemTy: resultRegTypes[i]); |
| 243 | if (cgm.getTargetCIRGenInfo().isScalarizableAsmOperand(cgf, ty: truncTy)) { |
| 244 | builder.createStore(loc: cgf.getLoc(srcLoc: s.getAsmLoc()), val: tmp, dst: addr); |
| 245 | continue; |
| 246 | } |
| 247 | |
| 248 | QualType ty = |
| 249 | cgf.getContext().getIntTypeForBitwidth(DestWidth: size, /*Signed=*/false); |
| 250 | if (ty.isNull()) { |
| 251 | const Expr *outExpr = s.getOutputExpr(i); |
| 252 | cgm.getDiags().Report(Loc: outExpr->getExprLoc(), |
| 253 | DiagID: diag::err_store_value_to_reg); |
| 254 | return; |
| 255 | } |
| 256 | dest = cgf.makeAddrLValue(addr, ty); |
| 257 | } |
| 258 | |
| 259 | cgf.emitStoreThroughLValue(src: RValue::get(v: tmp), dst: dest); |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | mlir::LogicalResult CIRGenFunction::emitAsmStmt(const AsmStmt &s) { |
| 264 | FullExprCleanupScope cleanups(*this, &s); |
| 265 | |
| 266 | // Assemble the final asm string. |
| 267 | std::string asmString = s.generateAsmString(C: getContext()); |
| 268 | SourceLocation srcLoc = s.getAsmLoc(); |
| 269 | mlir::Location loc = getLoc(srcLoc); |
| 270 | |
| 271 | // Get all the output and input constraints together. |
| 272 | SmallVector<TargetInfo::ConstraintInfo> outputConstraintInfos; |
| 273 | SmallVector<TargetInfo::ConstraintInfo> inputConstraintInfos; |
| 274 | collectInOutConstraintInfos(cgf: *this, s, out&: outputConstraintInfos, |
| 275 | in&: inputConstraintInfos); |
| 276 | |
| 277 | bool isGCCAsmGoto = false; |
| 278 | |
| 279 | std::string constraints; |
| 280 | SmallVector<LValue> resultRegDests; |
| 281 | SmallVector<QualType> resultRegQualTys; |
| 282 | SmallVector<mlir::Type> resultRegTypes; |
| 283 | SmallVector<mlir::Type> resultTruncRegTypes; |
| 284 | SmallVector<mlir::Type> argTypes; |
| 285 | SmallVector<mlir::Type> argElemTypes; |
| 286 | SmallVector<mlir::Value> args; |
| 287 | SmallVector<mlir::Value> outArgs; |
| 288 | SmallVector<mlir::Value> inArgs; |
| 289 | SmallVector<mlir::Value> inOutArgs; |
| 290 | llvm::BitVector resultTypeRequiresCast; |
| 291 | llvm::BitVector resultRegIsFlagReg; |
| 292 | |
| 293 | // Keep track of input constraints. |
| 294 | std::string inOutConstraints; |
| 295 | SmallVector<mlir::Type> inOutArgTypes; |
| 296 | SmallVector<mlir::Type> inOutArgElemTypes; |
| 297 | |
| 298 | // Keep track of out constraints for tied input operand. |
| 299 | SmallVector<std::string> outputConstraints; |
| 300 | |
| 301 | // Keep track of defined physregs. |
| 302 | llvm::SmallSet<std::string, 8> physRegOutputs; |
| 303 | |
| 304 | // An inline asm can be marked readonly if it meets the following conditions: |
| 305 | // - it doesn't have any sideeffects |
| 306 | // - it doesn't clobber memory |
| 307 | // - it doesn't return a value by-reference |
| 308 | // It can be marked readnone if it doesn't have any input memory constraints |
| 309 | // in addition to meeting the conditions listed above. |
| 310 | bool readOnly = true, readNone = true; |
| 311 | |
| 312 | std::string outputConstraint; |
| 313 | for (unsigned i = 0, e = s.getNumOutputs(); i != e; ++i) { |
| 314 | TargetInfo::ConstraintInfo &info = outputConstraintInfos[i]; |
| 315 | |
| 316 | // Simplify the output constraint. |
| 317 | outputConstraint = s.getOutputConstraint(i); |
| 318 | outputConstraint = getTarget().simplifyConstraint( |
| 319 | Constraint: StringRef(outputConstraint).drop_front()); |
| 320 | |
| 321 | const Expr *outExpr = s.getOutputExpr(i); |
| 322 | outExpr = outExpr->IgnoreParenNoopCasts(Ctx: getContext()); |
| 323 | |
| 324 | std::string gccReg; |
| 325 | outputConstraint = s.addVariableConstraints( |
| 326 | Constraint: outputConstraint, AsmExpr: *outExpr, Target: getTarget(), EarlyClobber: info.earlyClobber(), |
| 327 | UnsupportedCB: [&](const Stmt *unspStmt, StringRef msg) { |
| 328 | cgm.errorUnsupported(s: unspStmt, type: msg); |
| 329 | }, |
| 330 | GCCReg: &gccReg); |
| 331 | |
| 332 | // Give an error on multiple outputs to same physreg. |
| 333 | if (!gccReg.empty() && !physRegOutputs.insert(V: gccReg).second) |
| 334 | cgm.error(loc: srcLoc, error: "multiple outputs to hard register: " + gccReg); |
| 335 | |
| 336 | outputConstraints.push_back(Elt: outputConstraint); |
| 337 | LValue dest = emitLValue(e: outExpr); |
| 338 | |
| 339 | if (!constraints.empty()) |
| 340 | constraints += ','; |
| 341 | |
| 342 | // If this is a register output, then make the inline a sm return it |
| 343 | // by-value. If this is a memory result, return the value by-reference. |
| 344 | QualType qty = outExpr->getType(); |
| 345 | const bool isScalarOrAggregate = |
| 346 | hasScalarEvaluationKind(type: qty) || hasAggregateEvaluationKind(type: qty); |
| 347 | if (!info.allowsMemory() && isScalarOrAggregate) { |
| 348 | constraints += "=" + outputConstraint; |
| 349 | resultRegQualTys.push_back(Elt: qty); |
| 350 | resultRegDests.push_back(Elt: dest); |
| 351 | |
| 352 | bool isFlagReg = llvm::StringRef(outputConstraint).starts_with(Prefix: "{@cc" ); |
| 353 | resultRegIsFlagReg.push_back(Val: isFlagReg); |
| 354 | |
| 355 | mlir::Type ty = convertTypeForMem(t: qty); |
| 356 | const bool requiresCast = |
| 357 | info.allowsRegister() && |
| 358 | (cgm.getTargetCIRGenInfo().isScalarizableAsmOperand(cgf&: *this, ty) || |
| 359 | isa<cir::RecordType, cir::ArrayType>(Val: ty)); |
| 360 | |
| 361 | resultTruncRegTypes.push_back(Elt: ty); |
| 362 | resultTypeRequiresCast.push_back(Val: requiresCast); |
| 363 | |
| 364 | if (requiresCast) { |
| 365 | unsigned size = getContext().getTypeSize(T: qty); |
| 366 | if (size == 0) |
| 367 | cgm.error(loc: outExpr->getExprLoc(), error: "output size should not be zero" ); |
| 368 | |
| 369 | ty = cir::IntType::get(context: &getMLIRContext(), width: size, is_signed: false); |
| 370 | } |
| 371 | |
| 372 | resultRegTypes.push_back(Elt: ty); |
| 373 | // If this output is tied to an input, and if the input is larger, then |
| 374 | // we need to set the actual result type of the inline asm node to be the |
| 375 | // same as the input type. |
| 376 | if (info.hasMatchingInput()) { |
| 377 | unsigned inputNo; |
| 378 | for (inputNo = 0; inputNo != s.getNumInputs(); ++inputNo) { |
| 379 | TargetInfo::ConstraintInfo &input = inputConstraintInfos[inputNo]; |
| 380 | if (input.hasTiedOperand() && input.getTiedOperand() == i) |
| 381 | break; |
| 382 | } |
| 383 | assert(inputNo != s.getNumInputs() && "Didn't find matching input!" ); |
| 384 | |
| 385 | QualType inputTy = s.getInputExpr(i: inputNo)->getType(); |
| 386 | QualType outputType = outExpr->getType(); |
| 387 | |
| 388 | uint64_t inputSize = getContext().getTypeSize(T: inputTy); |
| 389 | if (getContext().getTypeSize(T: outputType) < inputSize) { |
| 390 | // Form the asm to return the value as a larger integer or fp type. |
| 391 | resultRegTypes.back() = convertType(t: inputTy); |
| 392 | } |
| 393 | |
| 394 | if (mlir::Type adjTy = cgm.getTargetCIRGenInfo().adjustInlineAsmType( |
| 395 | cgf&: *this, constraint: outputConstraint, ty: resultRegTypes.back())) |
| 396 | resultRegTypes.back() = adjTy; |
| 397 | else |
| 398 | cgm.getDiags().Report(Loc: srcLoc, DiagID: diag::err_asm_invalid_type_in_input) |
| 399 | << outExpr->getType() << outputConstraint; |
| 400 | } |
| 401 | |
| 402 | // Update largest vector width for any vector types. |
| 403 | assert(!cir::MissingFeatures::asmVectorType()); |
| 404 | } else { |
| 405 | Address destAddr = dest.getAddress(); |
| 406 | |
| 407 | // Matrix types in memory are represented by arrays, but accessed through |
| 408 | // vector pointers, with the alignment specified on the access operation. |
| 409 | // For inline assembly, update pointer arguments to use vector pointers. |
| 410 | // Otherwise there will be a mis-match if the matrix is also an |
| 411 | // input-argument which is represented as vector. |
| 412 | if (isa<MatrixType>(Val: outExpr->getType().getCanonicalType())) |
| 413 | destAddr = |
| 414 | destAddr.withElementType(builder, ElemTy: convertType(t: outExpr->getType())); |
| 415 | |
| 416 | argTypes.push_back(Elt: destAddr.getType()); |
| 417 | argElemTypes.push_back(Elt: destAddr.getElementType()); |
| 418 | outArgs.push_back(Elt: destAddr.getPointer()); |
| 419 | args.push_back(Elt: destAddr.getPointer()); |
| 420 | constraints += "=*" ; |
| 421 | constraints += outputConstraint; |
| 422 | readOnly = readNone = false; |
| 423 | } |
| 424 | |
| 425 | if (info.isReadWrite()) { |
| 426 | inOutConstraints += ','; |
| 427 | const Expr *inputExpr = s.getOutputExpr(i); |
| 428 | |
| 429 | // argValue: mlir::Value, argElementType: mlir::Type. |
| 430 | auto [argValue, argElementType] = |
| 431 | emitAsmInputLValue(info, inputValue: dest, inputType: inputExpr->getType(), constraintString&: inOutConstraints, |
| 432 | loc: inputExpr->getExprLoc()); |
| 433 | |
| 434 | if (mlir::Type adjTy = getTargetHooks().adjustInlineAsmType( |
| 435 | cgf&: *this, constraint: outputConstraint, ty: argValue.getType())) |
| 436 | argValue = builder.createBitcast(src: argValue, newTy: adjTy); |
| 437 | |
| 438 | // Update largest vector width for any vector types. |
| 439 | assert(!cir::MissingFeatures::asmVectorType()); |
| 440 | |
| 441 | // Only tie earlyclobber physregs. |
| 442 | if (info.allowsRegister() && (gccReg.empty() || info.earlyClobber())) |
| 443 | inOutConstraints += llvm::utostr(X: i); |
| 444 | else |
| 445 | inOutConstraints += outputConstraint; |
| 446 | |
| 447 | inOutArgTypes.push_back(Elt: argValue.getType()); |
| 448 | inOutArgElemTypes.push_back(Elt: argElementType); |
| 449 | inOutArgs.push_back(Elt: argValue); |
| 450 | } |
| 451 | |
| 452 | } // iterate over output operands |
| 453 | |
| 454 | for (unsigned i = 0, e = s.getNumInputs(); i != e; ++i) { |
| 455 | TargetInfo::ConstraintInfo &info = inputConstraintInfos[i]; |
| 456 | const Expr *inputExpr = s.getInputExpr(i); |
| 457 | |
| 458 | if (info.allowsMemory()) |
| 459 | readNone = false; |
| 460 | |
| 461 | if (!constraints.empty()) |
| 462 | constraints += ','; |
| 463 | |
| 464 | std::string inputConstraint(s.getInputConstraint(i)); |
| 465 | inputConstraint = |
| 466 | getTarget().simplifyConstraint(Constraint: inputConstraint, OutCons: &outputConstraintInfos); |
| 467 | |
| 468 | inputConstraint = s.addVariableConstraints( |
| 469 | Constraint: inputConstraint, AsmExpr: *inputExpr->IgnoreParenNoopCasts(Ctx: getContext()), |
| 470 | Target: getTarget(), /*EarlyClobber=*/false, |
| 471 | UnsupportedCB: [&](const Stmt *unspStmt, StringRef msg) { |
| 472 | cgm.errorUnsupported(s: unspStmt, type: msg); |
| 473 | }); |
| 474 | |
| 475 | std::string replaceConstraint(inputConstraint); |
| 476 | // argValue: mlir::Value, argElementType: mlir::Type. |
| 477 | auto [argValue, argElemType] = emitAsmInput(info, inputExpr, constraintString&: constraints); |
| 478 | |
| 479 | // If this input argument is tied to a larger output result, extend the |
| 480 | // input to be the same size as the output. The LLVM backend wants to see |
| 481 | // the input and output of a matching constraint be the same size. Note |
| 482 | // that GCC does not define what the top bits are here. We use zext because |
| 483 | // that is usually cheaper, but LLVM IR should really get an anyext someday. |
| 484 | if (info.hasTiedOperand()) { |
| 485 | unsigned output = info.getTiedOperand(); |
| 486 | QualType outputType = s.getOutputExpr(i: output)->getType(); |
| 487 | QualType inputTy = inputExpr->getType(); |
| 488 | |
| 489 | if (getContext().getTypeSize(T: outputType) > |
| 490 | getContext().getTypeSize(T: inputTy)) { |
| 491 | // Use ptrtoint as appropriate so that we can do our extension. |
| 492 | if (isa<cir::PointerType>(Val: argValue.getType())) |
| 493 | argValue = builder.createPtrToInt(src: argValue, newTy: uIntPtrTy); |
| 494 | mlir::Type outputTy = convertType(t: outputType); |
| 495 | if (isa<cir::IntType>(Val: outputTy)) |
| 496 | argValue = builder.createIntCast(src: argValue, newTy: outputTy); |
| 497 | else if (isa<cir::PointerType>(Val: outputTy)) |
| 498 | argValue = builder.createIntCast(src: argValue, newTy: uIntPtrTy); |
| 499 | else if (isa<cir::FPTypeInterface>(Val: outputTy)) |
| 500 | argValue = builder.createFloatingCast(v: argValue, destType: outputTy); |
| 501 | } |
| 502 | |
| 503 | // Deal with the tied operands' constraint code in adjustInlineAsmType. |
| 504 | replaceConstraint = outputConstraints[output]; |
| 505 | } |
| 506 | |
| 507 | if (mlir::Type adjTy = getTargetHooks().adjustInlineAsmType( |
| 508 | cgf&: *this, constraint: replaceConstraint, ty: argValue.getType())) |
| 509 | argValue = builder.createBitcast(src: argValue, newTy: adjTy); |
| 510 | else |
| 511 | cgm.getDiags().Report(Loc: s.getAsmLoc(), DiagID: diag::err_asm_invalid_type_in_input) |
| 512 | << inputExpr->getType() << inputConstraint; |
| 513 | |
| 514 | // Update largest vector width for any vector types. |
| 515 | assert(!cir::MissingFeatures::asmVectorType()); |
| 516 | |
| 517 | argTypes.push_back(Elt: argValue.getType()); |
| 518 | argElemTypes.push_back(Elt: argElemType); |
| 519 | inArgs.push_back(Elt: argValue); |
| 520 | args.push_back(Elt: argValue); |
| 521 | constraints += inputConstraint; |
| 522 | } // iterate over input operands |
| 523 | |
| 524 | // Append the "input" part of inout constraints. |
| 525 | for (unsigned i = 0, e = inOutArgs.size(); i != e; ++i) { |
| 526 | args.push_back(Elt: inOutArgs[i]); |
| 527 | argTypes.push_back(Elt: inOutArgTypes[i]); |
| 528 | argElemTypes.push_back(Elt: inOutArgElemTypes[i]); |
| 529 | } |
| 530 | constraints += inOutConstraints; |
| 531 | |
| 532 | bool hasUnwindClobber = false; |
| 533 | collectClobbers(cgf: *this, s, constraints, hasUnwindClobber, readOnly, readNone); |
| 534 | |
| 535 | std::array<mlir::ValueRange, 3> operands = {outArgs, inArgs, inOutArgs}; |
| 536 | |
| 537 | mlir::Type resultType; |
| 538 | |
| 539 | if (resultRegTypes.size() == 1) |
| 540 | resultType = resultRegTypes[0]; |
| 541 | else if (resultRegTypes.size() > 1) |
| 542 | resultType = builder.getAnonRecordTy( |
| 543 | members: resultRegTypes, /*packed=*/false, |
| 544 | memberKinds: cir::RecordType::getAllDataKinds(members: resultRegTypes)); |
| 545 | |
| 546 | bool hasSideEffect = s.isVolatile() || s.getNumOutputs() == 0; |
| 547 | // Asm may contain an intrinsically convergent op such as a barrier. |
| 548 | assert(!cir::MissingFeatures::asmNoConvergent()); |
| 549 | bool isConvergent = getLangOpts().assumeFunctionsAreConvergent(); |
| 550 | |
| 551 | std::vector<mlir::Value> regResults; |
| 552 | cir::InlineAsmOp ia = cir::InlineAsmOp::create( |
| 553 | builder, location: getLoc(srcLoc: s.getAsmLoc()), res: resultType, asm_operands: operands, asm_string: asmString, |
| 554 | constraints, side_effects: hasSideEffect, convergent: isConvergent, asm_flavor: inferFlavor(cgm, s), |
| 555 | operand_attrs: mlir::ArrayAttr()); |
| 556 | |
| 557 | if (isGCCAsmGoto) { |
| 558 | assert(!cir::MissingFeatures::asmGoto()); |
| 559 | } else if (hasUnwindClobber) { |
| 560 | assert(!cir::MissingFeatures::asmUnwindClobber()); |
| 561 | } else { |
| 562 | assert(!cir::MissingFeatures::asmMemoryEffects()); |
| 563 | |
| 564 | mlir::Value result; |
| 565 | if (ia.getNumResults()) |
| 566 | result = ia.getResult(i: 0); |
| 567 | |
| 568 | llvm::SmallVector<mlir::Attribute> operandAttrs; |
| 569 | |
| 570 | for (auto [idx, typ] : llvm::enumerate(First&: argElemTypes)) { |
| 571 | if (typ) { |
| 572 | [[maybe_unused]] mlir::Value op = args[idx]; |
| 573 | assert(mlir::isa<cir::PointerType>(op.getType()) && |
| 574 | "pointer type expected" ); |
| 575 | assert(cast<cir::PointerType>(op.getType()).getPointee() == typ && |
| 576 | "element type differs from pointee type!" ); |
| 577 | |
| 578 | operandAttrs.push_back(Elt: mlir::UnitAttr::get(context: &getMLIRContext())); |
| 579 | } else { |
| 580 | // We need to add an attribute for every arg since later, during |
| 581 | // the lowering to LLVM IR the attributes will be assigned to the |
| 582 | // CallInsn argument by index, i.e. we can't skip null type here |
| 583 | operandAttrs.push_back(Elt: mlir::DictionaryAttr::get(context: &getMLIRContext())); |
| 584 | } |
| 585 | } |
| 586 | assert(args.size() == operandAttrs.size() && |
| 587 | "The number of attributes is not even with the number of operands" ); |
| 588 | |
| 589 | ia.setOperandAttrsAttr(builder.getArrayAttr(value: operandAttrs)); |
| 590 | |
| 591 | if (resultRegTypes.size() == 1) { |
| 592 | regResults.push_back(x: result); |
| 593 | } else if (resultRegTypes.size() > 1) { |
| 594 | CharUnits alignment = CharUnits::One(); |
| 595 | mlir::Value dest = |
| 596 | emitAlloca(name: "__asm_result" , ty: resultType, loc, alignment, insertIntoFnEntryBlock: false); |
| 597 | Address addr = Address(dest, alignment); |
| 598 | builder.createStore(loc, val: result, dst: addr); |
| 599 | |
| 600 | for (unsigned i = 0, e = resultRegTypes.size(); i != e; ++i) { |
| 601 | cir::PointerType typ = builder.getPointerTo( |
| 602 | ty: resultRegTypes[i], |
| 603 | as: mlir::cast<cir::PointerType>(Val: dest.getType()).getAddrSpace()); |
| 604 | cir::GetMemberOp ptr = builder.createGetMember(loc, resultTy: typ, base: dest, name: "" , index: i); |
| 605 | cir::LoadOp tmp = builder.createLoad(loc, addr: Address(ptr, alignment)); |
| 606 | regResults.push_back(x: tmp); |
| 607 | } |
| 608 | } |
| 609 | } |
| 610 | |
| 611 | emitAsmStores(cgf&: *this, s, regResults, resultRegTypes, resultTruncRegTypes, |
| 612 | resultRegDests, resultRegQualTys, resultTypeRequiresCast, |
| 613 | resultRegIsFlagReg); |
| 614 | |
| 615 | return mlir::success(); |
| 616 | } |
| 617 | |