| 1 | //===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // \file |
| 10 | // This file implements a TargetTransformInfo analysis pass specific to the |
| 11 | // AMDGPU target machine. It uses the target's detailed information to provide |
| 12 | // more precise answers to certain TTI queries, while letting the target |
| 13 | // independent and default TTI implementations handle the rest. |
| 14 | // |
| 15 | //===----------------------------------------------------------------------===// |
| 16 | |
| 17 | #include "AMDGPUTargetTransformInfo.h" |
| 18 | #include "AMDGPUSubtarget.h" |
| 19 | #include "AMDGPUTargetMachine.h" |
| 20 | #include "MCTargetDesc/AMDGPUMCTargetDesc.h" |
| 21 | #include "SIModeRegisterDefaults.h" |
| 22 | #include "llvm/ADT/SmallBitVector.h" |
| 23 | #include "llvm/Analysis/InlineCost.h" |
| 24 | #include "llvm/Analysis/LoopInfo.h" |
| 25 | #include "llvm/Analysis/ValueTracking.h" |
| 26 | #include "llvm/CodeGen/Analysis.h" |
| 27 | #include "llvm/IR/Function.h" |
| 28 | #include "llvm/IR/IRBuilder.h" |
| 29 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
| 30 | #include "llvm/IR/PatternMatch.h" |
| 31 | #include "llvm/Support/KnownBits.h" |
| 32 | #include <optional> |
| 33 | |
| 34 | using namespace llvm; |
| 35 | |
| 36 | #define DEBUG_TYPE "AMDGPUtti" |
| 37 | |
| 38 | static bool dependsOnLocalPhi(const Loop *L, const Value *Cond, |
| 39 | unsigned Depth = 0) { |
| 40 | const Instruction *I = dyn_cast<Instruction>(Val: Cond); |
| 41 | if (!I) |
| 42 | return false; |
| 43 | |
| 44 | if (!L->contains(Inst: I)) |
| 45 | return false; |
| 46 | for (const Value *V : I->operand_values()) { |
| 47 | if (const PHINode *PHI = dyn_cast<PHINode>(Val: V)) { |
| 48 | if (llvm::none_of(Range: L->getSubLoops(), P: [PHI](const Loop* SubLoop) { |
| 49 | return SubLoop->contains(Inst: PHI); })) |
| 50 | return true; |
| 51 | } else if (Depth < 10 && dependsOnLocalPhi(L, Cond: V, Depth: Depth+1)) |
| 52 | return true; |
| 53 | } |
| 54 | return false; |
| 55 | } |
| 56 | |
| 57 | AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F) |
| 58 | : BaseT(TM, F.getDataLayout()), CLOpts(TM->getCLOpts()), |
| 59 | TargetTriple(F.getParent()->getTargetTriple()), |
| 60 | ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))), |
| 61 | TLI(ST->getTargetLowering()) {} |
| 62 | |
| 63 | void AMDGPUTTIImpl::( |
| 64 | Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, |
| 65 | OptimizationRemarkEmitter *ORE) const { |
| 66 | const Function &F = *L->getHeader()->getParent(); |
| 67 | UP.Threshold = static_cast<unsigned>( |
| 68 | F.getFnAttributeAsParsedInteger(Kind: "amdgpu-unroll-threshold" , Default: 300)); |
| 69 | UP.PartialThreshold = static_cast<unsigned>( |
| 70 | F.getFnAttributeAsParsedInteger(Kind: "amdgpu-partial-unroll-threshold" , Default: 150)); |
| 71 | UP.MaxCount = std::numeric_limits<unsigned>::max(); |
| 72 | UP.Partial = true; |
| 73 | |
| 74 | // Conditional branch in a loop back edge needs 3 additional exec |
| 75 | // manipulations in average. |
| 76 | UP.BEInsns += 3; |
| 77 | |
| 78 | // We want to run unroll even for the loops which have been vectorized. |
| 79 | UP.UnrollVectorizedLoop = true; |
| 80 | |
| 81 | // Enable runtime unrolling for loops whose trip count is not known at |
| 82 | // compile time. |
| 83 | UP.Runtime = true; |
| 84 | |
| 85 | // Maximum alloca size than can fit registers. Reserve 16 registers. |
| 86 | const unsigned MaxAlloca = (256 - 16) * 4; |
| 87 | unsigned ThresholdPrivate = CLOpts.unroll_threshold_private; |
| 88 | unsigned ThresholdLocal = CLOpts.unroll_threshold_local; |
| 89 | |
| 90 | // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the |
| 91 | // provided threshold value as the default for Threshold |
| 92 | if (MDNode *LoopUnrollThreshold = |
| 93 | findOptionMDForLoop(TheLoop: L, Name: "amdgpu.loop.unroll.threshold" )) { |
| 94 | if (LoopUnrollThreshold->getNumOperands() == 2) { |
| 95 | ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>( |
| 96 | MD: LoopUnrollThreshold->getOperand(I: 1)); |
| 97 | if (MetaThresholdValue) { |
| 98 | // We will also use the supplied value for PartialThreshold for now. |
| 99 | // We may introduce additional metadata if it becomes necessary in the |
| 100 | // future. |
| 101 | UP.Threshold = |
| 102 | static_cast<unsigned>(MetaThresholdValue->getSExtValue()); |
| 103 | UP.PartialThreshold = UP.Threshold; |
| 104 | ThresholdPrivate = std::min(a: ThresholdPrivate, b: UP.Threshold); |
| 105 | ThresholdLocal = std::min(a: ThresholdLocal, b: UP.Threshold); |
| 106 | } |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | unsigned MaxBoost = std::max(a: ThresholdPrivate, b: ThresholdLocal); |
| 111 | for (const BasicBlock *BB : L->getBlocks()) { |
| 112 | const DataLayout &DL = BB->getDataLayout(); |
| 113 | unsigned LocalGEPsSeen = 0; |
| 114 | |
| 115 | if (llvm::any_of(Range: L->getSubLoops(), P: [BB](const Loop* SubLoop) { |
| 116 | return SubLoop->contains(BB); })) |
| 117 | continue; // Block belongs to an inner loop. |
| 118 | |
| 119 | for (const Instruction &I : *BB) { |
| 120 | // Unroll a loop which contains an "if" statement whose condition |
| 121 | // defined by a PHI belonging to the loop. This may help to eliminate |
| 122 | // if region and potentially even PHI itself, saving on both divergence |
| 123 | // and registers used for the PHI. |
| 124 | // Add a small bonus for each of such "if" statements. |
| 125 | if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: &I)) { |
| 126 | if (UP.Threshold < MaxBoost) { |
| 127 | BasicBlock *Succ0 = Br->getSuccessor(i: 0); |
| 128 | BasicBlock *Succ1 = Br->getSuccessor(i: 1); |
| 129 | if ((L->contains(BB: Succ0) && L->isLoopExiting(BB: Succ0)) || |
| 130 | (L->contains(BB: Succ1) && L->isLoopExiting(BB: Succ1))) |
| 131 | continue; |
| 132 | if (dependsOnLocalPhi(L, Cond: Br->getCondition())) { |
| 133 | UP.Threshold += CLOpts.unroll_threshold_if; |
| 134 | LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold |
| 135 | << " for loop:\n" |
| 136 | << *L << " due to " << *Br << '\n'); |
| 137 | if (UP.Threshold >= MaxBoost) |
| 138 | return; |
| 139 | } |
| 140 | } |
| 141 | continue; |
| 142 | } |
| 143 | |
| 144 | const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &I); |
| 145 | if (!GEP) |
| 146 | continue; |
| 147 | |
| 148 | unsigned AS = GEP->getAddressSpace(); |
| 149 | unsigned Threshold = 0; |
| 150 | if (AS == AMDGPUAS::PRIVATE_ADDRESS) |
| 151 | Threshold = ThresholdPrivate; |
| 152 | else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) |
| 153 | Threshold = ThresholdLocal; |
| 154 | else |
| 155 | continue; |
| 156 | |
| 157 | if (UP.Threshold >= Threshold) |
| 158 | continue; |
| 159 | |
| 160 | if (AS == AMDGPUAS::PRIVATE_ADDRESS) { |
| 161 | const Value *Ptr = GEP->getPointerOperand(); |
| 162 | const AllocaInst *Alloca = |
| 163 | dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Ptr)); |
| 164 | if (!Alloca || !Alloca->isStaticAlloca()) |
| 165 | continue; |
| 166 | auto AllocaSize = Alloca->getAllocationSize(DL); |
| 167 | if (!AllocaSize || AllocaSize->getFixedValue() > MaxAlloca) |
| 168 | continue; |
| 169 | } else if (AS == AMDGPUAS::LOCAL_ADDRESS || |
| 170 | AS == AMDGPUAS::REGION_ADDRESS) { |
| 171 | LocalGEPsSeen++; |
| 172 | // Inhibit unroll for local memory if we have seen addressing not to |
| 173 | // a variable, most likely we will be unable to combine it. |
| 174 | // Do not unroll too deep inner loops for local memory to give a chance |
| 175 | // to unroll an outer loop for a more important reason. |
| 176 | if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 || |
| 177 | (!isa<GlobalVariable>(Val: GEP->getPointerOperand()) && |
| 178 | !isa<Argument>(Val: GEP->getPointerOperand()))) |
| 179 | continue; |
| 180 | LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n" |
| 181 | << *L << " due to LDS use.\n" ); |
| 182 | UP.Runtime = CLOpts.unroll_runtime_local; |
| 183 | } |
| 184 | |
| 185 | // Check if GEP depends on a value defined by this loop itself. |
| 186 | bool HasLoopDef = false; |
| 187 | for (const Value *Op : GEP->operands()) { |
| 188 | const Instruction *Inst = dyn_cast<Instruction>(Val: Op); |
| 189 | if (!Inst || L->isLoopInvariant(V: Op)) |
| 190 | continue; |
| 191 | |
| 192 | if (llvm::any_of(Range: L->getSubLoops(), P: [Inst](const Loop* SubLoop) { |
| 193 | return SubLoop->contains(Inst); })) |
| 194 | continue; |
| 195 | HasLoopDef = true; |
| 196 | break; |
| 197 | } |
| 198 | if (!HasLoopDef) |
| 199 | continue; |
| 200 | |
| 201 | // We want to do whatever we can to limit the number of alloca |
| 202 | // instructions that make it through to the code generator. allocas |
| 203 | // require us to use indirect addressing, which is slow and prone to |
| 204 | // compiler bugs. If this loop does an address calculation on an |
| 205 | // alloca ptr, then we want to use a higher than normal loop unroll |
| 206 | // threshold. This will give SROA a better chance to eliminate these |
| 207 | // allocas. |
| 208 | // |
| 209 | // We also want to have more unrolling for local memory to let ds |
| 210 | // instructions with different offsets combine. |
| 211 | // |
| 212 | // Don't use the maximum allowed value here as it will make some |
| 213 | // programs way too big. |
| 214 | UP.Threshold = Threshold; |
| 215 | LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold |
| 216 | << " for loop:\n" |
| 217 | << *L << " due to " << *GEP << '\n'); |
| 218 | if (UP.Threshold >= MaxBoost) |
| 219 | return; |
| 220 | } |
| 221 | |
| 222 | // If we got a GEP in a small BB from inner loop then increase max trip |
| 223 | // count to analyze for better estimation cost in unroll |
| 224 | if (L->isInnermost() && BB->size() < CLOpts.unroll_max_block_to_analyze) |
| 225 | UP.MaxIterationsCountToAnalyze = 32; |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, |
| 230 | TTI::PeelingPreferences &PP) const { |
| 231 | BaseT::getPeelingPreferences(L, SE, PP); |
| 232 | } |
| 233 | |
| 234 | uint64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const { |
| 235 | return 1024; |
| 236 | } |
| 237 | |
| 238 | GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F) |
| 239 | : BaseT(TM, F.getDataLayout()), CLOpts(TM->getCLOpts()), |
| 240 | ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))), |
| 241 | TLI(ST->getTargetLowering()), CommonTTI(TM, F), |
| 242 | IsGraphics(AMDGPU::isGraphics(CC: F.getCallingConv())) { |
| 243 | SIModeRegisterDefaults Mode(F, *ST); |
| 244 | HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign(); |
| 245 | } |
| 246 | |
| 247 | bool GCNTTIImpl::hasBranchDivergence(const Function *F) const { |
| 248 | return !F || !ST->isSingleLaneExecution(Kernel: *F); |
| 249 | } |
| 250 | |
| 251 | unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const { |
| 252 | // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector |
| 253 | // registers. See getRegisterClassForType for the implementation. |
| 254 | // In this case vector registers are not vector in terms of |
| 255 | // VGPRs, but those which can hold multiple values. |
| 256 | |
| 257 | // This is really the number of registers to fill when vectorizing / |
| 258 | // interleaving loops, so we lie to avoid trying to use all registers. |
| 259 | return 4; |
| 260 | } |
| 261 | |
| 262 | TypeSize |
| 263 | GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const { |
| 264 | switch (K) { |
| 265 | case TargetTransformInfo::RGK_Scalar: |
| 266 | return TypeSize::getFixed(ExactSize: 32); |
| 267 | case TargetTransformInfo::RGK_FixedWidthVector: |
| 268 | return TypeSize::getFixed( |
| 269 | ExactSize: (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()) ? 128 |
| 270 | : ST->hasAnyPackedFP32Ops() ? 64 |
| 271 | : 32); |
| 272 | case TargetTransformInfo::RGK_ScalableVector: |
| 273 | return TypeSize::getScalable(MinimumSize: 0); |
| 274 | } |
| 275 | llvm_unreachable("Unsupported register kind" ); |
| 276 | } |
| 277 | |
| 278 | unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const { |
| 279 | return 32; |
| 280 | } |
| 281 | |
| 282 | unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const { |
| 283 | if (Opcode == Instruction::Load || Opcode == Instruction::Store) |
| 284 | return 32 * 4 / ElemWidth; |
| 285 | // For a given width return the max 0number of elements that can be combined |
| 286 | // into a wider bit value: |
| 287 | return (ElemWidth == 8 && ST->has16BitInsts()) ? 4 |
| 288 | : (ElemWidth == 16 && ST->has16BitInsts()) ? 2 |
| 289 | : (ElemWidth == 32 && ST->hasAnyPackedFP32Ops()) ? 2 |
| 290 | : (ElemWidth == 64 && |
| 291 | (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops())) |
| 292 | ? 2 |
| 293 | : 1; |
| 294 | } |
| 295 | |
| 296 | bool GCNTTIImpl::preferSLPInstCountCheck() const { |
| 297 | // The integer inst-count heuristic causes regressions on gfx94x and gfx950 |
| 298 | // because 2-element vector trees that pass the scalar/vector instruction |
| 299 | // count comparison still widen scalar moves (e.g. v_mov_b32 to v_mov_b64) |
| 300 | // after codegen, increasing register pressure and throughput cost without |
| 301 | // reducing the total instruction count. |
| 302 | return !ST->hasGFX940Insts() && !ST->hasGFX950Insts(); |
| 303 | } |
| 304 | |
| 305 | unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize, |
| 306 | unsigned ChainSizeInBytes, |
| 307 | VectorType *VecTy) const { |
| 308 | unsigned VecRegBitWidth = VF * LoadSize; |
| 309 | if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32) |
| 310 | // TODO: Support element-size less than 32bit? |
| 311 | return 128 / LoadSize; |
| 312 | |
| 313 | return VF; |
| 314 | } |
| 315 | |
| 316 | unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize, |
| 317 | unsigned ChainSizeInBytes, |
| 318 | VectorType *VecTy) const { |
| 319 | unsigned VecRegBitWidth = VF * StoreSize; |
| 320 | if (VecRegBitWidth > 128) |
| 321 | return 128 / StoreSize; |
| 322 | |
| 323 | return VF; |
| 324 | } |
| 325 | |
| 326 | unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const { |
| 327 | if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS || |
| 328 | AddrSpace == AMDGPUAS::CONSTANT_ADDRESS || |
| 329 | AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT || |
| 330 | AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER || |
| 331 | AddrSpace == AMDGPUAS::BUFFER_RESOURCE || |
| 332 | AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) { |
| 333 | return 512; |
| 334 | } |
| 335 | |
| 336 | if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) |
| 337 | return 8 * ST->getMaxPrivateElementSize(); |
| 338 | |
| 339 | // Common to flat, global, local and region. Assume for unknown addrspace. |
| 340 | return 128; |
| 341 | } |
| 342 | |
| 343 | bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, |
| 344 | Align Alignment, |
| 345 | unsigned AddrSpace) const { |
| 346 | // We allow vectorization of flat stores, even though we may need to decompose |
| 347 | // them later if they may access private memory. We don't have enough context |
| 348 | // here, and legalization can handle it. |
| 349 | if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) { |
| 350 | return (Alignment >= 4 || ST->hasUnalignedScratchAccessEnabled()) && |
| 351 | ChainSizeInBytes <= ST->getMaxPrivateElementSize(); |
| 352 | } |
| 353 | return true; |
| 354 | } |
| 355 | |
| 356 | bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, |
| 357 | Align Alignment, |
| 358 | unsigned AddrSpace) const { |
| 359 | return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace); |
| 360 | } |
| 361 | |
| 362 | bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, |
| 363 | Align Alignment, |
| 364 | unsigned AddrSpace) const { |
| 365 | return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace); |
| 366 | } |
| 367 | |
| 368 | uint64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const { |
| 369 | return 1024; |
| 370 | } |
| 371 | |
| 372 | Type *GCNTTIImpl::getMemcpyLoopLoweringType( |
| 373 | LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, |
| 374 | unsigned DestAddrSpace, Align SrcAlign, Align DestAlign, |
| 375 | std::optional<uint32_t> AtomicElementSize) const { |
| 376 | |
| 377 | if (AtomicElementSize) |
| 378 | return Type::getIntNTy(C&: Context, N: *AtomicElementSize * 8); |
| 379 | |
| 380 | // 16-byte accesses achieve the highest copy throughput. |
| 381 | // If the operation has a fixed known length that is large enough, it is |
| 382 | // worthwhile to return an even wider type and let legalization lower it into |
| 383 | // multiple accesses, effectively unrolling the memcpy loop. |
| 384 | // We also rely on legalization to decompose into smaller accesses for |
| 385 | // subtargets and address spaces where it is necessary. |
| 386 | // |
| 387 | // Don't unroll if Length is not a constant, since unrolling leads to worse |
| 388 | // performance for length values that are smaller or slightly larger than the |
| 389 | // total size of the type returned here. Mitigating that would require a more |
| 390 | // complex lowering for variable-length memcpy and memmove. |
| 391 | unsigned I32EltsInVector = 4; |
| 392 | if (CLOpts.memcpy_loop_unroll > 0 && isa<ConstantInt>(Val: Length)) |
| 393 | return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), |
| 394 | NumElts: CLOpts.memcpy_loop_unroll * I32EltsInVector); |
| 395 | |
| 396 | return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: I32EltsInVector); |
| 397 | } |
| 398 | |
| 399 | void GCNTTIImpl::getMemcpyLoopResidualLoweringType( |
| 400 | SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, |
| 401 | unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, |
| 402 | Align SrcAlign, Align DestAlign, |
| 403 | std::optional<uint32_t> AtomicCpySize) const { |
| 404 | |
| 405 | if (AtomicCpySize) |
| 406 | BaseT::getMemcpyLoopResidualLoweringType( |
| 407 | OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign, |
| 408 | DestAlign, AtomicCpySize); |
| 409 | |
| 410 | Type *I32x4Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: 4); |
| 411 | while (RemainingBytes >= 16) { |
| 412 | OpsOut.push_back(Elt: I32x4Ty); |
| 413 | RemainingBytes -= 16; |
| 414 | } |
| 415 | |
| 416 | Type *I64Ty = Type::getInt64Ty(C&: Context); |
| 417 | while (RemainingBytes >= 8) { |
| 418 | OpsOut.push_back(Elt: I64Ty); |
| 419 | RemainingBytes -= 8; |
| 420 | } |
| 421 | |
| 422 | Type *I32Ty = Type::getInt32Ty(C&: Context); |
| 423 | while (RemainingBytes >= 4) { |
| 424 | OpsOut.push_back(Elt: I32Ty); |
| 425 | RemainingBytes -= 4; |
| 426 | } |
| 427 | |
| 428 | Type *I16Ty = Type::getInt16Ty(C&: Context); |
| 429 | while (RemainingBytes >= 2) { |
| 430 | OpsOut.push_back(Elt: I16Ty); |
| 431 | RemainingBytes -= 2; |
| 432 | } |
| 433 | |
| 434 | Type *I8Ty = Type::getInt8Ty(C&: Context); |
| 435 | while (RemainingBytes) { |
| 436 | OpsOut.push_back(Elt: I8Ty); |
| 437 | --RemainingBytes; |
| 438 | } |
| 439 | } |
| 440 | |
| 441 | unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF, |
| 442 | bool HasUnorderedReductions) const { |
| 443 | // Disable unrolling if the loop is not vectorized. |
| 444 | // TODO: Enable this again. |
| 445 | if (VF.isScalar()) |
| 446 | return 1; |
| 447 | |
| 448 | return 8; |
| 449 | } |
| 450 | |
| 451 | bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, |
| 452 | MemIntrinsicInfo &Info) const { |
| 453 | switch (Inst->getIntrinsicID()) { |
| 454 | case Intrinsic::amdgcn_ds_ordered_add: |
| 455 | case Intrinsic::amdgcn_ds_ordered_swap: { |
| 456 | auto *Ordering = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 2)); |
| 457 | auto *Volatile = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 4)); |
| 458 | if (!Ordering || !Volatile) |
| 459 | return false; // Invalid. |
| 460 | |
| 461 | uint64_t OrderingVal = Ordering->getZExtValue(); |
| 462 | if (!isValidAtomicOrdering(I: OrderingVal)) |
| 463 | return false; |
| 464 | |
| 465 | Info.PtrVal = Inst->getArgOperand(i: 0); |
| 466 | Info.Ordering = static_cast<AtomicOrdering>(OrderingVal); |
| 467 | Info.ReadMem = true; |
| 468 | Info.WriteMem = true; |
| 469 | Info.IsVolatile = !Volatile->isZero(); |
| 470 | return true; |
| 471 | } |
| 472 | default: |
| 473 | return false; |
| 474 | } |
| 475 | } |
| 476 | |
| 477 | /// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are |
| 478 | /// expected to fuse during instruction selection. \p Ty is the type the fused |
| 479 | /// operation runs on. |
| 480 | static bool canFuseFMulWithFAddSub(const SITargetLowering &TLI, Type *Ty, |
| 481 | const Instruction *FMul, |
| 482 | const Instruction *FAddSub) { |
| 483 | assert((FAddSub->getOpcode() == Instruction::FAdd || |
| 484 | FAddSub->getOpcode() == Instruction::FSub) && |
| 485 | "Expected an fadd or an fsub" ); |
| 486 | |
| 487 | // The mad forms fuse exactly without fast-math flags but flush denormals. |
| 488 | // An fma forms only when it is not slower than the separate operations. |
| 489 | const Function &F = *FAddSub->getFunction(); |
| 490 | const bool HasFMAD = TLI.isFMADLegal(F, Ty); |
| 491 | const bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(F, Ty); |
| 492 | if (!HasFMAD && !HasFMA) |
| 493 | return false; |
| 494 | |
| 495 | // Without a mad the pair fuses only when both carry contract. |
| 496 | return HasFMAD || (FAddSub->hasAllowContract() && FMul->hasAllowContract()); |
| 497 | } |
| 498 | |
| 499 | /// An fma holds one multiply, so only one fmul operand fuses with \p FAddSub. |
| 500 | static const Instruction *getFusedFMul(const SITargetLowering &TLI, Type *Ty, |
| 501 | const Instruction *FAddSub) { |
| 502 | for (const Value *Op : FAddSub->operands()) { |
| 503 | const auto *FMul = dyn_cast<Instruction>(Val: Op); |
| 504 | if (FMul && FMul->getOpcode() == Instruction::FMul && FMul->hasOneUse() && |
| 505 | canFuseFMulWithFAddSub(TLI, Ty, FMul, FAddSub)) |
| 506 | return FMul; |
| 507 | } |
| 508 | return nullptr; |
| 509 | } |
| 510 | |
| 511 | static bool isFusedFMul(const SITargetLowering &TLI, Type *Ty, |
| 512 | const Instruction *FMul, const Instruction *FAddSub) { |
| 513 | const Instruction *Fused = getFusedFMul(TLI, Ty, FAddSub); |
| 514 | if (Fused == FMul) |
| 515 | return true; |
| 516 | // (a * b + c * d) + e becomes fma(a, b, fma(c, d, e)) if the outer fadd has |
| 517 | // reassoc. |
| 518 | if (!Fused || FAddSub->getOpcode() != Instruction::FAdd || |
| 519 | !FAddSub->hasOneUse()) |
| 520 | return false; |
| 521 | const auto *Outer = dyn_cast<BinaryOperator>(Val: *FAddSub->user_begin()); |
| 522 | return Outer && Outer->getOpcode() == Instruction::FAdd && |
| 523 | Outer->hasAllowReassoc() && |
| 524 | canFuseFMulWithFAddSub(TLI, Ty, FMul, FAddSub: Outer); |
| 525 | } |
| 526 | |
| 527 | InstructionCost GCNTTIImpl::getArithmeticInstrCost( |
| 528 | unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, |
| 529 | TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info, |
| 530 | ArrayRef<const Value *> Args, const Instruction *CtxI) const { |
| 531 | |
| 532 | // Legalize the type. |
| 533 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty); |
| 534 | int ISD = TLI->InstructionOpcodeToISD(Opcode); |
| 535 | |
| 536 | // Because we don't have any legal vector operations, but the legal types, we |
| 537 | // need to account for split vectors. |
| 538 | unsigned NElts = LT.second.isVector() ? |
| 539 | LT.second.getVectorNumElements() : 1; |
| 540 | |
| 541 | MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; |
| 542 | |
| 543 | switch (ISD) { |
| 544 | case ISD::SHL: |
| 545 | case ISD::SRL: |
| 546 | case ISD::SRA: |
| 547 | if (SLT == MVT::i64) |
| 548 | return get64BitInstrCost(CostKind) * LT.first * NElts; |
| 549 | |
| 550 | if (ST->has16BitInsts() && SLT == MVT::i16) |
| 551 | NElts = (NElts + 1) / 2; |
| 552 | |
| 553 | // i32 |
| 554 | return getFullRateInstrCost() * LT.first * NElts; |
| 555 | case ISD::ADD: |
| 556 | case ISD::SUB: |
| 557 | if (SLT == MVT::i64 && ST->hasAnyPackedU64Ops()) |
| 558 | NElts = (NElts + 1) / 2; |
| 559 | [[fallthrough]]; |
| 560 | case ISD::AND: |
| 561 | case ISD::OR: |
| 562 | case ISD::XOR: |
| 563 | if (SLT == MVT::i64) { |
| 564 | // and, or and xor are typically split into 2 VALU instructions. |
| 565 | return 2 * getFullRateInstrCost() * LT.first * NElts; |
| 566 | } |
| 567 | |
| 568 | if (ST->has16BitInsts() && SLT == MVT::i16) |
| 569 | NElts = (NElts + 1) / 2; |
| 570 | |
| 571 | return LT.first * NElts * getFullRateInstrCost(); |
| 572 | case ISD::MUL: { |
| 573 | const int QuarterRateCost = getQuarterRateInstrCost(CostKind); |
| 574 | if (SLT == MVT::i64) { |
| 575 | const int FullRateCost = getFullRateInstrCost(); |
| 576 | return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts; |
| 577 | } |
| 578 | |
| 579 | if (ST->has16BitInsts() && SLT == MVT::i16) |
| 580 | NElts = (NElts + 1) / 2; |
| 581 | |
| 582 | // i32 |
| 583 | return QuarterRateCost * NElts * LT.first; |
| 584 | } |
| 585 | case ISD::FMUL: |
| 586 | // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for |
| 587 | // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole |
| 588 | // fused operation. A vector fmul priced with a scalar context fuses with |
| 589 | // the scalar fadd|fsub only when it is emitted lane by lane. |
| 590 | if (CtxI && CtxI->hasOneUse() && |
| 591 | (CtxI->getType() == Ty || |
| 592 | getMaximumVF(ElemWidth: Ty->getScalarSizeInBits(), Opcode) == 1)) { |
| 593 | const auto *FAddSub = dyn_cast<BinaryOperator>(Val: *CtxI->user_begin()); |
| 594 | if (FAddSub && |
| 595 | (FAddSub->getOpcode() == Instruction::FAdd || |
| 596 | FAddSub->getOpcode() == Instruction::FSub) && |
| 597 | isFusedFMul(TLI: *TLI, Ty, FMul: CtxI, FAddSub)) |
| 598 | return TargetTransformInfo::TCC_Free; |
| 599 | } |
| 600 | [[fallthrough]]; |
| 601 | case ISD::FADD: |
| 602 | case ISD::FSUB: |
| 603 | if (ST->hasAnyPackedFP32Ops() && SLT == MVT::f32) |
| 604 | NElts = (NElts + 1) / 2; |
| 605 | if (ST->hasBF16PackedInsts() && SLT == MVT::bf16) |
| 606 | NElts = (NElts + 1) / 2; |
| 607 | if (SLT == MVT::f64) { |
| 608 | if (ST->hasAnyPackedFP64Ops()) |
| 609 | NElts = (NElts + 1) / 2; |
| 610 | return LT.first * NElts * get64BitInstrCost(CostKind); |
| 611 | } |
| 612 | |
| 613 | if (ST->has16BitInsts() && SLT == MVT::f16) |
| 614 | NElts = (NElts + 1) / 2; |
| 615 | |
| 616 | if (SLT == MVT::f32 || SLT == MVT::f16 || SLT == MVT::bf16) |
| 617 | return LT.first * NElts * getFullRateInstrCost(); |
| 618 | break; |
| 619 | case ISD::FDIV: |
| 620 | case ISD::FREM: |
| 621 | // FIXME: frem should be handled separately. The fdiv in it is most of it, |
| 622 | // but the current lowering is also not entirely correct. |
| 623 | if (SLT == MVT::f64) { |
| 624 | int Cost = 7 * get64BitInstrCost(CostKind) + |
| 625 | getQuarterRateInstrCost(CostKind) + |
| 626 | 3 * getHalfRateInstrCost(CostKind); |
| 627 | // Add cost of workaround. |
| 628 | if (!ST->hasUsableDivScaleConditionOutput()) |
| 629 | Cost += 3 * getFullRateInstrCost(); |
| 630 | |
| 631 | return LT.first * Cost * NElts; |
| 632 | } |
| 633 | |
| 634 | if (!Args.empty() && match(V: Args[0], P: PatternMatch::m_FPOne())) { |
| 635 | // TODO: This is more complicated, unsafe flags etc. |
| 636 | if ((SLT == MVT::f32 && !HasFP32Denormals) || |
| 637 | (SLT == MVT::f16 && ST->has16BitInsts())) { |
| 638 | return LT.first * getTransInstrCost(CostKind) * NElts; |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | if (SLT == MVT::f16 && ST->has16BitInsts()) { |
| 643 | // 2 x v_cvt_f32_f16 |
| 644 | // f32 rcp |
| 645 | // f32 fmul |
| 646 | // v_cvt_f16_f32 |
| 647 | // f16 div_fixup |
| 648 | int Cost = 4 * getFullRateInstrCost() + 2 * getTransInstrCost(CostKind); |
| 649 | return LT.first * Cost * NElts; |
| 650 | } |
| 651 | |
| 652 | if (SLT == MVT::f32 && (CtxI && CtxI->hasApproxFunc())) { |
| 653 | // Fast unsafe fdiv lowering: |
| 654 | // f32 rcp |
| 655 | // f32 fmul |
| 656 | int Cost = getTransInstrCost(CostKind) + getFullRateInstrCost(); |
| 657 | return LT.first * Cost * NElts; |
| 658 | } |
| 659 | |
| 660 | if (SLT == MVT::f32 || SLT == MVT::f16) { |
| 661 | // 4 more v_cvt_* insts without f16 insts support |
| 662 | int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() + |
| 663 | 1 * getTransInstrCost(CostKind); |
| 664 | |
| 665 | if (!HasFP32Denormals) { |
| 666 | // FP mode switches. |
| 667 | Cost += 2 * getFullRateInstrCost(); |
| 668 | } |
| 669 | |
| 670 | return LT.first * NElts * Cost; |
| 671 | } |
| 672 | break; |
| 673 | case ISD::FNEG: |
| 674 | // Use the backend' estimation. If fneg is not free each element will cost |
| 675 | // one additional instruction. |
| 676 | return TLI->isFNegFree(VT: SLT) ? 0 : NElts; |
| 677 | default: |
| 678 | break; |
| 679 | } |
| 680 | |
| 681 | return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info, |
| 682 | Args, CtxI); |
| 683 | } |
| 684 | |
| 685 | // Return true if there's a potential benefit from using v2f16/v2i16 |
| 686 | // instructions for an intrinsic, even if it requires nontrivial legalization. |
| 687 | static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) { |
| 688 | switch (ID) { |
| 689 | case Intrinsic::fma: |
| 690 | case Intrinsic::fmuladd: |
| 691 | case Intrinsic::copysign: |
| 692 | case Intrinsic::minimumnum: |
| 693 | case Intrinsic::maximumnum: |
| 694 | case Intrinsic::canonicalize: |
| 695 | // There's a small benefit to using vector ops in the legalized code. |
| 696 | case Intrinsic::round: |
| 697 | case Intrinsic::uadd_sat: |
| 698 | case Intrinsic::usub_sat: |
| 699 | case Intrinsic::sadd_sat: |
| 700 | case Intrinsic::ssub_sat: |
| 701 | case Intrinsic::abs: |
| 702 | return true; |
| 703 | default: |
| 704 | return false; |
| 705 | } |
| 706 | } |
| 707 | |
| 708 | InstructionCost |
| 709 | GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, |
| 710 | TTI::TargetCostKind CostKind) const { |
| 711 | switch (ICA.getID()) { |
| 712 | case Intrinsic::fabs: |
| 713 | // Free source modifier in the common case. |
| 714 | return 0; |
| 715 | case Intrinsic::amdgcn_workitem_id_x: |
| 716 | case Intrinsic::amdgcn_workitem_id_y: |
| 717 | case Intrinsic::amdgcn_workitem_id_z: |
| 718 | // TODO: If hasPackedTID, or if the calling context is not an entry point |
| 719 | // there may be a bit instruction. |
| 720 | return 0; |
| 721 | case Intrinsic::amdgcn_workgroup_id_x: |
| 722 | case Intrinsic::amdgcn_workgroup_id_y: |
| 723 | case Intrinsic::amdgcn_workgroup_id_z: |
| 724 | case Intrinsic::amdgcn_lds_kernel_id: |
| 725 | case Intrinsic::amdgcn_dispatch_ptr: |
| 726 | case Intrinsic::amdgcn_dispatch_id: |
| 727 | case Intrinsic::amdgcn_implicitarg_ptr: |
| 728 | case Intrinsic::amdgcn_queue_ptr: |
| 729 | // Read from an argument register. |
| 730 | return 0; |
| 731 | default: |
| 732 | break; |
| 733 | } |
| 734 | |
| 735 | Type *RetTy = ICA.getReturnType(); |
| 736 | |
| 737 | Intrinsic::ID IID = ICA.getID(); |
| 738 | switch (IID) { |
| 739 | case Intrinsic::exp: |
| 740 | case Intrinsic::exp2: |
| 741 | case Intrinsic::exp10: { |
| 742 | // Legalize the type. |
| 743 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy); |
| 744 | MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; |
| 745 | unsigned NElts = |
| 746 | LT.second.isVector() ? LT.second.getVectorNumElements() : 1; |
| 747 | |
| 748 | if (SLT == MVT::f64) { |
| 749 | unsigned NumOps = 20; |
| 750 | if (IID == Intrinsic::exp) |
| 751 | ++NumOps; |
| 752 | else if (IID == Intrinsic::exp10) |
| 753 | NumOps += 3; |
| 754 | |
| 755 | return LT.first * NElts * NumOps * get64BitInstrCost(CostKind); |
| 756 | } |
| 757 | |
| 758 | if (SLT == MVT::f32) { |
| 759 | unsigned NumFullRateOps = 0; |
| 760 | // v_exp_f32 (transcendental). |
| 761 | unsigned NumTransOps = 1; |
| 762 | |
| 763 | if (!ICA.getFlags().approxFunc() && IID != Intrinsic::exp2) { |
| 764 | // Non-AFN exp/exp10: range reduction + v_exp_f32 + ldexp + |
| 765 | // overflow/underflow checks (lowerFEXP). Denorm is also handled. |
| 766 | // FMA preamble: ~13 full-rate ops; non-FMA: ~17. |
| 767 | NumFullRateOps = ST->hasFastFMAF32() ? 13 : 17; |
| 768 | } else { |
| 769 | if (IID == Intrinsic::exp) { |
| 770 | // lowerFEXPUnsafe: fmul (base conversion) + v_exp_f32. |
| 771 | NumFullRateOps = 1; |
| 772 | } else if (IID == Intrinsic::exp10) { |
| 773 | // lowerFEXP10Unsafe: 3 fmul + 2 v_exp_f32 (double-exp2). |
| 774 | NumFullRateOps = 3; |
| 775 | NumTransOps = 2; |
| 776 | } |
| 777 | // Denorm scaling adds setcc + select + fadd + select + fmul. |
| 778 | if (HasFP32Denormals) |
| 779 | NumFullRateOps += 5; |
| 780 | } |
| 781 | |
| 782 | InstructionCost Cost = NumFullRateOps * getFullRateInstrCost() + |
| 783 | NumTransOps * getTransInstrCost(CostKind); |
| 784 | return LT.first * NElts * Cost; |
| 785 | } |
| 786 | |
| 787 | break; |
| 788 | } |
| 789 | case Intrinsic::log: |
| 790 | case Intrinsic::log2: |
| 791 | case Intrinsic::log10: { |
| 792 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy); |
| 793 | MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; |
| 794 | unsigned NElts = |
| 795 | LT.second.isVector() ? LT.second.getVectorNumElements() : 1; |
| 796 | |
| 797 | if (SLT == MVT::f32) { |
| 798 | unsigned NumFullRateOps = 0; |
| 799 | |
| 800 | if (IID == Intrinsic::log2) { |
| 801 | // LowerFLOG2: just v_log_f32. |
| 802 | } else if (ICA.getFlags().approxFunc()) { |
| 803 | // LowerFLOGUnsafe: v_log_f32 + fmul (base conversion). |
| 804 | NumFullRateOps = 1; |
| 805 | } else { |
| 806 | // LowerFLOGCommon non-AFN: v_log_f32 + extended-precision |
| 807 | // multiply + finite check. |
| 808 | NumFullRateOps = ST->hasFastFMAF32() ? 8 : 11; |
| 809 | } |
| 810 | |
| 811 | if (HasFP32Denormals) |
| 812 | NumFullRateOps += 5; |
| 813 | |
| 814 | InstructionCost Cost = |
| 815 | NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind); |
| 816 | return LT.first * NElts * Cost; |
| 817 | } |
| 818 | |
| 819 | break; |
| 820 | } |
| 821 | case Intrinsic::sin: |
| 822 | case Intrinsic::cos: { |
| 823 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy); |
| 824 | MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; |
| 825 | unsigned NElts = |
| 826 | LT.second.isVector() ? LT.second.getVectorNumElements() : 1; |
| 827 | |
| 828 | if (SLT == MVT::f32) { |
| 829 | // LowerTrig: fmul(1/2pi) + v_sin/v_cos. |
| 830 | unsigned NumFullRateOps = ST->hasTrigReducedRange() ? 2 : 1; |
| 831 | |
| 832 | InstructionCost Cost = |
| 833 | NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind); |
| 834 | return LT.first * NElts * Cost; |
| 835 | } |
| 836 | |
| 837 | break; |
| 838 | } |
| 839 | case Intrinsic::sqrt: { |
| 840 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy); |
| 841 | MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; |
| 842 | unsigned NElts = |
| 843 | LT.second.isVector() ? LT.second.getVectorNumElements() : 1; |
| 844 | |
| 845 | if (SLT == MVT::f32) { |
| 846 | unsigned NumFullRateOps = 0; |
| 847 | |
| 848 | if (!ICA.getFlags().approxFunc()) { |
| 849 | // lowerFSQRTF32 non-AFN: v_sqrt_f32 + refinement + scale fixup. |
| 850 | NumFullRateOps = HasFP32Denormals ? 17 : 16; |
| 851 | } |
| 852 | |
| 853 | InstructionCost Cost = |
| 854 | NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind); |
| 855 | return LT.first * NElts * Cost; |
| 856 | } |
| 857 | |
| 858 | break; |
| 859 | } |
| 860 | default: |
| 861 | break; |
| 862 | } |
| 863 | |
| 864 | if (!intrinsicHasPackedVectorBenefit(ID: ICA.getID())) |
| 865 | return BaseT::getIntrinsicInstrCost(ICA, CostKind); |
| 866 | |
| 867 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy); |
| 868 | MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; |
| 869 | unsigned NElts = LT.second.isVector() ? LT.second.getVectorNumElements() : 1; |
| 870 | |
| 871 | if ((ST->hasVOP3PInsts() && |
| 872 | (SLT == MVT::f16 || SLT == MVT::i16 || |
| 873 | (SLT == MVT::bf16 && ST->hasBF16PackedInsts()))) || |
| 874 | (ST->hasAnyPackedFP64Ops() && SLT == MVT::f64) || |
| 875 | (ST->hasAnyPackedU64Ops() && SLT == MVT::i64)) { |
| 876 | NElts = (NElts + 1) / 2; |
| 877 | } else if (SLT == MVT::f32) { |
| 878 | bool HasPk2FP32Op = ST->hasAnyPackedFP32Ops() && |
| 879 | IID != Intrinsic::minimumnum && |
| 880 | IID != Intrinsic::maximumnum; |
| 881 | NElts = HasPk2FP32Op ? (NElts + 1) / 2 : NElts; |
| 882 | } |
| 883 | |
| 884 | // TODO: Get more refined intrinsic costs? |
| 885 | unsigned InstRate = getQuarterRateInstrCost(CostKind); |
| 886 | |
| 887 | switch (ICA.getID()) { |
| 888 | case Intrinsic::fma: |
| 889 | case Intrinsic::fmuladd: |
| 890 | if (SLT == MVT::f64) { |
| 891 | InstRate = get64BitInstrCost(CostKind); |
| 892 | break; |
| 893 | } |
| 894 | |
| 895 | if ((SLT == MVT::f32 && ST->hasFastFMAF32()) || SLT == MVT::f16) |
| 896 | InstRate = getFullRateInstrCost(); |
| 897 | else { |
| 898 | InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind) |
| 899 | : getQuarterRateInstrCost(CostKind); |
| 900 | } |
| 901 | break; |
| 902 | case Intrinsic::copysign: |
| 903 | return NElts * getFullRateInstrCost(); |
| 904 | case Intrinsic::minimumnum: |
| 905 | case Intrinsic::maximumnum: { |
| 906 | // Instruction + 2 canonicalizes. For cases that need type promotion, we the |
| 907 | // promotion takes the place of the canonicalize. |
| 908 | unsigned NumOps = 3; |
| 909 | if (const IntrinsicInst *II = ICA.getInst()) { |
| 910 | // Directly legal with ieee=0 |
| 911 | // TODO: Not directly legal with strictfp |
| 912 | if (fpenvIEEEMode(I: *II) == KnownIEEEMode::Off) |
| 913 | NumOps = 1; |
| 914 | } |
| 915 | |
| 916 | unsigned BaseRate = |
| 917 | SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost(); |
| 918 | InstRate = BaseRate * NumOps; |
| 919 | break; |
| 920 | } |
| 921 | case Intrinsic::canonicalize: { |
| 922 | InstRate = |
| 923 | SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost(); |
| 924 | break; |
| 925 | } |
| 926 | case Intrinsic::uadd_sat: |
| 927 | case Intrinsic::usub_sat: |
| 928 | case Intrinsic::sadd_sat: |
| 929 | case Intrinsic::ssub_sat: { |
| 930 | if (SLT == MVT::i16 || SLT == MVT::i32) |
| 931 | InstRate = getFullRateInstrCost(); |
| 932 | |
| 933 | static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16}; |
| 934 | if (any_of(Range: ValidSatTys, P: equal_to(Arg&: LT.second))) |
| 935 | NElts = 1; |
| 936 | break; |
| 937 | } |
| 938 | case Intrinsic::abs: |
| 939 | // Expansion takes 2 instructions for VALU |
| 940 | if (SLT == MVT::i16 || SLT == MVT::i32) |
| 941 | InstRate = 2 * getFullRateInstrCost(); |
| 942 | break; |
| 943 | default: |
| 944 | break; |
| 945 | } |
| 946 | |
| 947 | return LT.first * NElts * InstRate; |
| 948 | } |
| 949 | |
| 950 | InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode, |
| 951 | TTI::TargetCostKind CostKind, |
| 952 | const Instruction *I) const { |
| 953 | assert((I == nullptr || I->getOpcode() == Opcode) && |
| 954 | "Opcode should reflect passed instruction." ); |
| 955 | const bool SCost = |
| 956 | (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency); |
| 957 | const int CBrCost = SCost ? 5 : 7; |
| 958 | switch (Opcode) { |
| 959 | case Instruction::UncondBr: |
| 960 | // Branch instruction takes about 4 slots on gfx900. |
| 961 | return SCost ? 1 : 4; |
| 962 | case Instruction::CondBr: |
| 963 | // Suppose conditional branch takes additional 3 exec manipulations |
| 964 | // instructions in average. |
| 965 | return CBrCost; |
| 966 | case Instruction::Switch: { |
| 967 | const auto *SI = dyn_cast_or_null<SwitchInst>(Val: I); |
| 968 | // Each case (including default) takes 1 cmp + 1 cbr instructions in |
| 969 | // average. |
| 970 | return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1); |
| 971 | } |
| 972 | case Instruction::Ret: |
| 973 | return SCost ? 1 : 10; |
| 974 | } |
| 975 | return BaseT::getCFInstrCost(Opcode, CostKind, I); |
| 976 | } |
| 977 | |
| 978 | InstructionCost GCNTTIImpl::getCmpSelInstrCost( |
| 979 | unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, |
| 980 | TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info, |
| 981 | TTI::OperandValueInfo Op2Info, const Instruction *I) const { |
| 982 | // For size and latency cost kinds, return a low cost independent of vector |
| 983 | // width to enable SimplifyCFG's speculativelyExecuteBB optimization. |
| 984 | if (CostKind == TTI::TCK_SizeAndLatency) |
| 985 | return 1; |
| 986 | |
| 987 | return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, |
| 988 | Op1Info, Op2Info, I); |
| 989 | } |
| 990 | |
| 991 | // Measured packing cost of i1 for gfx9-12 is 4.0 to 4.8, up to 5.4 with |
| 992 | // true16; unpacking is 2.6 to 2.9. |
| 993 | static constexpr unsigned MaskPackCostPerElt = 4; |
| 994 | static constexpr unsigned MaskUnpackCostPerElt = 3; |
| 995 | |
| 996 | static std::optional<unsigned> getNumberOfPackedMaskElts(Type *Ty) { |
| 997 | auto *FVT = dyn_cast<FixedVectorType>(Val: Ty); |
| 998 | if (FVT && FVT->getElementType()->isIntegerTy(BitWidth: 1) && FVT->getNumElements() > 1) |
| 999 | return FVT->getNumElements(); |
| 1000 | return std::nullopt; |
| 1001 | } |
| 1002 | |
| 1003 | InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, |
| 1004 | Type *Src, |
| 1005 | TTI::CastContextHint CCH, |
| 1006 | TTI::TargetCostKind CostKind, |
| 1007 | const Instruction *I) const { |
| 1008 | // A bitcast between a vector of i1 and an integer packs or unpacks a mask. |
| 1009 | if (Opcode == Instruction::BitCast) { |
| 1010 | if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Src); |
| 1011 | Elts && Dst->isIntegerTy(BitWidth: *Elts)) |
| 1012 | return InstructionCost(MaskPackCostPerElt) * *Elts * |
| 1013 | getFullRateInstrCost(); |
| 1014 | if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Dst); |
| 1015 | Elts && Src->isIntegerTy(BitWidth: *Elts)) |
| 1016 | return InstructionCost(MaskUnpackCostPerElt) * *Elts * |
| 1017 | getFullRateInstrCost(); |
| 1018 | // The generic code scalarizes a vector cast into element casts, which for |
| 1019 | // a bitcast may pair elements of different sizes. |
| 1020 | if (DL.getTypeSizeInBits(Ty: Src) == DL.getTypeSizeInBits(Ty: Dst)) { |
| 1021 | // Integer and FP values share registers. A legalized vector may need its |
| 1022 | // lanes repacked. |
| 1023 | EVT SrcVT = TLI->getValueType(DL, Ty: Src); |
| 1024 | EVT DstVT = TLI->getValueType(DL, Ty: Dst); |
| 1025 | if (TLI->isTypeLegal(VT: SrcVT) && TLI->isTypeLegal(VT: DstVT)) |
| 1026 | return 0; |
| 1027 | LLVMContext &Ctx = Src->getContext(); |
| 1028 | if (!SrcVT.isVector() && !DstVT.isVector() && |
| 1029 | TLI->getRegisterType(Context&: Ctx, VT: SrcVT).getSizeInBits() == |
| 1030 | TLI->getRegisterType(Context&: Ctx, VT: DstVT).getSizeInBits()) |
| 1031 | return 0; |
| 1032 | } |
| 1033 | } |
| 1034 | |
| 1035 | // Each f32 lane is rounded and the halves are packed in pairs. A packed |
| 1036 | // conversion rounds a pair at once and true16 writes either half. |
| 1037 | if (auto *SrcVTy = dyn_cast<FixedVectorType>(Val: Src); |
| 1038 | SrcVTy && Opcode == Instruction::FPTrunc && ST->has16BitInsts() && |
| 1039 | SrcVTy->getElementType()->isFloatTy() && |
| 1040 | Dst->getScalarType()->isHalfTy()) { |
| 1041 | const unsigned NElts = SrcVTy->getNumElements(); |
| 1042 | const unsigned Ops = ST->hasCvtPkF16F32Inst() ? divideCeil(Numerator: NElts, Denominator: 2) |
| 1043 | : ST->useRealTrue16Insts() ? NElts |
| 1044 | : NElts + NElts / 2; |
| 1045 | return InstructionCost(Ops) * getFullRateInstrCost(); |
| 1046 | } |
| 1047 | |
| 1048 | const int ISD = TLI->InstructionOpcodeToISD(Opcode); |
| 1049 | switch (ISD) { |
| 1050 | case ISD::SINT_TO_FP: |
| 1051 | case ISD::UINT_TO_FP: |
| 1052 | case ISD::FP_TO_SINT: |
| 1053 | case ISD::FP_TO_UINT: |
| 1054 | break; |
| 1055 | default: |
| 1056 | return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
| 1057 | } |
| 1058 | |
| 1059 | const bool IsIntToFP = ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP; |
| 1060 | Type *FPTy = (IsIntToFP ? Dst : Src)->getScalarType(); |
| 1061 | if (!FPTy->isHalfTy() && !FPTy->isBFloatTy() && !FPTy->isFloatTy() && |
| 1062 | !FPTy->isDoubleTy()) |
| 1063 | return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
| 1064 | |
| 1065 | unsigned NElts = 1; |
| 1066 | if (auto *VT = dyn_cast<FixedVectorType>(Val: Src)) |
| 1067 | NElts = VT->getNumElements(); |
| 1068 | |
| 1069 | const unsigned SrcBits = Src->getScalarSizeInBits(); |
| 1070 | const unsigned DstBits = Dst->getScalarSizeInBits(); |
| 1071 | const bool IsSigned = ISD == ISD::SINT_TO_FP || ISD == ISD::FP_TO_SINT; |
| 1072 | const unsigned IntBits = IsIntToFP ? SrcBits : DstBits; |
| 1073 | const bool UsesInt64 = IntBits > 32 && IntBits <= 64; |
| 1074 | |
| 1075 | auto Scale = [&](unsigned FullRateOps, |
| 1076 | unsigned FP64Ops = 0) -> InstructionCost { |
| 1077 | return NElts * (InstructionCost(FullRateOps) * getFullRateInstrCost() + |
| 1078 | InstructionCost(FP64Ops) * get64BitInstrCost(CostKind)); |
| 1079 | }; |
| 1080 | |
| 1081 | if (IsIntToFP) { |
| 1082 | // A scalar load of a whole number of bytes that is not a power of two is |
| 1083 | // split and its high part load extends the source. A constant address |
| 1084 | // space or invariant global load aligned to 4 bytes may be widened instead |
| 1085 | // and then still needs the extension. So does a buffer fat pointer load. |
| 1086 | const auto *Load = |
| 1087 | I && Src->isIntegerTy() && I->getOperand(i: 0)->getType() == Src |
| 1088 | ? dyn_cast<LoadInst>(Val: I->getOperand(i: 0)) |
| 1089 | : nullptr; |
| 1090 | const unsigned AS = Load ? Load->getPointerAddressSpace() : 0; |
| 1091 | const bool LoadMayWiden = |
| 1092 | Load && Load->getAlign() >= Align(4) && |
| 1093 | (AS == AMDGPUAS::CONSTANT_ADDRESS || |
| 1094 | AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT || |
| 1095 | (AS == AMDGPUAS::GLOBAL_ADDRESS && |
| 1096 | Load->hasMetadata(KindID: LLVMContext::MD_invariant_load))); |
| 1097 | const bool LoadExtends = Load && Load->isSimple() && Load->hasOneUse() && |
| 1098 | !LoadMayWiden && |
| 1099 | AS != AMDGPUAS::BUFFER_FAT_POINTER && |
| 1100 | AS != AMDGPUAS::BUFFER_STRIDED_POINTER && |
| 1101 | SrcBits % 8 == 0 && !isPowerOf2_32(Value: SrcBits); |
| 1102 | const unsigned ExtOps = |
| 1103 | UsesInt64 && SrcBits < 64 && !LoadExtends ? (IsSigned ? 2 : 1) : 0; |
| 1104 | if (FPTy->isBFloatTy()) { |
| 1105 | if (SrcBits < 8 || (SrcBits > 32 && !UsesInt64)) |
| 1106 | return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
| 1107 | |
| 1108 | // Each integer is converted to f32 first. |
| 1109 | InstructionCost FloatCost = |
| 1110 | Scale(UsesInt64 ? ExtOps + (IsSigned ? 12 : 8) : 1); |
| 1111 | if (SrcBits < 32) |
| 1112 | FloatCost = getCastInstrCost( |
| 1113 | Opcode, Dst: Dst->getWithNewType(EltTy: Type::getFloatTy(C&: Dst->getContext())), |
| 1114 | Src, CCH, CostKind, I); |
| 1115 | |
| 1116 | // Native rounding can convert a pair. With 16 bit instructions the |
| 1117 | // expansion extracts the low significand bit, adds the rounding bias, |
| 1118 | // preserves NaNs and shifts the result. Without gfx9 instructions the two |
| 1119 | // additions cannot use v_add3_u32. |
| 1120 | InstructionCost RoundCost = |
| 1121 | ST->hasBF16ConversionInsts() |
| 1122 | ? InstructionCost(divideCeil(Numerator: NElts, Denominator: 2)) * getFullRateInstrCost() |
| 1123 | : Scale(!ST->has16BitInsts() ? 1 |
| 1124 | : ST->hasGFX9Insts() ? 6 |
| 1125 | : 7); |
| 1126 | return FloatCost + RoundCost; |
| 1127 | } |
| 1128 | |
| 1129 | // No instruction converts from a 64 bit integer. |
| 1130 | if (UsesInt64) { |
| 1131 | if (FPTy->isDoubleTy()) { |
| 1132 | // Two conversions, ldexp and add, all using the FP64 rate. |
| 1133 | return Scale(ExtOps, 4); |
| 1134 | } |
| 1135 | if (FPTy->isFloatTy()) |
| 1136 | return Scale(ExtOps + (IsSigned ? 12 : 8)); |
| 1137 | return Scale(ExtOps + (IsSigned ? 13 : 9)); |
| 1138 | } |
| 1139 | |
| 1140 | // A narrow vector source is converted lane by lane. |
| 1141 | if (SrcBits >= 8 && SrcBits < 32 && isa<FixedVectorType>(Val: Src)) { |
| 1142 | if (FPTy->isDoubleTy()) |
| 1143 | return Scale(1 + (SrcBits < 16 && IsSigned && ST->has16BitInsts()), 1); |
| 1144 | if (FPTy->isHalfTy()) { |
| 1145 | const InstructionCost PairCost = |
| 1146 | InstructionCost(NElts / 2) * getFullRateInstrCost(); |
| 1147 | // Lanes wider than 16 bits, and every lane without 16 bit instructions, |
| 1148 | // are converted to f32 first. With the packed conversion each pair is |
| 1149 | // then converted at once. Otherwise each lane is converted, and without |
| 1150 | // real true16 each pair of halves is packed. |
| 1151 | if (!ST->has16BitInsts() || SrcBits > 16) { |
| 1152 | auto *FloatTy = |
| 1153 | FixedVectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()), NumElts: NElts); |
| 1154 | const InstructionCost FloatCost = |
| 1155 | getCastInstrCost(Opcode, Dst: FloatTy, Src, CCH, CostKind); |
| 1156 | if (ST->has16BitInsts() && ST->hasCvtPkF16F32Inst()) |
| 1157 | return FloatCost + InstructionCost(divideCeil(Numerator: NElts, Denominator: 2)) * |
| 1158 | getFullRateInstrCost(); |
| 1159 | const unsigned PackOps = |
| 1160 | !ST->has16BitInsts() ? 2 : !ST->useRealTrue16Insts(); |
| 1161 | return FloatCost + Scale(1) + PackOps * PairCost; |
| 1162 | } |
| 1163 | // Each lane is converted from a 16 bit subword. Lanes narrower than 16 |
| 1164 | // bits are extended lane by lane, except bytes with SDWA. Signed bytes |
| 1165 | // are sign extended a pair at a time. Real true16 reads the high |
| 1166 | // subword in place, so only signed lanes narrower than 16 bits pay per |
| 1167 | // pair. Otherwise wider lanes shift the high subword of each pair down |
| 1168 | // without SDWA, and each pair of halves is packed. |
| 1169 | const unsigned PerElt = |
| 1170 | 1 + (SrcBits == 8 ? !ST->hasSDWA() : SrcBits < 16); |
| 1171 | const bool RealTrue16 = ST->useRealTrue16Insts(); |
| 1172 | const unsigned PerPair = |
| 1173 | !RealTrue16 + (SrcBits == 8 || RealTrue16 ? SrcBits < 16 && IsSigned |
| 1174 | : !ST->hasSDWA()); |
| 1175 | return Scale(PerElt) + PerPair * PairCost; |
| 1176 | } |
| 1177 | // An unsigned byte is converted straight out of its register. |
| 1178 | if (SrcBits == 8 && !IsSigned) |
| 1179 | return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
| 1180 | if (SrcBits < 16 && !IsSigned) |
| 1181 | return Scale(2); |
| 1182 | unsigned PerElt = ST->hasSDWA() && SrcBits <= 16 ? 1 : 2; |
| 1183 | if (SrcBits < 16 && ST->has16BitInsts()) |
| 1184 | ++PerElt; |
| 1185 | return Scale(PerElt); |
| 1186 | } |
| 1187 | |
| 1188 | // A narrow scalar source is extended before the conversion. An unsigned |
| 1189 | // byte is converted straight out of its register, and a signed one with |
| 1190 | // SDWA. A source wider than a byte but narrower than 16 bits is masked |
| 1191 | // or sign extended first. Without 16 bit instructions a half result is |
| 1192 | // rounded from f32. |
| 1193 | if (SrcBits >= 8 && SrcBits < 32) { |
| 1194 | if (LoadExtends) { |
| 1195 | if (FPTy->isDoubleTy()) |
| 1196 | return Scale(0, 1); |
| 1197 | return Scale(FPTy->isHalfTy() ? 2 : 1); |
| 1198 | } |
| 1199 | const bool Narrow = SrcBits > 8 && SrcBits < 16; |
| 1200 | const bool SignExtend16 = IsSigned && Narrow && ST->has16BitInsts(); |
| 1201 | if (FPTy->isDoubleTy()) |
| 1202 | return Scale(1 + 2 * SignExtend16, 1); |
| 1203 | if (SrcBits > 16) |
| 1204 | return Scale(FPTy->isHalfTy() ? 3 : 2); |
| 1205 | if (FPTy->isHalfTy() && ST->has16BitInsts()) |
| 1206 | return Scale(Narrow ? (IsSigned ? 3 : 2) |
| 1207 | : SrcBits == 8 && !ST->hasSDWA() ? 2 |
| 1208 | : 1); |
| 1209 | const InstructionCost FloatCost = |
| 1210 | SignExtend16 ? Scale(ST->hasSDWA() ? 3 : 4) |
| 1211 | : Narrow ? Scale(2) |
| 1212 | : SrcBits == 8 && !IsSigned ? Scale(1) |
| 1213 | : Scale(ST->hasSDWA() ? 1 : 2); |
| 1214 | return FPTy->isHalfTy() ? FloatCost + Scale(1) : FloatCost; |
| 1215 | } |
| 1216 | |
| 1217 | return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
| 1218 | } |
| 1219 | |
| 1220 | // No instruction converts to a 64 bit integer. |
| 1221 | if (UsesInt64) { |
| 1222 | const bool IsSigned64 = IsSigned || (DstBits < 64 && NElts == 1); |
| 1223 | if (FPTy->isDoubleTy()) { |
| 1224 | // With native trunc/floor there are six FP64 operations. The expanded |
| 1225 | // rounding sequence has seven, plus integer operations and constants. |
| 1226 | return ST->haveRoundOpsF64() ? Scale(1, 6) : Scale(22, 7); |
| 1227 | } |
| 1228 | // The f32 expansion has one fma, which is quarter rate without fast FMA. |
| 1229 | const InstructionCost SlowFMACost = |
| 1230 | ST->hasFastFMAF32() ? 0 |
| 1231 | : NElts * (getQuarterRateInstrCost(CostKind) - |
| 1232 | getFullRateInstrCost()); |
| 1233 | if (FPTy->isFloatTy()) |
| 1234 | return Scale(IsSigned64 ? 13 : 6) + SlowFMACost; |
| 1235 | if (FPTy->isBFloatTy()) { |
| 1236 | // Unlike half, bf16 does not fit in i32. Extend to f32 and use the full |
| 1237 | // i64 expansion. |
| 1238 | return Scale(1 + (IsSigned64 ? 13 : 6)) + SlowFMACost; |
| 1239 | } |
| 1240 | return Scale(3); |
| 1241 | } |
| 1242 | |
| 1243 | return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); |
| 1244 | } |
| 1245 | |
| 1246 | InstructionCost |
| 1247 | GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, |
| 1248 | std::optional<FastMathFlags> FMF, |
| 1249 | TTI::TargetCostKind CostKind) const { |
| 1250 | if (TTI::requiresOrderedReduction(FMF)) |
| 1251 | return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); |
| 1252 | |
| 1253 | // An add or xor reduction over a vector of i1 becomes a bit count over the |
| 1254 | // packed mask; the generic model prices a shuffle tree and misses that. |
| 1255 | if (Opcode == Instruction::Add || Opcode == Instruction::Xor) { |
| 1256 | if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty)) |
| 1257 | return InstructionCost(MaskPackCostPerElt) * *Elts * |
| 1258 | getFullRateInstrCost(); |
| 1259 | } |
| 1260 | |
| 1261 | EVT OrigTy = TLI->getValueType(DL, Ty); |
| 1262 | |
| 1263 | // Computes cost on targets that have packed math instructions(which support |
| 1264 | // 16-bit types only). |
| 1265 | if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16) |
| 1266 | return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); |
| 1267 | |
| 1268 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty); |
| 1269 | return LT.first * getFullRateInstrCost(); |
| 1270 | } |
| 1271 | |
| 1272 | InstructionCost |
| 1273 | GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, |
| 1274 | FastMathFlags FMF, |
| 1275 | TTI::TargetCostKind CostKind) const { |
| 1276 | EVT OrigTy = TLI->getValueType(DL, Ty); |
| 1277 | |
| 1278 | // Computes cost on targets that have packed math instructions(which support |
| 1279 | // 16-bit types only). |
| 1280 | if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16) |
| 1281 | return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind); |
| 1282 | |
| 1283 | std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty); |
| 1284 | return LT.first * getHalfRateInstrCost(CostKind); |
| 1285 | } |
| 1286 | |
| 1287 | InstructionCost GCNTTIImpl::getVectorInstrCost( |
| 1288 | unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index, |
| 1289 | const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const { |
| 1290 | switch (Opcode) { |
| 1291 | case Instruction::ExtractElement: |
| 1292 | case Instruction::InsertElement: { |
| 1293 | TypeSize EltSize = |
| 1294 | DL.getTypeSizeInBits(Ty: cast<VectorType>(Val: ValTy)->getElementType()); |
| 1295 | // Dynamic indexing isn't free and is best avoided. |
| 1296 | if (Index == ~0u) |
| 1297 | return 2; |
| 1298 | if (EltSize < 32) { |
| 1299 | if (EltSize == 16 && Index == 0 && ST->has16BitInsts()) |
| 1300 | return 0; |
| 1301 | // Inserts of booleans are free. |
| 1302 | // TODO: Extracts are free too. |
| 1303 | if (EltSize == 1 && Opcode == Instruction::InsertElement) |
| 1304 | return TargetTransformInfo::TCC_Free; |
| 1305 | // Extract element sequences of consecutive i8 values that match a |
| 1306 | // register size are free most likely. It is not possible to know |
| 1307 | // if this extract is part of a consecutive sequence so this may |
| 1308 | // apply more generally. |
| 1309 | if (Opcode == Instruction::ExtractElement && EltSize == 8) { |
| 1310 | if (auto *FVTy = dyn_cast<FixedVectorType>(Val: ValTy)) { |
| 1311 | unsigned NumElts = FVTy->getNumElements(); |
| 1312 | if (NumElts >= 4 && isPowerOf2_32(Value: NumElts)) |
| 1313 | return 0; |
| 1314 | } |
| 1315 | } |
| 1316 | return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1, |
| 1317 | VIC); |
| 1318 | } |
| 1319 | |
| 1320 | // Packed f32 pair formation for v_pk_*_f32. A constant-index insert writes |
| 1321 | // one subregister, so a lane holding a computed value costs at most one |
| 1322 | // v_mov_b32 at any width. A loaded lane lands in place, and arguments and |
| 1323 | // constants are already in a register, so both are free. SLP prices a |
| 1324 | // buildvector root with the insertelements themselves as Op1, and those |
| 1325 | // lanes need no packing either. Only throughput sees the move; size-based |
| 1326 | // users such as the inliner and the unroller keep inserts free. |
| 1327 | auto *VecTy = dyn_cast<FixedVectorType>(Val: ValTy); |
| 1328 | const auto *InsertedInst = dyn_cast_or_null<Instruction>(Val: Op1); |
| 1329 | if (Opcode == Instruction::InsertElement && EltSize == 32 && |
| 1330 | CostKind == TTI::TCK_RecipThroughput && ST->hasAnyPackedFP32Ops() && |
| 1331 | InsertedInst && VecTy && VecTy->getElementType()->isFloatTy() && |
| 1332 | !isa<LoadInst, InsertElementInst>(Val: InsertedInst)) |
| 1333 | return getFullRateInstrCost(); |
| 1334 | |
| 1335 | // Extracts are just reads of a subregister, so are free. Inserts are |
| 1336 | // considered free because we don't want to have any cost for scalarizing |
| 1337 | // operations, and we don't have to copy into a different register class. |
| 1338 | return 0; |
| 1339 | } |
| 1340 | default: |
| 1341 | return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1, |
| 1342 | VIC); |
| 1343 | } |
| 1344 | } |
| 1345 | |
| 1346 | /// Analyze if the results of inline asm are divergent. If \p Indices is empty, |
| 1347 | /// this is analyzing the collective result of all output registers. Otherwise, |
| 1348 | /// this is only querying a specific result index if this returns multiple |
| 1349 | /// registers in a struct. |
| 1350 | bool GCNTTIImpl::isInlineAsmSourceOfDivergence( |
| 1351 | const CallBase *CB, ArrayRef<unsigned> Indices) const { |
| 1352 | // TODO: Handle complex extract indices |
| 1353 | if (Indices.size() > 1) |
| 1354 | return true; |
| 1355 | |
| 1356 | const DataLayout &DL = CB->getDataLayout(); |
| 1357 | const SIRegisterInfo *TRI = ST->getRegisterInfo(); |
| 1358 | TargetLowering::AsmOperandInfoVector TargetConstraints = |
| 1359 | TLI->ParseConstraints(DL, TRI: ST->getRegisterInfo(), Call: *CB); |
| 1360 | |
| 1361 | const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0]; |
| 1362 | |
| 1363 | int OutputIdx = 0; |
| 1364 | for (auto &TC : TargetConstraints) { |
| 1365 | if (TC.Type != InlineAsm::isOutput) |
| 1366 | continue; |
| 1367 | |
| 1368 | // Skip outputs we don't care about. |
| 1369 | if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++) |
| 1370 | continue; |
| 1371 | |
| 1372 | TLI->ComputeConstraintToUse(OpInfo&: TC, Op: SDValue()); |
| 1373 | |
| 1374 | const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint( |
| 1375 | TRI, Constraint: TC.ConstraintCode, VT: TC.ConstraintVT).second; |
| 1376 | |
| 1377 | // For AGPR constraints null is returned on subtargets without AGPRs, so |
| 1378 | // assume divergent for null. |
| 1379 | if (!RC || !TRI->isSGPRClass(RC)) |
| 1380 | return true; |
| 1381 | } |
| 1382 | |
| 1383 | return false; |
| 1384 | } |
| 1385 | |
| 1386 | bool GCNTTIImpl::isReadRegisterSourceOfDivergence( |
| 1387 | const IntrinsicInst *ReadReg) const { |
| 1388 | Metadata *MD = |
| 1389 | cast<MetadataAsValue>(Val: ReadReg->getArgOperand(i: 0))->getMetadata(); |
| 1390 | StringRef RegName = |
| 1391 | cast<MDString>(Val: cast<MDNode>(Val: MD)->getOperand(I: 0))->getString(); |
| 1392 | |
| 1393 | // Special case registers that look like VCC. |
| 1394 | MVT VT = MVT::getVT(Ty: ReadReg->getType()); |
| 1395 | if (VT == MVT::i1) |
| 1396 | return true; |
| 1397 | |
| 1398 | // Special case scalar registers that start with 'v'. |
| 1399 | if (RegName.starts_with(Prefix: "vcc" ) || RegName.empty()) |
| 1400 | return false; |
| 1401 | |
| 1402 | // VGPR or AGPR is divergent. There aren't any specially named vector |
| 1403 | // registers. |
| 1404 | return RegName[0] == 'v' || RegName[0] == 'a'; |
| 1405 | } |
| 1406 | |
| 1407 | /// \returns true if the result of the value could potentially be |
| 1408 | /// different across workitems in a wavefront. |
| 1409 | bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const { |
| 1410 | if (const Argument *A = dyn_cast<Argument>(Val: V)) |
| 1411 | return !AMDGPU::isArgPassedInSGPR(Arg: A); |
| 1412 | |
| 1413 | // Loads from the private and flat address spaces are divergent, because |
| 1414 | // threads can execute the load instruction with the same inputs and get |
| 1415 | // different results. So are loads from the VGPR address space, since each |
| 1416 | // lane reads its own registers. |
| 1417 | // |
| 1418 | // All other loads are not divergent, because if threads issue loads with the |
| 1419 | // same arguments, they will always get the same result. |
| 1420 | if (const LoadInst *Load = dyn_cast<LoadInst>(Val: V)) |
| 1421 | return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS || |
| 1422 | Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS || |
| 1423 | Load->getPointerAddressSpace() == AMDGPUAS::VGPR; |
| 1424 | |
| 1425 | // Atomics are divergent because they are executed sequentially: when an |
| 1426 | // atomic operation refers to the same address in each thread, then each |
| 1427 | // thread after the first sees the value written by the previous thread as |
| 1428 | // original value. |
| 1429 | if (isa<AtomicRMWInst, AtomicCmpXchgInst>(Val: V)) |
| 1430 | return true; |
| 1431 | |
| 1432 | if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) { |
| 1433 | Intrinsic::ID IID = Intrinsic->getIntrinsicID(); |
| 1434 | switch (IID) { |
| 1435 | case Intrinsic::read_register: |
| 1436 | return isReadRegisterSourceOfDivergence(ReadReg: Intrinsic); |
| 1437 | case Intrinsic::amdgcn_workitem_id_y: |
| 1438 | case Intrinsic::amdgcn_workitem_id_z: { |
| 1439 | const Function *F = Intrinsic->getFunction(); |
| 1440 | bool HasUniformYZ = |
| 1441 | ST->hasWavefrontsEvenlySplittingXDim(F: *F, /*RequiresUniformYZ=*/true); |
| 1442 | std::optional<unsigned> ThisDimSize = ST->getReqdWorkGroupSize( |
| 1443 | F: *F, Dim: IID == Intrinsic::amdgcn_workitem_id_y ? 1 : 2); |
| 1444 | return !HasUniformYZ && (!ThisDimSize || *ThisDimSize != 1); |
| 1445 | } |
| 1446 | default: |
| 1447 | return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID); |
| 1448 | } |
| 1449 | } |
| 1450 | |
| 1451 | // Assume all function calls are a source of divergence. |
| 1452 | if (const CallBase *CB = dyn_cast<CallBase>(Val: V)) { |
| 1453 | if (CB->isInlineAsm()) |
| 1454 | return isInlineAsmSourceOfDivergence(CB); |
| 1455 | return true; |
| 1456 | } |
| 1457 | |
| 1458 | // Assume all function calls are a source of divergence. |
| 1459 | if (isa<InvokeInst>(Val: V)) |
| 1460 | return true; |
| 1461 | |
| 1462 | // If the target supports globally addressable scratch, the mapping from |
| 1463 | // scratch memory to the flat aperture changes therefore an address space cast |
| 1464 | // is no longer uniform. |
| 1465 | if (auto *CastI = dyn_cast<AddrSpaceCastInst>(Val: V)) { |
| 1466 | return CastI->getSrcAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS && |
| 1467 | CastI->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS && |
| 1468 | ST->hasGloballyAddressableScratch(); |
| 1469 | } |
| 1470 | |
| 1471 | return false; |
| 1472 | } |
| 1473 | |
| 1474 | bool GCNTTIImpl::isAlwaysUniform(const Value *V) const { |
| 1475 | if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) |
| 1476 | return AMDGPU::isIntrinsicAlwaysUniform(IntrID: Intrinsic->getIntrinsicID()); |
| 1477 | |
| 1478 | if (const CallBase *CB = dyn_cast<CallBase>(Val: V)) { |
| 1479 | if (CB->isInlineAsm()) |
| 1480 | return !isInlineAsmSourceOfDivergence(CB); |
| 1481 | return false; |
| 1482 | } |
| 1483 | |
| 1484 | // In most cases TID / wavefrontsize is uniform. |
| 1485 | // |
| 1486 | // However, if a kernel has uneven dimesions we can have a value of |
| 1487 | // workitem-id-x divided by the wavefrontsize non-uniform. For example |
| 1488 | // dimensions (65, 2) will have workitems with address (64, 0) and (0, 1) |
| 1489 | // packed into a same wave which gives 1 and 0 after the division by 64 |
| 1490 | // respectively. |
| 1491 | // |
| 1492 | // The X dimension doesn't reset within a wave if either both the Y |
| 1493 | // and Z dimensions are of length 1, or if the X dimension's required |
| 1494 | // size is a power of 2. Note, however, if the X dimension's maximum |
| 1495 | // size is a power of 2 < the wavefront size, division by the wavefront |
| 1496 | // size is guaranteed to yield 0, so this is also a no-reset case. |
| 1497 | bool XDimDoesntResetWithinWaves = false; |
| 1498 | if (auto *I = dyn_cast<Instruction>(Val: V)) { |
| 1499 | const Function *F = I->getFunction(); |
| 1500 | XDimDoesntResetWithinWaves = ST->hasWavefrontsEvenlySplittingXDim(F: *F); |
| 1501 | } |
| 1502 | using namespace llvm::PatternMatch; |
| 1503 | uint64_t C; |
| 1504 | auto MatchTidXCall = m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(); |
| 1505 | auto MaybeMaskedTidX = |
| 1506 | m_CombineOr(Ps: m_c_And(L: MatchTidXCall, R: m_Constant()), Ps: MatchTidXCall); |
| 1507 | auto MaybeCastTidX = m_CastOrSelf(Op: MaybeMaskedTidX); |
| 1508 | auto MaybeMaskedCastTidX = |
| 1509 | m_CombineOr(Ps: m_c_And(L: MaybeCastTidX, R: m_Constant()), Ps: MaybeCastTidX); |
| 1510 | if (match(V, P: m_LShr(L: MaybeMaskedCastTidX, R: m_ConstantInt(V&: C)))) |
| 1511 | return C >= ST->getWavefrontSizeLog2() && XDimDoesntResetWithinWaves; |
| 1512 | |
| 1513 | Constant *Mask; |
| 1514 | if (match(V, P: m_c_And( |
| 1515 | L: m_CastOrSelf(Op: m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>()), |
| 1516 | R: m_Constant(C&: Mask)))) { |
| 1517 | return computeKnownBits(V: Mask, DL).countMinTrailingZeros() >= |
| 1518 | ST->getWavefrontSizeLog2() && |
| 1519 | XDimDoesntResetWithinWaves; |
| 1520 | } |
| 1521 | |
| 1522 | const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(Val: V); |
| 1523 | if (!ExtValue) |
| 1524 | return false; |
| 1525 | |
| 1526 | const CallBase *CB = dyn_cast<CallBase>(Val: ExtValue->getOperand(i_nocapture: 0)); |
| 1527 | if (!CB) |
| 1528 | return false; |
| 1529 | |
| 1530 | if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: CB)) { |
| 1531 | switch (Intrinsic->getIntrinsicID()) { |
| 1532 | default: |
| 1533 | return false; |
| 1534 | case Intrinsic::amdgcn_if: |
| 1535 | case Intrinsic::amdgcn_else: { |
| 1536 | ArrayRef<unsigned> Indices = ExtValue->getIndices(); |
| 1537 | return Indices.size() == 1 && Indices[0] == 1; |
| 1538 | } |
| 1539 | } |
| 1540 | } |
| 1541 | |
| 1542 | // If we have inline asm returning mixed SGPR and VGPR results, we inferred |
| 1543 | // divergent for the overall struct return. We need to override it in the |
| 1544 | // case we're extracting an SGPR component here. |
| 1545 | if (CB->isInlineAsm()) |
| 1546 | return !isInlineAsmSourceOfDivergence(CB, Indices: ExtValue->getIndices()); |
| 1547 | |
| 1548 | return false; |
| 1549 | } |
| 1550 | |
| 1551 | bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes, |
| 1552 | Intrinsic::ID IID) const { |
| 1553 | switch (IID) { |
| 1554 | case Intrinsic::amdgcn_is_shared: |
| 1555 | case Intrinsic::amdgcn_is_private: |
| 1556 | case Intrinsic::amdgcn_flat_atomic_fmax_num: |
| 1557 | case Intrinsic::amdgcn_flat_atomic_fmin_num: |
| 1558 | case Intrinsic::amdgcn_load_to_lds: |
| 1559 | case Intrinsic::amdgcn_make_buffer_rsrc: |
| 1560 | case Intrinsic::amdgcn_flat_prefetch: |
| 1561 | OpIndexes.push_back(Elt: 0); |
| 1562 | return true; |
| 1563 | default: |
| 1564 | return false; |
| 1565 | } |
| 1566 | } |
| 1567 | |
| 1568 | Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, |
| 1569 | Value *OldV, |
| 1570 | Value *NewV) const { |
| 1571 | auto IntrID = II->getIntrinsicID(); |
| 1572 | switch (IntrID) { |
| 1573 | case Intrinsic::amdgcn_is_shared: |
| 1574 | case Intrinsic::amdgcn_is_private: { |
| 1575 | unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ? |
| 1576 | AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS; |
| 1577 | unsigned NewAS = NewV->getType()->getPointerAddressSpace(); |
| 1578 | LLVMContext &Ctx = NewV->getType()->getContext(); |
| 1579 | ConstantInt *NewVal = (TrueAS == NewAS) ? |
| 1580 | ConstantInt::getTrue(Context&: Ctx) : ConstantInt::getFalse(Context&: Ctx); |
| 1581 | return NewVal; |
| 1582 | } |
| 1583 | case Intrinsic::amdgcn_flat_atomic_fmax_num: |
| 1584 | case Intrinsic::amdgcn_flat_atomic_fmin_num: { |
| 1585 | Type *DestTy = II->getType(); |
| 1586 | Type *SrcTy = NewV->getType(); |
| 1587 | unsigned NewAS = SrcTy->getPointerAddressSpace(); |
| 1588 | if (!AMDGPU::isExtendedGlobalAddrSpace(AS: NewAS)) |
| 1589 | return nullptr; |
| 1590 | Module *M = II->getModule(); |
| 1591 | Function *NewDecl = Intrinsic::getOrInsertDeclaration( |
| 1592 | M, id: II->getIntrinsicID(), OverloadTys: {DestTy, SrcTy, DestTy}); |
| 1593 | II->setArgOperand(i: 0, v: NewV); |
| 1594 | II->setCalledFunction(NewDecl); |
| 1595 | return II; |
| 1596 | } |
| 1597 | case Intrinsic::amdgcn_load_to_lds: { |
| 1598 | Type *SrcTy = NewV->getType(); |
| 1599 | Module *M = II->getModule(); |
| 1600 | Function *NewDecl = |
| 1601 | Intrinsic::getOrInsertDeclaration(M, id: II->getIntrinsicID(), OverloadTys: {SrcTy}); |
| 1602 | II->setArgOperand(i: 0, v: NewV); |
| 1603 | II->setCalledFunction(NewDecl); |
| 1604 | return II; |
| 1605 | } |
| 1606 | case Intrinsic::amdgcn_make_buffer_rsrc: { |
| 1607 | Type *SrcTy = NewV->getType(); |
| 1608 | Type *DstTy = II->getType(); |
| 1609 | Type *NumRecordsTy = II->getArgOperand(i: 2)->getType(); |
| 1610 | Module *M = II->getModule(); |
| 1611 | Function *NewDecl = Intrinsic::getOrInsertDeclaration( |
| 1612 | M, id: II->getIntrinsicID(), OverloadTys: {DstTy, SrcTy, NumRecordsTy}); |
| 1613 | II->setArgOperand(i: 0, v: NewV); |
| 1614 | II->setCalledFunction(NewDecl); |
| 1615 | return II; |
| 1616 | } |
| 1617 | case Intrinsic::amdgcn_flat_prefetch: { |
| 1618 | unsigned NewAS = NewV->getType()->getPointerAddressSpace(); |
| 1619 | if (!AMDGPU::isExtendedGlobalAddrSpace(AS: NewAS)) |
| 1620 | return nullptr; |
| 1621 | // llvm.amdgcn.global.prefetch only takes a global pointer. |
| 1622 | if (NewAS != AMDGPUAS::GLOBAL_ADDRESS) |
| 1623 | NewV = IRBuilder<>(II).CreateAddrSpaceCast( |
| 1624 | V: NewV, DestTy: PointerType::get(C&: II->getContext(), AddressSpace: AMDGPUAS::GLOBAL_ADDRESS)); |
| 1625 | Function *NewDecl = Intrinsic::getOrInsertDeclaration( |
| 1626 | M: II->getModule(), id: Intrinsic::amdgcn_global_prefetch); |
| 1627 | II->setArgOperand(i: 0, v: NewV); |
| 1628 | II->setCalledFunction(NewDecl); |
| 1629 | return II; |
| 1630 | } |
| 1631 | default: |
| 1632 | return nullptr; |
| 1633 | } |
| 1634 | } |
| 1635 | |
| 1636 | InstructionCost GCNTTIImpl::getShuffleCost( |
| 1637 | TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, |
| 1638 | TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index, |
| 1639 | VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI, |
| 1640 | TTI::VectorInstrContext VIC) const { |
| 1641 | if (!isa<FixedVectorType>(Val: SrcTy)) |
| 1642 | return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index, |
| 1643 | SubTp); |
| 1644 | |
| 1645 | Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp); |
| 1646 | |
| 1647 | TypeSize ScalarSize = DL.getTypeSizeInBits(Ty: SrcTy->getElementType()); |
| 1648 | |
| 1649 | // A v_pk_*_f32 operand is an aligned register pair. A lane already in its |
| 1650 | // half of a source pair is read in place, and a splat of the low half is an |
| 1651 | // op_sel_hi. Every other lane costs a v_mov_b32 to move it into place. |
| 1652 | if (ScalarSize == 32 && CostKind == TTI::TCK_RecipThroughput && |
| 1653 | ST->hasAnyPackedFP32Ops() && SrcTy->getElementType()->isFloatTy() && |
| 1654 | !Mask.empty()) { |
| 1655 | const int NumSrcElts = cast<FixedVectorType>(Val: SrcTy)->getNumElements(); |
| 1656 | // Source pairs of the second operand are numbered after the first. |
| 1657 | auto GetPair = [&](int Idx) { |
| 1658 | return Idx < NumSrcElts ? Idx / 2 : NumSrcElts + (Idx - NumSrcElts) / 2; |
| 1659 | }; |
| 1660 | auto GetHalf = [&](int Idx) { |
| 1661 | return Idx < NumSrcElts ? Idx % 2 : (Idx - NumSrcElts) % 2; |
| 1662 | }; |
| 1663 | unsigned Cost = 0; |
| 1664 | for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += 2) { |
| 1665 | const int Lo = Mask[DstIdx]; |
| 1666 | const int Hi = |
| 1667 | DstIdx + 1 < Mask.size() ? Mask[DstIdx + 1] : PoisonMaskElem; |
| 1668 | const bool LoInPlace = Lo != PoisonMaskElem && GetHalf(Lo) == 0; |
| 1669 | const bool HiInPlace = Hi != PoisonMaskElem && GetHalf(Hi) == 1; |
| 1670 | if (Lo != PoisonMaskElem && Lo == Hi && LoInPlace) |
| 1671 | continue; |
| 1672 | if (LoInPlace && HiInPlace) |
| 1673 | Cost += GetPair(Lo) != GetPair(Hi); |
| 1674 | else |
| 1675 | Cost += (Lo != PoisonMaskElem && !LoInPlace) + |
| 1676 | (Hi != PoisonMaskElem && !HiInPlace); |
| 1677 | } |
| 1678 | return Cost; |
| 1679 | } |
| 1680 | |
| 1681 | if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && |
| 1682 | (ScalarSize == 16 || ScalarSize == 8)) { |
| 1683 | // Larger vector widths may require additional instructions, but are |
| 1684 | // typically cheaper than scalarized versions. |
| 1685 | // |
| 1686 | // We assume that shuffling at a register granularity can be done for free. |
| 1687 | // This is not true for vectors fed into memory instructions, but it is |
| 1688 | // effectively true for all other shuffling. The emphasis of the logic here |
| 1689 | // is to assist generic transform in cleaning up / canonicalizing those |
| 1690 | // shuffles. |
| 1691 | |
| 1692 | // With op_sel VOP3P instructions freely can access the low half or high |
| 1693 | // half of a register, so any swizzle of two elements is free. |
| 1694 | if (auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy)) { |
| 1695 | unsigned NumSrcElts = SrcVecTy->getNumElements(); |
| 1696 | if (ST->hasVOP3PInsts() && ScalarSize == 16 && NumSrcElts == 2 && |
| 1697 | (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Reverse || |
| 1698 | Kind == TTI::SK_PermuteSingleSrc)) |
| 1699 | return 0; |
| 1700 | } |
| 1701 | |
| 1702 | unsigned EltsPerReg = 32 / static_cast<unsigned>(ScalarSize); |
| 1703 | switch (Kind) { |
| 1704 | case TTI::SK_Broadcast: |
| 1705 | // A single v_perm_b32 can be re-used for all destination registers. |
| 1706 | return 1; |
| 1707 | case TTI::SK_Reverse: |
| 1708 | // One instruction per register. |
| 1709 | if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy)) |
| 1710 | return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg); |
| 1711 | return InstructionCost::getInvalid(); |
| 1712 | case TTI::SK_ExtractSubvector: |
| 1713 | if (Index % EltsPerReg == 0) |
| 1714 | return 0; // Shuffling at register granularity |
| 1715 | if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy)) |
| 1716 | return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg); |
| 1717 | return InstructionCost::getInvalid(); |
| 1718 | case TTI::SK_InsertSubvector: { |
| 1719 | auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy); |
| 1720 | if (!DstVecTy) |
| 1721 | return InstructionCost::getInvalid(); |
| 1722 | unsigned NumDstElts = DstVecTy->getNumElements(); |
| 1723 | unsigned NumInsertElts = cast<FixedVectorType>(Val: SubTp)->getNumElements(); |
| 1724 | unsigned EndIndex = Index + NumInsertElts; |
| 1725 | unsigned BeginSubIdx = Index % EltsPerReg; |
| 1726 | unsigned EndSubIdx = EndIndex % EltsPerReg; |
| 1727 | InstructionCost Cost = 0; |
| 1728 | |
| 1729 | if (BeginSubIdx != 0) { |
| 1730 | // Need to shift the inserted vector into place. The cost is the number |
| 1731 | // of destination registers overlapped by the inserted vector. |
| 1732 | Cost = divideCeil(Numerator: EndIndex, Denominator: EltsPerReg) - (Index / EltsPerReg); |
| 1733 | } |
| 1734 | |
| 1735 | // If the last register overlap is partial, there may be three source |
| 1736 | // registers feeding into it; that takes an extra instruction. |
| 1737 | if (EndIndex < NumDstElts && BeginSubIdx < EndSubIdx) |
| 1738 | Cost += 1; |
| 1739 | |
| 1740 | return Cost; |
| 1741 | } |
| 1742 | case TTI::SK_Splice: { |
| 1743 | auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy); |
| 1744 | if (!DstVecTy) |
| 1745 | return InstructionCost::getInvalid(); |
| 1746 | unsigned NumElts = DstVecTy->getNumElements(); |
| 1747 | assert(NumElts == cast<FixedVectorType>(SrcTy)->getNumElements()); |
| 1748 | // Determine the sub-region of the result vector that requires |
| 1749 | // sub-register shuffles / mixing. |
| 1750 | unsigned EltsFromLHS = NumElts - Index; |
| 1751 | bool LHSIsAligned = (Index % EltsPerReg) == 0; |
| 1752 | bool RHSIsAligned = (EltsFromLHS % EltsPerReg) == 0; |
| 1753 | if (LHSIsAligned && RHSIsAligned) |
| 1754 | return 0; |
| 1755 | if (LHSIsAligned && !RHSIsAligned) |
| 1756 | return divideCeil(Numerator: NumElts, Denominator: EltsPerReg) - (EltsFromLHS / EltsPerReg); |
| 1757 | if (!LHSIsAligned && RHSIsAligned) |
| 1758 | return divideCeil(Numerator: EltsFromLHS, Denominator: EltsPerReg); |
| 1759 | return divideCeil(Numerator: NumElts, Denominator: EltsPerReg); |
| 1760 | } |
| 1761 | default: |
| 1762 | break; |
| 1763 | } |
| 1764 | |
| 1765 | if (!Mask.empty()) { |
| 1766 | unsigned NumSrcElts = cast<FixedVectorType>(Val: SrcTy)->getNumElements(); |
| 1767 | |
| 1768 | // Generically estimate the cost by assuming that each destination |
| 1769 | // register is derived from sources via v_perm_b32 instructions if it |
| 1770 | // can't be copied as-is. |
| 1771 | // |
| 1772 | // For each destination register, derive the cost of obtaining it based |
| 1773 | // on the number of source registers that feed into it. |
| 1774 | InstructionCost Cost = 0; |
| 1775 | for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += EltsPerReg) { |
| 1776 | SmallVector<int, 4> Regs; |
| 1777 | bool Aligned = true; |
| 1778 | for (unsigned I = 0; I < EltsPerReg && DstIdx + I < Mask.size(); ++I) { |
| 1779 | int SrcIdx = Mask[DstIdx + I]; |
| 1780 | if (SrcIdx == -1) |
| 1781 | continue; |
| 1782 | int Reg; |
| 1783 | if (SrcIdx < (int)NumSrcElts) { |
| 1784 | Reg = SrcIdx / EltsPerReg; |
| 1785 | if (SrcIdx % EltsPerReg != I) |
| 1786 | Aligned = false; |
| 1787 | } else { |
| 1788 | Reg = NumSrcElts + (SrcIdx - NumSrcElts) / EltsPerReg; |
| 1789 | if ((SrcIdx - NumSrcElts) % EltsPerReg != I) |
| 1790 | Aligned = false; |
| 1791 | } |
| 1792 | if (!llvm::is_contained(Range&: Regs, Element: Reg)) |
| 1793 | Regs.push_back(Elt: Reg); |
| 1794 | } |
| 1795 | if (Regs.size() >= 2) |
| 1796 | Cost += Regs.size() - 1; |
| 1797 | else if (!Aligned) |
| 1798 | Cost += 1; |
| 1799 | } |
| 1800 | return Cost; |
| 1801 | } |
| 1802 | } |
| 1803 | |
| 1804 | return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index, |
| 1805 | SubTp); |
| 1806 | } |
| 1807 | |
| 1808 | /// Whether it is profitable to sink the operands of an |
| 1809 | /// Instruction I to the basic block of I. |
| 1810 | /// This helps using several modifiers (like abs and neg) more often. |
| 1811 | bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I, |
| 1812 | SmallVectorImpl<Use *> &Ops) const { |
| 1813 | using namespace PatternMatch; |
| 1814 | |
| 1815 | // The cost model prices this fmul as free assuming it fuses with its |
| 1816 | // fadd/fsub user, which needs them in one block. Sink a stranded |
| 1817 | // loop-invariant fmul back to the user when they would fuse. Single use only, |
| 1818 | // so this stays a move. |
| 1819 | if (I->getOpcode() == Instruction::FAdd || |
| 1820 | I->getOpcode() == Instruction::FSub) { |
| 1821 | const Instruction *FMul = getFusedFMul(TLI: *TLI, Ty: I->getType(), FAddSub: I); |
| 1822 | if (FMul && FMul->getParent() != I->getParent()) |
| 1823 | Ops.push_back(Elt: &I->getOperandUse(i: I->getOperand(i: 0) == FMul ? 0 : 1)); |
| 1824 | } |
| 1825 | |
| 1826 | for (auto &Op : I->operands()) { |
| 1827 | // Ensure we are not already sinking this operand. |
| 1828 | if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op.get(); })) |
| 1829 | continue; |
| 1830 | |
| 1831 | // TODO: Confirm whether m_FNegOrFSubNegZero is correct. |
| 1832 | if (match(V: &Op, P: m_FAbs(Op0: m_Value())) || |
| 1833 | match(V: &Op, P: m_FNegOrFSubNegZero(X: m_Value()))) { |
| 1834 | Ops.push_back(Elt: &Op); |
| 1835 | continue; |
| 1836 | } |
| 1837 | |
| 1838 | // Check for zero-cost multiple use InsertElement/ExtractElement |
| 1839 | // instructions |
| 1840 | if (Instruction *OpInst = dyn_cast<Instruction>(Val: Op.get())) { |
| 1841 | if (OpInst->getType()->isVectorTy() && OpInst->getNumOperands() > 1) { |
| 1842 | Instruction *VecOpInst = dyn_cast<Instruction>(Val: OpInst->getOperand(i: 0)); |
| 1843 | if (VecOpInst && VecOpInst->hasOneUse()) |
| 1844 | continue; |
| 1845 | |
| 1846 | if (getVectorInstrCost(Opcode: OpInst->getOpcode(), ValTy: OpInst->getType(), |
| 1847 | CostKind: TTI::TCK_RecipThroughput, Index: 0, |
| 1848 | Op0: OpInst->getOperand(i: 0), |
| 1849 | Op1: OpInst->getOperand(i: 1)) == 0) { |
| 1850 | Ops.push_back(Elt: &Op); |
| 1851 | continue; |
| 1852 | } |
| 1853 | } |
| 1854 | } |
| 1855 | |
| 1856 | if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: Op.get())) { |
| 1857 | |
| 1858 | TypeSize EltSize = DL.getTypeSizeInBits( |
| 1859 | Ty: cast<VectorType>(Val: Shuffle->getType())->getElementType()); |
| 1860 | |
| 1861 | // For i32 (or greater) shufflevectors, these will be lowered into a |
| 1862 | // series of insert / extract elements, which will be coalesced away. |
| 1863 | if (EltSize < 16 || !ST->has16BitInsts()) |
| 1864 | continue; |
| 1865 | |
| 1866 | int NumSubElts, SubIndex; |
| 1867 | if (Shuffle->changesLength()) { |
| 1868 | if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding()) { |
| 1869 | Ops.push_back(Elt: &Op); |
| 1870 | continue; |
| 1871 | } |
| 1872 | |
| 1873 | if ((Shuffle->isExtractSubvectorMask(Index&: SubIndex) || |
| 1874 | Shuffle->isInsertSubvectorMask(NumSubElts, Index&: SubIndex)) && |
| 1875 | !(SubIndex & 0x1)) { |
| 1876 | Ops.push_back(Elt: &Op); |
| 1877 | continue; |
| 1878 | } |
| 1879 | } |
| 1880 | |
| 1881 | if (Shuffle->isReverse() || Shuffle->isZeroEltSplat() || |
| 1882 | Shuffle->isSingleSource()) { |
| 1883 | Ops.push_back(Elt: &Op); |
| 1884 | continue; |
| 1885 | } |
| 1886 | } |
| 1887 | } |
| 1888 | |
| 1889 | return !Ops.empty(); |
| 1890 | } |
| 1891 | |
| 1892 | bool GCNTTIImpl::areInlineCompatible(const Function *Caller, |
| 1893 | const Function *Callee) const { |
| 1894 | const TargetMachine &TM = getTLI()->getTargetMachine(); |
| 1895 | const GCNSubtarget *CallerST |
| 1896 | = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller)); |
| 1897 | const GCNSubtarget *CalleeST |
| 1898 | = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee)); |
| 1899 | |
| 1900 | if (!BaseT::areInlineCompatible(Caller, Callee)) |
| 1901 | return false; |
| 1902 | |
| 1903 | // FIXME: dx10_clamp can just take the caller setting, but there seems to be |
| 1904 | // no way to support merge for backend defined attributes. |
| 1905 | SIModeRegisterDefaults CallerMode(*Caller, *CallerST); |
| 1906 | SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST); |
| 1907 | if (!CallerMode.isInlineCompatible(CalleeMode)) |
| 1908 | return false; |
| 1909 | |
| 1910 | if (Callee->hasFnAttribute(Kind: Attribute::AlwaysInline) || |
| 1911 | Callee->hasFnAttribute(Kind: Attribute::InlineHint)) |
| 1912 | return true; |
| 1913 | |
| 1914 | // Hack to make compile times reasonable. |
| 1915 | if (CLOpts.inline_max_bb) { |
| 1916 | // Single BB does not increase total BB amount. |
| 1917 | if (Callee->size() == 1) |
| 1918 | return true; |
| 1919 | size_t BBSize = Caller->size() + Callee->size() - 1; |
| 1920 | if (BBSize > CLOpts.inline_max_bb) { |
| 1921 | LLVM_DEBUG(dbgs() << "AMDGPU inline max-BB rejected inlining " |
| 1922 | << Callee->getName() << " into " << Caller->getName() |
| 1923 | << ": caller BBs=" << Caller->size() << ", callee BBs=" |
| 1924 | << Callee->size() << ", combined BBs=" << BBSize |
| 1925 | << ", max BBs=" << CLOpts.inline_max_bb << '\n'); |
| 1926 | return false; |
| 1927 | } |
| 1928 | } |
| 1929 | |
| 1930 | return true; |
| 1931 | } |
| 1932 | |
| 1933 | static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB, |
| 1934 | const SITargetLowering *TLI, |
| 1935 | const GCNTTIImpl *TTIImpl) { |
| 1936 | const int NrOfSGPRUntilSpill = 26; |
| 1937 | const int NrOfVGPRUntilSpill = 32; |
| 1938 | |
| 1939 | const DataLayout &DL = TTIImpl->getDataLayout(); |
| 1940 | |
| 1941 | unsigned adjustThreshold = 0; |
| 1942 | int SGPRsInUse = 0; |
| 1943 | int VGPRsInUse = 0; |
| 1944 | for (const Use &A : CB->args()) { |
| 1945 | SmallVector<EVT, 4> ValueVTs; |
| 1946 | ComputeValueVTs(TLI: *TLI, DL, Ty: A.get()->getType(), ValueVTs); |
| 1947 | for (auto ArgVT : ValueVTs) { |
| 1948 | unsigned CCRegNum = TLI->getNumRegistersForCallingConv( |
| 1949 | Context&: CB->getContext(), CC: CB->getCallingConv(), VT: ArgVT); |
| 1950 | if (AMDGPU::isArgPassedInSGPR(CB, ArgNo: CB->getArgOperandNo(U: &A))) |
| 1951 | SGPRsInUse += CCRegNum; |
| 1952 | else |
| 1953 | VGPRsInUse += CCRegNum; |
| 1954 | } |
| 1955 | } |
| 1956 | |
| 1957 | // The cost of passing function arguments through the stack: |
| 1958 | // 1 instruction to put a function argument on the stack in the caller. |
| 1959 | // 1 instruction to take a function argument from the stack in callee. |
| 1960 | // 1 instruction is explicitly take care of data dependencies in callee |
| 1961 | // function. |
| 1962 | InstructionCost ArgStackCost(1); |
| 1963 | ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost( |
| 1964 | Opcode: Instruction::Store, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4), |
| 1965 | AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency); |
| 1966 | ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost( |
| 1967 | Opcode: Instruction::Load, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4), |
| 1968 | AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency); |
| 1969 | |
| 1970 | // The penalty cost is computed relative to the cost of instructions and does |
| 1971 | // not model any storage costs. |
| 1972 | int InstrCost = InlineConstants::getInstrCost(C: CB->getContext()); |
| 1973 | adjustThreshold += |
| 1974 | static_cast<unsigned>(std::max(a: 0, b: SGPRsInUse - NrOfSGPRUntilSpill) * |
| 1975 | ArgStackCost.getValue() * InstrCost); |
| 1976 | adjustThreshold += |
| 1977 | static_cast<unsigned>(std::max(a: 0, b: VGPRsInUse - NrOfVGPRUntilSpill) * |
| 1978 | ArgStackCost.getValue() * InstrCost); |
| 1979 | return adjustThreshold; |
| 1980 | } |
| 1981 | |
| 1982 | static TypeSize getCallArgsTotalAllocaSize(const CallBase *CB, |
| 1983 | const DataLayout &DL) { |
| 1984 | // If we have a pointer to a private array passed into a function |
| 1985 | // it will not be optimized out, leaving scratch usage. |
| 1986 | // This function calculates the total size in bytes of the memory that would |
| 1987 | // end in scratch if the call was not inlined. |
| 1988 | TypeSize AllocaSize = TypeSize::getFixed(ExactSize: 0); |
| 1989 | SmallPtrSet<const AllocaInst *, 8> AIVisited; |
| 1990 | for (Value *PtrArg : CB->args()) { |
| 1991 | PointerType *Ty = dyn_cast<PointerType>(Val: PtrArg->getType()); |
| 1992 | if (!Ty) |
| 1993 | continue; |
| 1994 | |
| 1995 | unsigned AddrSpace = Ty->getAddressSpace(); |
| 1996 | if (AddrSpace != AMDGPUAS::FLAT_ADDRESS && |
| 1997 | AddrSpace != AMDGPUAS::PRIVATE_ADDRESS) |
| 1998 | continue; |
| 1999 | |
| 2000 | const AllocaInst *AI = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: PtrArg)); |
| 2001 | if (!AI || !AI->isStaticAlloca() || !AIVisited.insert(Ptr: AI).second) |
| 2002 | continue; |
| 2003 | |
| 2004 | if (auto Size = AI->getAllocationSize(DL)) |
| 2005 | AllocaSize += *Size; |
| 2006 | } |
| 2007 | return AllocaSize; |
| 2008 | } |
| 2009 | |
| 2010 | int GCNTTIImpl::getInliningLastCallToStaticBonus() const { |
| 2011 | return BaseT::getInliningLastCallToStaticBonus() * |
| 2012 | getInliningThresholdMultiplier(); |
| 2013 | } |
| 2014 | |
| 2015 | unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const { |
| 2016 | unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, TTIImpl: this); |
| 2017 | |
| 2018 | // Private object passed as arguments may end up in scratch usage if the call |
| 2019 | // is not inlined. Increase the inline threshold to promote inlining. |
| 2020 | TypeSize AllocaSize = getCallArgsTotalAllocaSize(CB, DL); |
| 2021 | if (AllocaSize.getFixedValue() > 0) |
| 2022 | Threshold += CLOpts.inline_arg_alloca_cost; |
| 2023 | return Threshold; |
| 2024 | } |
| 2025 | |
| 2026 | unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB, |
| 2027 | const AllocaInst *AI) const { |
| 2028 | |
| 2029 | // Below the cutoff, assume that the private memory objects would be |
| 2030 | // optimized |
| 2031 | auto AllocaSize = getCallArgsTotalAllocaSize(CB, DL).getFixedValue(); |
| 2032 | if (AllocaSize <= CLOpts.inline_arg_alloca_cutoff) |
| 2033 | return 0; |
| 2034 | |
| 2035 | // Above the cutoff, we give a cost to each private memory object |
| 2036 | // depending its size. If the array can be optimized by SROA this cost is not |
| 2037 | // added to the total-cost in the inliner cost analysis. |
| 2038 | // |
| 2039 | // We choose the total cost of the alloca such that their sum cancels the |
| 2040 | // bonus given in the threshold (ArgAllocaCost). |
| 2041 | // |
| 2042 | // Cost_Alloca_0 + ... + Cost_Alloca_N == ArgAllocaCost |
| 2043 | // |
| 2044 | // Awkwardly, the ArgAllocaCost bonus is multiplied by threshold-multiplier, |
| 2045 | // the single-bb bonus and the vector-bonus. |
| 2046 | // |
| 2047 | // We compensate the first two multipliers, by repeating logic from the |
| 2048 | // inliner-cost in here. The vector-bonus is 0 on AMDGPU. |
| 2049 | static_assert(InlinerVectorBonusPercent == 0, "vector bonus assumed to be 0" ); |
| 2050 | unsigned Threshold = |
| 2051 | CLOpts.inline_arg_alloca_cost * getInliningThresholdMultiplier(); |
| 2052 | |
| 2053 | bool SingleBB = none_of(Range&: *CB->getCalledFunction(), P: [](const BasicBlock &BB) { |
| 2054 | return BB.getTerminator()->getNumSuccessors() > 1; |
| 2055 | }); |
| 2056 | if (SingleBB) { |
| 2057 | Threshold += Threshold / 2; |
| 2058 | } |
| 2059 | |
| 2060 | auto ArgAllocaSize = AI->getAllocationSize(DL); |
| 2061 | if (!ArgAllocaSize) |
| 2062 | return 0; |
| 2063 | |
| 2064 | // Attribute the bonus proportionally to the alloca size |
| 2065 | unsigned AllocaThresholdBonus = static_cast<unsigned>( |
| 2066 | (Threshold * ArgAllocaSize->getFixedValue()) / AllocaSize); |
| 2067 | |
| 2068 | return AllocaThresholdBonus; |
| 2069 | } |
| 2070 | |
| 2071 | void GCNTTIImpl::(Loop *L, ScalarEvolution &SE, |
| 2072 | TTI::UnrollingPreferences &UP, |
| 2073 | OptimizationRemarkEmitter *ORE) const { |
| 2074 | CommonTTI.getUnrollingPreferences(L, SE, UP, ORE); |
| 2075 | } |
| 2076 | |
| 2077 | void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, |
| 2078 | TTI::PeelingPreferences &PP) const { |
| 2079 | CommonTTI.getPeelingPreferences(L, SE, PP); |
| 2080 | } |
| 2081 | |
| 2082 | int GCNTTIImpl::getTransInstrCost(TTI::TargetCostKind CostKind) const { |
| 2083 | return getQuarterRateInstrCost(CostKind); |
| 2084 | } |
| 2085 | |
| 2086 | int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const { |
| 2087 | return ST->hasFullRate64Ops() |
| 2088 | ? getFullRateInstrCost() |
| 2089 | : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind) |
| 2090 | : getQuarterRateInstrCost(CostKind); |
| 2091 | } |
| 2092 | |
| 2093 | std::pair<InstructionCost, MVT> |
| 2094 | GCNTTIImpl::getTypeLegalizationCost(Type *Ty) const { |
| 2095 | std::pair<InstructionCost, MVT> Cost = BaseT::getTypeLegalizationCost(Ty); |
| 2096 | auto Size = DL.getTypeSizeInBits(Ty); |
| 2097 | // Maximum load or store can handle 8 dwords for scalar and 4 for |
| 2098 | // vector ALU. Let's assume anything above 8 dwords is expensive |
| 2099 | // even if legal. |
| 2100 | if (Size <= 256) |
| 2101 | return Cost; |
| 2102 | |
| 2103 | Cost.first += (Size + 255) / 256; |
| 2104 | return Cost; |
| 2105 | } |
| 2106 | |
| 2107 | unsigned GCNTTIImpl::getCacheLineSize() const { |
| 2108 | if (ST->hasVmemPrefInsts() || ST->hasSmemPrefetchInsts()) |
| 2109 | return ST->getDataCacheLineSize(); |
| 2110 | return 0; |
| 2111 | } |
| 2112 | |
| 2113 | unsigned GCNTTIImpl::getPrefetchDistance() const { |
| 2114 | return ST->hasPrefetch() ? 128 : 0; |
| 2115 | } |
| 2116 | |
| 2117 | bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const { |
| 2118 | return AMDGPU::isFlatGlobalAddrSpace(AS); |
| 2119 | } |
| 2120 | |
| 2121 | void GCNTTIImpl::collectKernelLaunchBounds( |
| 2122 | const Function &F, |
| 2123 | SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const { |
| 2124 | SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F); |
| 2125 | LB.push_back(Elt: {"amdgpu-max-num-workgroups[0]" , MaxNumWorkgroups[0]}); |
| 2126 | LB.push_back(Elt: {"amdgpu-max-num-workgroups[1]" , MaxNumWorkgroups[1]}); |
| 2127 | LB.push_back(Elt: {"amdgpu-max-num-workgroups[2]" , MaxNumWorkgroups[2]}); |
| 2128 | std::pair<unsigned, unsigned> FlatWorkGroupSize = |
| 2129 | ST->getFlatWorkGroupSizes(F); |
| 2130 | LB.push_back(Elt: {"amdgpu-flat-work-group-size[0]" , FlatWorkGroupSize.first}); |
| 2131 | LB.push_back(Elt: {"amdgpu-flat-work-group-size[1]" , FlatWorkGroupSize.second}); |
| 2132 | std::pair<unsigned, unsigned> WavesPerEU = ST->getWavesPerEU(F); |
| 2133 | LB.push_back(Elt: {"amdgpu-waves-per-eu[0]" , WavesPerEU.first}); |
| 2134 | LB.push_back(Elt: {"amdgpu-waves-per-eu[1]" , WavesPerEU.second}); |
| 2135 | } |
| 2136 | |
| 2137 | GCNTTIImpl::KnownIEEEMode |
| 2138 | GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const { |
| 2139 | if (!ST->hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) |
| 2140 | return KnownIEEEMode::On; // Only mode on gfx1170+ |
| 2141 | |
| 2142 | const Function *F = I.getFunction(); |
| 2143 | if (!F) |
| 2144 | return KnownIEEEMode::Unknown; |
| 2145 | |
| 2146 | Attribute IEEEAttr = F->getFnAttribute(Kind: "amdgpu-ieee" ); |
| 2147 | if (IEEEAttr.isValid()) |
| 2148 | return IEEEAttr.getValueAsBool() ? KnownIEEEMode::On : KnownIEEEMode::Off; |
| 2149 | |
| 2150 | return AMDGPU::isShader(CC: F->getCallingConv()) ? KnownIEEEMode::Off |
| 2151 | : KnownIEEEMode::On; |
| 2152 | } |
| 2153 | |
| 2154 | static unsigned getNumMemoryInsts(unsigned Bytes, unsigned MaxBytes, |
| 2155 | bool HasDwordx3) { |
| 2156 | unsigned NumInsts = Bytes / MaxBytes; |
| 2157 | unsigned Rem = Bytes % MaxBytes; |
| 2158 | if (HasDwordx3 && Rem % 16 >= 12) { |
| 2159 | ++NumInsts; |
| 2160 | Rem -= 12; |
| 2161 | } |
| 2162 | return NumInsts + llvm::popcount(Value: Rem); |
| 2163 | } |
| 2164 | |
| 2165 | static bool hasUniformPointerOperand(const Instruction *I) { |
| 2166 | const auto *LI = dyn_cast_or_null<LoadInst>(Val: I); |
| 2167 | if (!LI) |
| 2168 | return false; |
| 2169 | const Value *Ptr = LI->getPointerOperand(); |
| 2170 | const DataLayout &DL = LI->getDataLayout(); |
| 2171 | APInt Offset(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0); |
| 2172 | Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset, |
| 2173 | /*AllowNonInbounds=*/true); |
| 2174 | return AMDGPU::isTriviallyUniform(V: Ptr, CtxI: LI); |
| 2175 | } |
| 2176 | |
| 2177 | static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST, |
| 2178 | const DataLayout &DL, |
| 2179 | const Instruction *I, |
| 2180 | unsigned Bytes) { |
| 2181 | const Value *Ptr = I ? getLoadStorePointerOperand(V: I) : nullptr; |
| 2182 | if (!Ptr || |
| 2183 | Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER) |
| 2184 | return {false, false}; |
| 2185 | APInt Offset(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0); |
| 2186 | bool KnownOffset = true; |
| 2187 | while (const auto *GEP = dyn_cast<GEPOperator>(Val: Ptr)) { |
| 2188 | KnownOffset = KnownOffset && GEP->accumulateConstantOffset(DL, Offset); |
| 2189 | Ptr = GEP->getPointerOperand(); |
| 2190 | } |
| 2191 | const auto *Cast = dyn_cast<AddrSpaceCastInst>(Val: Ptr); |
| 2192 | if (Cast) { |
| 2193 | if (Cast->getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE) |
| 2194 | return {false, false}; |
| 2195 | Ptr = Cast->getPointerOperand(); |
| 2196 | } |
| 2197 | const auto *Rsrc = dyn_cast<IntrinsicInst>(Val: Ptr); |
| 2198 | if (Rsrc && Rsrc->getIntrinsicID() != Intrinsic::amdgcn_make_buffer_rsrc) |
| 2199 | Rsrc = nullptr; |
| 2200 | if (!Cast && !Rsrc) |
| 2201 | return {false, false}; |
| 2202 | |
| 2203 | bool NoWrapFromMax = |
| 2204 | KnownOffset && |
| 2205 | Offset.ule(RHS: APInt::getAllOnes(numBits: Offset.getBitWidth()) - Bytes); |
| 2206 | const auto *NumRecords = |
| 2207 | Rsrc ? dyn_cast<ConstantInt>(Val: Rsrc->getArgOperand(i: 2)) : nullptr; |
| 2208 | std::optional<unsigned> NumRecordsWidth = |
| 2209 | ST.getBufferResourceNumRecordsWidth(); |
| 2210 | if (!NumRecords || !NumRecordsWidth) |
| 2211 | return {NoWrapFromMax, false}; |
| 2212 | APInt Bound = NumRecords->getValue().zextOrTrunc(width: *NumRecordsWidth); |
| 2213 | if (Bound.isAllOnes()) |
| 2214 | return {NoWrapFromMax, true}; |
| 2215 | if (!KnownOffset) |
| 2216 | return {false, false}; |
| 2217 | APInt BoundsDiff = Bound.zext(width: 64) - Offset.zext(width: 64); |
| 2218 | return {NoWrapFromMax, BoundsDiff.sge(RHS: Bytes) || BoundsDiff.isNonPositive()}; |
| 2219 | } |
| 2220 | |
| 2221 | InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, |
| 2222 | Align Alignment, |
| 2223 | unsigned AddressSpace, |
| 2224 | TTI::TargetCostKind CostKind, |
| 2225 | TTI::OperandValueInfo OpInfo, |
| 2226 | const Instruction *I) const { |
| 2227 | Type *EltTy = Src->getScalarType(); |
| 2228 | if ((Opcode != Instruction::Load && Opcode != Instruction::Store) || |
| 2229 | CostKind == TTI::TCK_Latency || isa<ScalableVectorType>(Val: Src) || |
| 2230 | (!EltTy->isIntOrPtrTy() && !EltTy->isFloatingPointTy())) |
| 2231 | return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, |
| 2232 | CostKind, OpInfo, I); |
| 2233 | |
| 2234 | unsigned EltBits = std::max<unsigned>(a: 8, b: DL.getTypeSizeInBits(Ty: EltTy)); |
| 2235 | |
| 2236 | auto *VecTy = dyn_cast<FixedVectorType>(Val: Src); |
| 2237 | if (VecTy && AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER && |
| 2238 | EltBits % 8 == 0 && !isPowerOf2_32(Value: EltBits) && |
| 2239 | !TLI->shouldCombineMemoryType(VT: TLI->getValueType(DL, Ty: Src))) { |
| 2240 | Align EltAlign = commonAlignment(A: Alignment, Offset: EltBits / 8); |
| 2241 | return VecTy->getNumElements() * getMemoryOpCost(Opcode, Src: EltTy, Alignment: EltAlign, |
| 2242 | AddressSpace, CostKind, |
| 2243 | OpInfo); |
| 2244 | } |
| 2245 | |
| 2246 | unsigned Bytes = DL.getTypeStoreSize(Ty: Src); |
| 2247 | if (Opcode == Instruction::Load && |
| 2248 | (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS || |
| 2249 | AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && |
| 2250 | Alignment >= std::min(a: Bytes, b: 4u) && |
| 2251 | (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT || |
| 2252 | hasUniformPointerOperand(I))) { |
| 2253 | unsigned Dwords = divideCeil(Numerator: Bytes, Denominator: 4); |
| 2254 | unsigned MaxBytes = getLoadStoreVecRegBitWidth(AddrSpace: AddressSpace) / 8; |
| 2255 | bool HasDwordx3 = ST->hasScalarDwordx3Loads(); |
| 2256 | // The last byte of a 7-byte remainder is not widened into the scalar load. |
| 2257 | if (!VecTy && Bytes % 4) |
| 2258 | return getNumMemoryInsts(Bytes: Dwords * 4, MaxBytes: 32, HasDwordx3) + (Bytes % 8 == 7); |
| 2259 | if (!HasDwordx3 && Alignment >= 8 && Dwords % 4 == 3) |
| 2260 | ++Dwords; |
| 2261 | return getNumMemoryInsts(Bytes: Dwords * 4, MaxBytes, HasDwordx3); |
| 2262 | } |
| 2263 | |
| 2264 | unsigned MaxBytes = |
| 2265 | std::min(a: 16u, b: getLoadStoreVecRegBitWidth(AddrSpace: AddressSpace) / 8); |
| 2266 | if (AddressSpace == AMDGPUAS::FLAT_ADDRESS && |
| 2267 | !ST->hasMultiDwordFlatScratchAddressing()) |
| 2268 | MaxBytes = getLoadStoreVecRegBitWidth(AddrSpace: AMDGPUAS::PRIVATE_ADDRESS) / 8; |
| 2269 | if (AddressSpace == AMDGPUAS::FLAT_ADDRESS && |
| 2270 | ST->hasLDSMisalignedBugInWGPMode()) |
| 2271 | MaxBytes = std::clamp<uint64_t>(val: Alignment.value(), lo: 4, hi: MaxBytes); |
| 2272 | bool IsDS = AddressSpace == AMDGPUAS::LOCAL_ADDRESS || |
| 2273 | AddressSpace == AMDGPUAS::REGION_ADDRESS; |
| 2274 | bool HasDwordx3 = ST->hasDwordx3LoadStores(); |
| 2275 | bool HasUnalignedAccess = ST->hasUnalignedBufferAccessEnabled(); |
| 2276 | if (AddressSpace == AMDGPUAS::PRIVATE_ADDRESS || |
| 2277 | AddressSpace == AMDGPUAS::FLAT_ADDRESS) |
| 2278 | HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled(); |
| 2279 | else if (IsDS) { |
| 2280 | HasUnalignedAccess = ST->hasUnalignedDSAccessEnabled(); |
| 2281 | if ((Alignment >= 4 || ST->hasLDSMisalignedBugInWGPMode()) && Alignment < 8) |
| 2282 | MaxBytes = 8; |
| 2283 | HasDwordx3 = HasDwordx3 && |
| 2284 | (Alignment >= 16 || (Alignment < 4 && HasUnalignedAccess)); |
| 2285 | } |
| 2286 | if (AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) { |
| 2287 | auto [NoWrapFromMax, NoPartialOOB] = |
| 2288 | getKnownBufferOobProperties(ST: *ST, DL, I, Bytes); |
| 2289 | MaxBytes = ST->getMaxBufferIntrinsicWidth(DL, T: Src, A: Alignment, NoWrapFromMax, |
| 2290 | NoPartialOOB) / |
| 2291 | 8; |
| 2292 | } |
| 2293 | if (Alignment < 4 && !HasUnalignedAccess) |
| 2294 | MaxBytes = std::min<unsigned>(a: MaxBytes, b: Alignment.value()); |
| 2295 | bool MergesDwords = AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER || |
| 2296 | (ST->hasFlatInstOffsets() && |
| 2297 | (AMDGPU::isExtendedGlobalAddrSpace(AS: AddressSpace) || |
| 2298 | (AddressSpace == AMDGPUAS::FLAT_ADDRESS && |
| 2299 | !ST->hasFlatSegmentOffsetBug()))); |
| 2300 | |
| 2301 | if (Opcode == Instruction::Load && |
| 2302 | AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER) { |
| 2303 | if (VecTy && Alignment >= PowerOf2Ceil(A: Bytes)) |
| 2304 | return divideCeil(Numerator: Bytes, Denominator: MaxBytes); |
| 2305 | if (EltBits <= 16 && Alignment >= 4 && Bytes % 4) { |
| 2306 | unsigned Piece = std::min(a: EltBits == 8 ? 8u : 16u, b: MaxBytes); |
| 2307 | unsigned Granule = std::min<unsigned>(a: Alignment.value(), b: Piece); |
| 2308 | if (!IsDS && !MergesDwords) |
| 2309 | return Bytes / Piece + llvm::popcount(Value: Bytes % Piece / Granule) + |
| 2310 | (Bytes % Granule != 0); |
| 2311 | unsigned Tail = Bytes % Granule; |
| 2312 | if (Tail > EltBits / 8) { |
| 2313 | Bytes += std::max<unsigned>(a: 4, b: PowerOf2Ceil(A: Tail)) - Tail; |
| 2314 | if (IsDS) |
| 2315 | HasDwordx3 = false; |
| 2316 | } |
| 2317 | } |
| 2318 | } |
| 2319 | if (!MergesDwords && Bytes % MaxBytes > 12) |
| 2320 | HasDwordx3 = false; |
| 2321 | return getNumMemoryInsts(Bytes, MaxBytes, HasDwordx3); |
| 2322 | } |
| 2323 | |
| 2324 | unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const { |
| 2325 | if (VectorType *VecTy = dyn_cast<VectorType>(Val: Tp)) { |
| 2326 | if (VecTy->getElementType()->isIntegerTy(BitWidth: 8)) { |
| 2327 | unsigned ElementCount = VecTy->getElementCount().getFixedValue(); |
| 2328 | return divideCeil(Numerator: ElementCount - 1, Denominator: 4); |
| 2329 | } |
| 2330 | } |
| 2331 | return BaseT::getNumberOfParts(Tp); |
| 2332 | } |
| 2333 | |
| 2334 | ValueUniformity GCNTTIImpl::getValueUniformity(const Value *V) const { |
| 2335 | if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) { |
| 2336 | switch (Intrinsic->getIntrinsicID()) { |
| 2337 | case Intrinsic::amdgcn_wave_shuffle: |
| 2338 | return ValueUniformity::Custom; |
| 2339 | default: |
| 2340 | break; |
| 2341 | } |
| 2342 | } |
| 2343 | |
| 2344 | if (isAlwaysUniform(V)) |
| 2345 | return ValueUniformity::AlwaysUniform; |
| 2346 | |
| 2347 | if (isSourceOfDivergence(V)) |
| 2348 | return ValueUniformity::NeverUniform; |
| 2349 | |
| 2350 | return ValueUniformity::Default; |
| 2351 | } |
| 2352 | |
| 2353 | InstructionCost GCNTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, |
| 2354 | StackOffset BaseOffset, |
| 2355 | bool HasBaseReg, int64_t Scale, |
| 2356 | unsigned AddrSpace) const { |
| 2357 | if (HasBaseReg && Scale != 0) { |
| 2358 | // gfx1250+ can fold base+scale*index when scale matches the memory access |
| 2359 | // size (scale_offset bit). Supported for flat/global/constant/scratch |
| 2360 | // (VMEM, max 128 bits) and constant_32bit (SMRD, capped to 128 bits here). |
| 2361 | if (getST()->hasScaleOffset() && Ty && Ty->isSized() && |
| 2362 | (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace) || |
| 2363 | AddrSpace == AMDGPUAS::FLAT_ADDRESS || |
| 2364 | AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)) { |
| 2365 | TypeSize StoreSize = getDataLayout().getTypeStoreSize(Ty); |
| 2366 | if (TypeSize::isKnownLE(LHS: StoreSize, RHS: TypeSize::getFixed(ExactSize: 16)) && |
| 2367 | static_cast<int64_t>(StoreSize.getFixedValue()) == Scale) |
| 2368 | return 0; |
| 2369 | } |
| 2370 | return 1; |
| 2371 | } |
| 2372 | return BaseT::getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale, |
| 2373 | AddrSpace); |
| 2374 | } |
| 2375 | |
| 2376 | bool GCNTTIImpl::isLSRCostLess(const TTI::LSRCost &A, |
| 2377 | const TTI::LSRCost &B) const { |
| 2378 | // Favor lower per-iteration work over preheader/setup costs. |
| 2379 | // AMDGPU lacks rich addressing modes, so ScaleCost is folded into the |
| 2380 | // effective instruction count (base+scale*index requires a separate ADD). |
| 2381 | unsigned EffInsnsA = A.Insns + A.ScaleCost; |
| 2382 | unsigned EffInsnsB = B.Insns + B.ScaleCost; |
| 2383 | |
| 2384 | return std::tie(args&: EffInsnsA, args: A.NumIVMuls, args: A.AddRecCost, args: A.NumBaseAdds, |
| 2385 | args: A.SetupCost, args: A.ImmCost, args: A.NumRegs) < |
| 2386 | std::tie(args&: EffInsnsB, args: B.NumIVMuls, args: B.AddRecCost, args: B.NumBaseAdds, |
| 2387 | args: B.SetupCost, args: B.ImmCost, args: B.NumRegs); |
| 2388 | } |
| 2389 | |
| 2390 | bool GCNTTIImpl::isNumRegsMajorCostOfLSR() const { |
| 2391 | // isLSRCostLess de-prioritizes register count; keep consistent. |
| 2392 | return false; |
| 2393 | } |
| 2394 | |
| 2395 | bool GCNTTIImpl::shouldDropLSRSolutionIfLessProfitable() const { |
| 2396 | // Prefer the baseline when LSR cannot clearly reduce per-iteration work. |
| 2397 | return true; |
| 2398 | } |
| 2399 | |
| 2400 | bool GCNTTIImpl::isUniform(const Instruction *I, |
| 2401 | const SmallBitVector &UniformArgs) const { |
| 2402 | const IntrinsicInst *Intrinsic = cast<IntrinsicInst>(Val: I); |
| 2403 | switch (Intrinsic->getIntrinsicID()) { |
| 2404 | case Intrinsic::amdgcn_wave_shuffle: |
| 2405 | // wave_shuffle(Value, Index): result is uniform when either Value or Index |
| 2406 | // is uniform. |
| 2407 | return UniformArgs[0] || UniformArgs[1]; |
| 2408 | default: |
| 2409 | llvm_unreachable("unexpected intrinsic in isUniform" ); |
| 2410 | } |
| 2411 | } |
| 2412 | |