| 1 | //===- llvm/CodeGen/GlobalISel/RegisterBankInfo.cpp --------------*- C++ -*-==// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | /// \file |
| 9 | /// This file implements the RegisterBankInfo class. |
| 10 | //===----------------------------------------------------------------------===// |
| 11 | |
| 12 | #include "llvm/CodeGen/RegisterBankInfo.h" |
| 13 | #include "llvm/ADT/APInt.h" |
| 14 | #include "llvm/ADT/SmallVector.h" |
| 15 | #include "llvm/ADT/Statistic.h" |
| 16 | #include "llvm/ADT/iterator_range.h" |
| 17 | #include "llvm/CodeGen/MachineFunction.h" |
| 18 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 19 | #include "llvm/CodeGen/RegisterBank.h" |
| 20 | #include "llvm/CodeGen/TargetOpcodes.h" |
| 21 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
| 22 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
| 23 | #include "llvm/Config/llvm-config.h" |
| 24 | #include "llvm/Support/Debug.h" |
| 25 | #include "llvm/Support/raw_ostream.h" |
| 26 | |
| 27 | #include <algorithm> // For std::max. |
| 28 | |
| 29 | #define DEBUG_TYPE "registerbankinfo" |
| 30 | |
| 31 | using namespace llvm; |
| 32 | |
| 33 | STATISTIC(NumPartialMappingsCreated, |
| 34 | "Number of partial mappings dynamically created" ); |
| 35 | STATISTIC(NumPartialMappingsAccessed, |
| 36 | "Number of partial mappings dynamically accessed" ); |
| 37 | STATISTIC(NumValueMappingsCreated, |
| 38 | "Number of value mappings dynamically created" ); |
| 39 | STATISTIC(NumValueMappingsAccessed, |
| 40 | "Number of value mappings dynamically accessed" ); |
| 41 | STATISTIC(NumOperandsMappingsCreated, |
| 42 | "Number of operands mappings dynamically created" ); |
| 43 | STATISTIC(NumOperandsMappingsAccessed, |
| 44 | "Number of operands mappings dynamically accessed" ); |
| 45 | STATISTIC(NumInstructionMappingsCreated, |
| 46 | "Number of instruction mappings dynamically created" ); |
| 47 | STATISTIC(NumInstructionMappingsAccessed, |
| 48 | "Number of instruction mappings dynamically accessed" ); |
| 49 | |
| 50 | const unsigned RegisterBankInfo::DefaultMappingID = UINT_MAX; |
| 51 | const unsigned RegisterBankInfo::InvalidMappingID = UINT_MAX - 1; |
| 52 | |
| 53 | //------------------------------------------------------------------------------ |
| 54 | // RegisterBankInfo implementation. |
| 55 | //------------------------------------------------------------------------------ |
| 56 | RegisterBankInfo::RegisterBankInfo(const RegisterBank **RegBanks, |
| 57 | unsigned NumRegBanks, const unsigned *Sizes, |
| 58 | unsigned HwMode) |
| 59 | : RegBanks(RegBanks), NumRegBanks(NumRegBanks), Sizes(Sizes), |
| 60 | HwMode(HwMode) { |
| 61 | #ifndef NDEBUG |
| 62 | for (unsigned Idx = 0, End = getNumRegBanks(); Idx != End; ++Idx) { |
| 63 | assert(RegBanks[Idx] != nullptr && "Invalid RegisterBank" ); |
| 64 | assert(RegBanks[Idx]->getID() == Idx && |
| 65 | "RegisterBank ID should match index" ); |
| 66 | } |
| 67 | #endif // NDEBUG |
| 68 | } |
| 69 | |
| 70 | bool RegisterBankInfo::verify(const TargetRegisterInfo &TRI) const { |
| 71 | #ifndef NDEBUG |
| 72 | for (unsigned Idx = 0, End = getNumRegBanks(); Idx != End; ++Idx) { |
| 73 | const RegisterBank &RegBank = getRegBank(Idx); |
| 74 | assert(Idx == RegBank.getID() && |
| 75 | "ID does not match the index in the array" ); |
| 76 | LLVM_DEBUG(dbgs() << "Verify " << RegBank << '\n'); |
| 77 | assert(RegBank.verify(*this, TRI) && "RegBank is invalid" ); |
| 78 | } |
| 79 | #endif // NDEBUG |
| 80 | return true; |
| 81 | } |
| 82 | |
| 83 | const RegisterBank * |
| 84 | RegisterBankInfo::getRegBank(Register Reg, const MachineRegisterInfo &MRI, |
| 85 | const TargetRegisterInfo &TRI) const { |
| 86 | if (!Reg.isVirtual()) { |
| 87 | // FIXME: This was probably a copy to a virtual register that does have a |
| 88 | // type we could use. |
| 89 | const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg); |
| 90 | return RC ? &getRegBankFromRegClass(RC: *RC, Ty: LLT()) : nullptr; |
| 91 | } |
| 92 | |
| 93 | const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); |
| 94 | if (auto *RB = dyn_cast_if_present<const RegisterBank *>(Val: RegClassOrBank)) |
| 95 | return RB; |
| 96 | if (auto *RC = |
| 97 | dyn_cast_if_present<const TargetRegisterClass *>(Val: RegClassOrBank)) |
| 98 | return &getRegBankFromRegClass(RC: *RC, Ty: MRI.getType(Reg)); |
| 99 | return nullptr; |
| 100 | } |
| 101 | |
| 102 | const RegisterBank *RegisterBankInfo::getRegBankFromConstraints( |
| 103 | const MachineInstr &MI, unsigned OpIdx, const TargetInstrInfo &TII, |
| 104 | const MachineRegisterInfo &MRI) const { |
| 105 | const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo(); |
| 106 | |
| 107 | // The mapping of the registers may be available via the |
| 108 | // register class constraints. |
| 109 | const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx, TII: &TII, TRI); |
| 110 | |
| 111 | if (!RC) |
| 112 | return nullptr; |
| 113 | |
| 114 | Register Reg = MI.getOperand(i: OpIdx).getReg(); |
| 115 | const RegisterBank &RegBank = getRegBankFromRegClass(RC: *RC, Ty: MRI.getType(Reg)); |
| 116 | // Check that the target properly implemented getRegBankFromRegClass. |
| 117 | assert(RegBank.covers(*RC) && |
| 118 | "The mapping of the register bank does not make sense" ); |
| 119 | return &RegBank; |
| 120 | } |
| 121 | |
| 122 | const TargetRegisterClass *RegisterBankInfo::constrainGenericRegister( |
| 123 | Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI) { |
| 124 | |
| 125 | // If the register already has a class, fallback to MRI::constrainRegClass. |
| 126 | auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); |
| 127 | if (isa<const TargetRegisterClass *>(Val: RegClassOrBank)) |
| 128 | return MRI.constrainRegClass(Reg, RC: &RC); |
| 129 | |
| 130 | const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank); |
| 131 | // Otherwise, all we can do is ensure the bank covers the class, and set it. |
| 132 | if (RB && !RB->covers(RC)) |
| 133 | return nullptr; |
| 134 | |
| 135 | // If nothing was set or the class is simply compatible, set it. |
| 136 | MRI.setRegClass(Reg, RC: &RC); |
| 137 | return &RC; |
| 138 | } |
| 139 | |
| 140 | /// Check whether or not \p MI should be treated like a copy |
| 141 | /// for the mappings. |
| 142 | /// Copy like instruction are special for mapping because |
| 143 | /// they don't have actual register constraints. Moreover, |
| 144 | /// they sometimes have register classes assigned and we can |
| 145 | /// just use that instead of failing to provide a generic mapping. |
| 146 | static bool isCopyLike(const MachineInstr &MI) { |
| 147 | return MI.isCopy() || MI.isPHI() || |
| 148 | MI.getOpcode() == TargetOpcode::REG_SEQUENCE; |
| 149 | } |
| 150 | |
| 151 | const RegisterBankInfo::InstructionMapping & |
| 152 | RegisterBankInfo::getInstrMappingImpl(const MachineInstr &MI) const { |
| 153 | // For copies we want to walk over the operands and try to find one |
| 154 | // that has a register bank since the instruction itself will not get |
| 155 | // us any constraint. |
| 156 | bool IsCopyLike = isCopyLike(MI); |
| 157 | // For copy like instruction, only the mapping of the definition |
| 158 | // is important. The rest is not constrained. |
| 159 | unsigned NumOperandsForMapping = IsCopyLike ? 1 : MI.getNumOperands(); |
| 160 | |
| 161 | const MachineFunction &MF = *MI.getMF(); |
| 162 | const TargetSubtargetInfo &STI = MF.getSubtarget(); |
| 163 | const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); |
| 164 | const MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 165 | // We may need to query the instruction encoding to guess the mapping. |
| 166 | const TargetInstrInfo &TII = *STI.getInstrInfo(); |
| 167 | |
| 168 | // Before doing anything complicated check if the mapping is not |
| 169 | // directly available. |
| 170 | bool CompleteMapping = true; |
| 171 | |
| 172 | SmallVector<const ValueMapping *, 8> OperandsMapping(NumOperandsForMapping); |
| 173 | for (unsigned OpIdx = 0, EndIdx = MI.getNumOperands(); OpIdx != EndIdx; |
| 174 | ++OpIdx) { |
| 175 | const MachineOperand &MO = MI.getOperand(i: OpIdx); |
| 176 | if (!MO.isReg()) |
| 177 | continue; |
| 178 | Register Reg = MO.getReg(); |
| 179 | if (!Reg) |
| 180 | continue; |
| 181 | // The register bank of Reg is just a side effect of the current |
| 182 | // excution and in particular, there is no reason to believe this |
| 183 | // is the best default mapping for the current instruction. Keep |
| 184 | // it as an alternative register bank if we cannot figure out |
| 185 | // something. |
| 186 | const RegisterBank *AltRegBank = getRegBank(Reg, MRI, TRI); |
| 187 | // For copy-like instruction, we want to reuse the register bank |
| 188 | // that is already set on Reg, if any, since those instructions do |
| 189 | // not have any constraints. |
| 190 | const RegisterBank *CurRegBank = IsCopyLike ? AltRegBank : nullptr; |
| 191 | if (!CurRegBank) { |
| 192 | // If this is a target specific instruction, we can deduce |
| 193 | // the register bank from the encoding constraints. |
| 194 | CurRegBank = getRegBankFromConstraints(MI, OpIdx, TII, MRI); |
| 195 | if (!CurRegBank) { |
| 196 | // All our attempts failed, give up. |
| 197 | CompleteMapping = false; |
| 198 | |
| 199 | if (!IsCopyLike) |
| 200 | // MI does not carry enough information to guess the mapping. |
| 201 | return getInvalidInstructionMapping(); |
| 202 | continue; |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | TypeSize Size = TRI.getRegSizeInBits(Reg, MRI); |
| 207 | const ValueMapping *ValMapping = |
| 208 | &getValueMapping(StartIdx: 0, Length: Size.getKnownMinValue(), RegBank: *CurRegBank); |
| 209 | if (IsCopyLike) { |
| 210 | if (!OperandsMapping[0]) { |
| 211 | if (MI.isRegSequence()) { |
| 212 | // For reg_sequence, the result size does not match the input. |
| 213 | unsigned ResultSize = |
| 214 | TRI.getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI); |
| 215 | OperandsMapping[0] = &getValueMapping(StartIdx: 0, Length: ResultSize, RegBank: *CurRegBank); |
| 216 | } else { |
| 217 | OperandsMapping[0] = ValMapping; |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | // The default handling assumes any register bank can be copied to any |
| 222 | // other. If this isn't the case, the target should specially deal with |
| 223 | // reg_sequence/phi. There may also be unsatisfiable copies. |
| 224 | for (; OpIdx != EndIdx; ++OpIdx) { |
| 225 | const MachineOperand &MO = MI.getOperand(i: OpIdx); |
| 226 | if (!MO.isReg()) |
| 227 | continue; |
| 228 | Register Reg = MO.getReg(); |
| 229 | if (!Reg) |
| 230 | continue; |
| 231 | |
| 232 | const RegisterBank *AltRegBank = getRegBank(Reg, MRI, TRI); |
| 233 | if (AltRegBank && cannotCopy(Dst: *CurRegBank, Src: *AltRegBank, |
| 234 | Size: TRI.getRegSizeInBits(Reg, MRI))) |
| 235 | return getInvalidInstructionMapping(); |
| 236 | } |
| 237 | |
| 238 | CompleteMapping = true; |
| 239 | break; |
| 240 | } |
| 241 | |
| 242 | OperandsMapping[OpIdx] = ValMapping; |
| 243 | } |
| 244 | |
| 245 | if (IsCopyLike && !CompleteMapping) { |
| 246 | // No way to deduce the type from what we have. |
| 247 | return getInvalidInstructionMapping(); |
| 248 | } |
| 249 | |
| 250 | assert(CompleteMapping && "Setting an uncomplete mapping" ); |
| 251 | return getInstructionMapping( |
| 252 | ID: DefaultMappingID, /*Cost*/ 1, |
| 253 | /*OperandsMapping*/ getOperandsMapping(OpdsMapping: OperandsMapping), |
| 254 | NumOperands: NumOperandsForMapping); |
| 255 | } |
| 256 | |
| 257 | /// Hashing function for PartialMapping. |
| 258 | static hash_code hashPartialMapping(unsigned StartIdx, unsigned Length, |
| 259 | const RegisterBank *RegBank) { |
| 260 | return hash_combine(args: StartIdx, args: Length, args: RegBank ? RegBank->getID() : 0); |
| 261 | } |
| 262 | |
| 263 | /// Overloaded version of hash_value for a PartialMapping. |
| 264 | hash_code |
| 265 | llvm::hash_value(const RegisterBankInfo::PartialMapping &PartMapping) { |
| 266 | return hashPartialMapping(StartIdx: PartMapping.StartIdx, Length: PartMapping.Length, |
| 267 | RegBank: PartMapping.RegBank); |
| 268 | } |
| 269 | |
| 270 | const RegisterBankInfo::PartialMapping & |
| 271 | RegisterBankInfo::getPartialMapping(unsigned StartIdx, unsigned Length, |
| 272 | const RegisterBank &RegBank) const { |
| 273 | ++NumPartialMappingsAccessed; |
| 274 | |
| 275 | hash_code Hash = hashPartialMapping(StartIdx, Length, RegBank: &RegBank); |
| 276 | auto [It, Inserted] = MapOfPartialMappings.try_emplace(Key: Hash); |
| 277 | if (!Inserted) |
| 278 | return *It->second; |
| 279 | |
| 280 | ++NumPartialMappingsCreated; |
| 281 | |
| 282 | auto &PartMapping = It->second; |
| 283 | PartMapping = std::make_unique<PartialMapping>(args&: StartIdx, args&: Length, args: RegBank); |
| 284 | return *PartMapping; |
| 285 | } |
| 286 | |
| 287 | const RegisterBankInfo::ValueMapping & |
| 288 | RegisterBankInfo::getValueMapping(unsigned StartIdx, unsigned Length, |
| 289 | const RegisterBank &RegBank) const { |
| 290 | return getValueMapping(BreakDown: &getPartialMapping(StartIdx, Length, RegBank), NumBreakDowns: 1); |
| 291 | } |
| 292 | |
| 293 | static hash_code |
| 294 | hashValueMapping(const RegisterBankInfo::PartialMapping *BreakDown, |
| 295 | unsigned NumBreakDowns) { |
| 296 | if (LLVM_LIKELY(NumBreakDowns == 1)) |
| 297 | return hash_value(PartMapping: *BreakDown); |
| 298 | SmallVector<size_t, 8> Hashes(NumBreakDowns); |
| 299 | for (unsigned Idx = 0; Idx != NumBreakDowns; ++Idx) |
| 300 | Hashes.push_back(Elt: hash_value(PartMapping: BreakDown[Idx])); |
| 301 | return hash_combine_range(R&: Hashes); |
| 302 | } |
| 303 | |
| 304 | const RegisterBankInfo::ValueMapping & |
| 305 | RegisterBankInfo::getValueMapping(const PartialMapping *BreakDown, |
| 306 | unsigned NumBreakDowns) const { |
| 307 | ++NumValueMappingsAccessed; |
| 308 | |
| 309 | hash_code Hash = hashValueMapping(BreakDown, NumBreakDowns); |
| 310 | auto [It, Inserted] = MapOfValueMappings.try_emplace(Key: Hash); |
| 311 | if (!Inserted) |
| 312 | return *It->second; |
| 313 | |
| 314 | ++NumValueMappingsCreated; |
| 315 | |
| 316 | auto &ValMapping = It->second; |
| 317 | ValMapping = std::make_unique<ValueMapping>(args&: BreakDown, args&: NumBreakDowns); |
| 318 | return *ValMapping; |
| 319 | } |
| 320 | |
| 321 | template <typename Iterator> |
| 322 | const RegisterBankInfo::ValueMapping * |
| 323 | RegisterBankInfo::getOperandsMapping(Iterator Begin, Iterator End) const { |
| 324 | |
| 325 | ++NumOperandsMappingsAccessed; |
| 326 | |
| 327 | // The addresses of the value mapping are unique. |
| 328 | // Therefore, we can use them directly to hash the operand mapping. |
| 329 | hash_code Hash = hash_combine_range(Begin, End); |
| 330 | auto &Res = MapOfOperandsMappings[Hash]; |
| 331 | if (Res) |
| 332 | return Res.get(); |
| 333 | |
| 334 | ++NumOperandsMappingsCreated; |
| 335 | |
| 336 | // Create the array of ValueMapping. |
| 337 | // Note: this array will not hash to this instance of operands |
| 338 | // mapping, because we use the pointer of the ValueMapping |
| 339 | // to hash and we expect them to uniquely identify an instance |
| 340 | // of value mapping. |
| 341 | Res = std::make_unique<ValueMapping[]>(std::distance(Begin, End)); |
| 342 | unsigned Idx = 0; |
| 343 | for (Iterator It = Begin; It != End; ++It, ++Idx) { |
| 344 | const ValueMapping *ValMap = *It; |
| 345 | if (!ValMap) |
| 346 | continue; |
| 347 | Res[Idx] = *ValMap; |
| 348 | } |
| 349 | return Res.get(); |
| 350 | } |
| 351 | |
| 352 | const RegisterBankInfo::ValueMapping *RegisterBankInfo::getOperandsMapping( |
| 353 | const SmallVectorImpl<const RegisterBankInfo::ValueMapping *> &OpdsMapping) |
| 354 | const { |
| 355 | return getOperandsMapping(Begin: OpdsMapping.begin(), End: OpdsMapping.end()); |
| 356 | } |
| 357 | |
| 358 | const RegisterBankInfo::ValueMapping *RegisterBankInfo::getOperandsMapping( |
| 359 | std::initializer_list<const RegisterBankInfo::ValueMapping *> OpdsMapping) |
| 360 | const { |
| 361 | return getOperandsMapping(Begin: OpdsMapping.begin(), End: OpdsMapping.end()); |
| 362 | } |
| 363 | |
| 364 | static hash_code |
| 365 | hashInstructionMapping(unsigned ID, unsigned Cost, |
| 366 | const RegisterBankInfo::ValueMapping *OperandsMapping, |
| 367 | unsigned NumOperands) { |
| 368 | return hash_combine(args: ID, args: Cost, args: OperandsMapping, args: NumOperands); |
| 369 | } |
| 370 | |
| 371 | const RegisterBankInfo::InstructionMapping & |
| 372 | RegisterBankInfo::getInstructionMappingImpl( |
| 373 | bool IsInvalid, unsigned ID, unsigned Cost, |
| 374 | const RegisterBankInfo::ValueMapping *OperandsMapping, |
| 375 | unsigned NumOperands) const { |
| 376 | assert(((IsInvalid && ID == InvalidMappingID && Cost == 0 && |
| 377 | OperandsMapping == nullptr && NumOperands == 0) || |
| 378 | !IsInvalid) && |
| 379 | "Mismatch argument for invalid input" ); |
| 380 | ++NumInstructionMappingsAccessed; |
| 381 | |
| 382 | hash_code Hash = |
| 383 | hashInstructionMapping(ID, Cost, OperandsMapping, NumOperands); |
| 384 | auto [It, Inserted] = MapOfInstructionMappings.try_emplace(Key: Hash); |
| 385 | if (!Inserted) |
| 386 | return *It->second; |
| 387 | |
| 388 | ++NumInstructionMappingsCreated; |
| 389 | |
| 390 | auto &InstrMapping = It->second; |
| 391 | InstrMapping = std::make_unique<InstructionMapping>( |
| 392 | args&: ID, args&: Cost, args&: OperandsMapping, args&: NumOperands); |
| 393 | return *InstrMapping; |
| 394 | } |
| 395 | |
| 396 | const RegisterBankInfo::InstructionMapping & |
| 397 | RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { |
| 398 | const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI); |
| 399 | if (Mapping.isValid()) |
| 400 | return Mapping; |
| 401 | llvm_unreachable("The target must implement this" ); |
| 402 | } |
| 403 | |
| 404 | RegisterBankInfo::InstructionMappings |
| 405 | RegisterBankInfo::getInstrPossibleMappings(const MachineInstr &MI) const { |
| 406 | InstructionMappings PossibleMappings; |
| 407 | const auto &Mapping = getInstrMapping(MI); |
| 408 | if (Mapping.isValid()) { |
| 409 | // Put the default mapping first. |
| 410 | PossibleMappings.push_back(Elt: &Mapping); |
| 411 | } |
| 412 | |
| 413 | // Then the alternative mapping, if any. |
| 414 | InstructionMappings AltMappings = getInstrAlternativeMappings(MI); |
| 415 | append_range(C&: PossibleMappings, R&: AltMappings); |
| 416 | #ifndef NDEBUG |
| 417 | for (const InstructionMapping *Mapping : PossibleMappings) |
| 418 | assert(Mapping->verify(MI) && "Mapping is invalid" ); |
| 419 | #endif |
| 420 | return PossibleMappings; |
| 421 | } |
| 422 | |
| 423 | RegisterBankInfo::InstructionMappings |
| 424 | RegisterBankInfo::getInstrAlternativeMappings(const MachineInstr &MI) const { |
| 425 | // No alternative for MI. |
| 426 | return InstructionMappings(); |
| 427 | } |
| 428 | |
| 429 | void RegisterBankInfo::applyDefaultMapping(const OperandsMapper &OpdMapper) { |
| 430 | MachineInstr &MI = OpdMapper.getMI(); |
| 431 | MachineRegisterInfo &MRI = OpdMapper.getMRI(); |
| 432 | LLVM_DEBUG(dbgs() << "Applying default-like mapping\n" ); |
| 433 | for (unsigned OpIdx = 0, |
| 434 | EndIdx = OpdMapper.getInstrMapping().getNumOperands(); |
| 435 | OpIdx != EndIdx; ++OpIdx) { |
| 436 | LLVM_DEBUG(dbgs() << "OpIdx " << OpIdx); |
| 437 | MachineOperand &MO = MI.getOperand(i: OpIdx); |
| 438 | if (!MO.isReg()) { |
| 439 | LLVM_DEBUG(dbgs() << " is not a register, nothing to be done\n" ); |
| 440 | continue; |
| 441 | } |
| 442 | if (!MO.getReg()) { |
| 443 | LLVM_DEBUG(dbgs() << " is $noreg, nothing to be done\n" ); |
| 444 | continue; |
| 445 | } |
| 446 | LLT Ty = MRI.getType(Reg: MO.getReg()); |
| 447 | if (!Ty.isValid()) |
| 448 | continue; |
| 449 | assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns != |
| 450 | 0 && |
| 451 | "Invalid mapping" ); |
| 452 | assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns == |
| 453 | 1 && |
| 454 | "This mapping is too complex for this function" ); |
| 455 | iterator_range<SmallVectorImpl<Register>::const_iterator> NewRegs = |
| 456 | OpdMapper.getVRegs(OpIdx); |
| 457 | if (NewRegs.empty()) { |
| 458 | LLVM_DEBUG(dbgs() << " has not been repaired, nothing to be done\n" ); |
| 459 | continue; |
| 460 | } |
| 461 | Register OrigReg = MO.getReg(); |
| 462 | Register NewReg = *NewRegs.begin(); |
| 463 | LLVM_DEBUG(dbgs() << " changed, replace " << printReg(OrigReg, nullptr)); |
| 464 | MO.setReg(NewReg); |
| 465 | LLVM_DEBUG(dbgs() << " with " << printReg(NewReg, nullptr)); |
| 466 | |
| 467 | // The OperandsMapper creates plain scalar, we may have to fix that. |
| 468 | // Check if the types match and if not, fix that. |
| 469 | LLT OrigTy = MRI.getType(Reg: OrigReg); |
| 470 | LLT NewTy = MRI.getType(Reg: NewReg); |
| 471 | if (OrigTy != NewTy) { |
| 472 | // The default mapping is not supposed to change the size of |
| 473 | // the storage. However, right now we don't necessarily bump all |
| 474 | // the types to storage size. For instance, we can consider |
| 475 | // s16 G_AND legal whereas the storage size is going to be 32. |
| 476 | assert( |
| 477 | TypeSize::isKnownLE(OrigTy.getSizeInBits(), NewTy.getSizeInBits()) && |
| 478 | "Types with difference size cannot be handled by the default " |
| 479 | "mapping" ); |
| 480 | LLVM_DEBUG(dbgs() << "\nChange type of new opd from " << NewTy << " to " |
| 481 | << OrigTy); |
| 482 | MRI.setType(VReg: NewReg, Ty: OrigTy); |
| 483 | } |
| 484 | LLVM_DEBUG(dbgs() << '\n'); |
| 485 | } |
| 486 | } |
| 487 | |
| 488 | //------------------------------------------------------------------------------ |
| 489 | // Helper classes implementation. |
| 490 | //------------------------------------------------------------------------------ |
| 491 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| 492 | LLVM_DUMP_METHOD void RegisterBankInfo::PartialMapping::dump() const { |
| 493 | print(dbgs()); |
| 494 | dbgs() << '\n'; |
| 495 | } |
| 496 | #endif |
| 497 | |
| 498 | bool RegisterBankInfo::PartialMapping::verify( |
| 499 | const RegisterBankInfo &RBI) const { |
| 500 | assert(RegBank && "Register bank not set" ); |
| 501 | assert(Length && "Empty mapping" ); |
| 502 | assert((StartIdx <= getHighBitIdx()) && "Overflow, switch to APInt?" ); |
| 503 | // Check if the minimum width fits into RegBank. |
| 504 | assert(RBI.getMaximumSize(RegBank->getID()) >= Length && |
| 505 | "Register bank too small for Mask" ); |
| 506 | return true; |
| 507 | } |
| 508 | |
| 509 | void RegisterBankInfo::PartialMapping::print(raw_ostream &OS) const { |
| 510 | OS << "[" << StartIdx << ", " << getHighBitIdx() << "], RegBank = " ; |
| 511 | if (RegBank) |
| 512 | OS << *RegBank; |
| 513 | else |
| 514 | OS << "nullptr" ; |
| 515 | } |
| 516 | |
| 517 | bool RegisterBankInfo::ValueMapping::partsAllUniform() const { |
| 518 | if (NumBreakDowns < 2) |
| 519 | return true; |
| 520 | |
| 521 | const PartialMapping *First = begin(); |
| 522 | for (const PartialMapping *Part = First + 1; Part != end(); ++Part) { |
| 523 | if (Part->Length != First->Length || Part->RegBank != First->RegBank) |
| 524 | return false; |
| 525 | } |
| 526 | |
| 527 | return true; |
| 528 | } |
| 529 | |
| 530 | bool RegisterBankInfo::ValueMapping::verify(const RegisterBankInfo &RBI, |
| 531 | TypeSize MeaningfulBitWidth) const { |
| 532 | assert(NumBreakDowns && "Value mapped nowhere?!" ); |
| 533 | unsigned OrigValueBitWidth = 0; |
| 534 | for (const RegisterBankInfo::PartialMapping &PartMap : *this) { |
| 535 | // Check that each register bank is big enough to hold the partial value: |
| 536 | // this check is done by PartialMapping::verify |
| 537 | assert(PartMap.verify(RBI) && "Partial mapping is invalid" ); |
| 538 | // The original value should completely be mapped. |
| 539 | // Thus the maximum accessed index + 1 is the size of the original value. |
| 540 | OrigValueBitWidth = |
| 541 | std::max(a: OrigValueBitWidth, b: PartMap.getHighBitIdx() + 1); |
| 542 | } |
| 543 | assert((MeaningfulBitWidth.isScalable() || |
| 544 | OrigValueBitWidth >= MeaningfulBitWidth) && |
| 545 | "Meaningful bits not covered by the mapping" ); |
| 546 | APInt ValueMask(OrigValueBitWidth, 0); |
| 547 | for (const RegisterBankInfo::PartialMapping &PartMap : *this) { |
| 548 | // Check that the union of the partial mappings covers the whole value, |
| 549 | // without overlaps. |
| 550 | // The high bit is exclusive in the APInt API, thus getHighBitIdx + 1. |
| 551 | APInt PartMapMask = APInt::getBitsSet(numBits: OrigValueBitWidth, loBit: PartMap.StartIdx, |
| 552 | hiBit: PartMap.getHighBitIdx() + 1); |
| 553 | ValueMask ^= PartMapMask; |
| 554 | assert((ValueMask & PartMapMask) == PartMapMask && |
| 555 | "Some partial mappings overlap" ); |
| 556 | } |
| 557 | assert(ValueMask.isAllOnes() && "Value is not fully mapped" ); |
| 558 | return true; |
| 559 | } |
| 560 | |
| 561 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| 562 | LLVM_DUMP_METHOD void RegisterBankInfo::ValueMapping::dump() const { |
| 563 | print(dbgs()); |
| 564 | dbgs() << '\n'; |
| 565 | } |
| 566 | #endif |
| 567 | |
| 568 | void RegisterBankInfo::ValueMapping::print(raw_ostream &OS) const { |
| 569 | OS << "#BreakDown: " << NumBreakDowns << " " ; |
| 570 | bool IsFirst = true; |
| 571 | for (const PartialMapping &PartMap : *this) { |
| 572 | if (!IsFirst) |
| 573 | OS << ", " ; |
| 574 | OS << '[' << PartMap << ']'; |
| 575 | IsFirst = false; |
| 576 | } |
| 577 | } |
| 578 | |
| 579 | bool RegisterBankInfo::InstructionMapping::verify( |
| 580 | const MachineInstr &MI) const { |
| 581 | // Check that all the register operands are properly mapped. |
| 582 | // Check the constructor invariant. |
| 583 | // For PHI, we only care about mapping the definition. |
| 584 | assert(NumOperands == (isCopyLike(MI) ? 1 : MI.getNumOperands()) && |
| 585 | "NumOperands must match, see constructor" ); |
| 586 | assert(MI.getParent() && MI.getMF() && |
| 587 | "MI must be connected to a MachineFunction" ); |
| 588 | const MachineFunction &MF = *MI.getMF(); |
| 589 | const RegisterBankInfo *RBI = MF.getSubtarget().getRegBankInfo(); |
| 590 | (void)RBI; |
| 591 | const MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 592 | |
| 593 | for (unsigned Idx = 0; Idx < NumOperands; ++Idx) { |
| 594 | const MachineOperand &MO = MI.getOperand(i: Idx); |
| 595 | if (!MO.isReg()) { |
| 596 | assert(!getOperandMapping(Idx).isValid() && |
| 597 | "We should not care about non-reg mapping" ); |
| 598 | continue; |
| 599 | } |
| 600 | Register Reg = MO.getReg(); |
| 601 | if (!Reg) |
| 602 | continue; |
| 603 | LLT Ty = MRI.getType(Reg); |
| 604 | if (!Ty.isValid()) |
| 605 | continue; |
| 606 | assert(getOperandMapping(Idx).isValid() && |
| 607 | "We must have a mapping for reg operands" ); |
| 608 | const RegisterBankInfo::ValueMapping &MOMapping = getOperandMapping(i: Idx); |
| 609 | (void)MOMapping; |
| 610 | // Register size in bits. |
| 611 | // This size must match what the mapping expects. |
| 612 | assert(MOMapping.verify( |
| 613 | *RBI, MF.getSubtarget().getRegisterInfo()->getRegSizeInBits( |
| 614 | Reg, MRI)) && |
| 615 | "Value mapping is invalid" ); |
| 616 | } |
| 617 | return true; |
| 618 | } |
| 619 | |
| 620 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| 621 | LLVM_DUMP_METHOD void RegisterBankInfo::InstructionMapping::dump() const { |
| 622 | print(dbgs()); |
| 623 | dbgs() << '\n'; |
| 624 | } |
| 625 | #endif |
| 626 | |
| 627 | void RegisterBankInfo::InstructionMapping::print(raw_ostream &OS) const { |
| 628 | OS << "ID: " << getID() << " Cost: " << getCost() << " Mapping: " ; |
| 629 | |
| 630 | for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { |
| 631 | const ValueMapping &ValMapping = getOperandMapping(i: OpIdx); |
| 632 | if (OpIdx) |
| 633 | OS << ", " ; |
| 634 | OS << "{ Idx: " << OpIdx << " Map: " << ValMapping << '}'; |
| 635 | } |
| 636 | } |
| 637 | |
| 638 | const int RegisterBankInfo::OperandsMapper::DontKnowIdx = -1; |
| 639 | |
| 640 | RegisterBankInfo::OperandsMapper::OperandsMapper( |
| 641 | MachineInstr &MI, const InstructionMapping &InstrMapping, |
| 642 | MachineRegisterInfo &MRI) |
| 643 | : MRI(MRI), MI(MI), InstrMapping(InstrMapping) { |
| 644 | unsigned NumOpds = InstrMapping.getNumOperands(); |
| 645 | OpToNewVRegIdx.resize(N: NumOpds, NV: OperandsMapper::DontKnowIdx); |
| 646 | assert(InstrMapping.verify(MI) && "Invalid mapping for MI" ); |
| 647 | } |
| 648 | |
| 649 | iterator_range<SmallVectorImpl<Register>::iterator> |
| 650 | RegisterBankInfo::OperandsMapper::getVRegsMem(unsigned OpIdx) { |
| 651 | assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access" ); |
| 652 | unsigned NumPartialVal = |
| 653 | getInstrMapping().getOperandMapping(i: OpIdx).NumBreakDowns; |
| 654 | int StartIdx = OpToNewVRegIdx[OpIdx]; |
| 655 | |
| 656 | if (StartIdx == OperandsMapper::DontKnowIdx) { |
| 657 | // This is the first time we try to access OpIdx. |
| 658 | // Create the cells that will hold all the partial values at the |
| 659 | // end of the list of NewVReg. |
| 660 | StartIdx = NewVRegs.size(); |
| 661 | OpToNewVRegIdx[OpIdx] = StartIdx; |
| 662 | for (unsigned i = 0; i < NumPartialVal; ++i) |
| 663 | NewVRegs.push_back(Elt: 0); |
| 664 | } |
| 665 | SmallVectorImpl<Register>::iterator End = |
| 666 | getNewVRegsEnd(StartIdx, NumVal: NumPartialVal); |
| 667 | |
| 668 | return make_range(x: &NewVRegs[StartIdx], y: End); |
| 669 | } |
| 670 | |
| 671 | SmallVectorImpl<Register>::const_iterator |
| 672 | RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx, |
| 673 | unsigned NumVal) const { |
| 674 | return const_cast<OperandsMapper *>(this)->getNewVRegsEnd(StartIdx, NumVal); |
| 675 | } |
| 676 | SmallVectorImpl<Register>::iterator |
| 677 | RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx, |
| 678 | unsigned NumVal) { |
| 679 | assert((NewVRegs.size() == StartIdx + NumVal || |
| 680 | NewVRegs.size() > StartIdx + NumVal) && |
| 681 | "NewVRegs too small to contain all the partial mapping" ); |
| 682 | return NewVRegs.size() <= StartIdx + NumVal ? NewVRegs.end() |
| 683 | : &NewVRegs[StartIdx + NumVal]; |
| 684 | } |
| 685 | |
| 686 | void RegisterBankInfo::OperandsMapper::createVRegs(unsigned OpIdx) { |
| 687 | assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access" ); |
| 688 | iterator_range<SmallVectorImpl<Register>::iterator> NewVRegsForOpIdx = |
| 689 | getVRegsMem(OpIdx); |
| 690 | const ValueMapping &ValMapping = getInstrMapping().getOperandMapping(i: OpIdx); |
| 691 | const PartialMapping *PartMap = ValMapping.begin(); |
| 692 | for (Register &NewVReg : NewVRegsForOpIdx) { |
| 693 | assert(PartMap != ValMapping.end() && "Out-of-bound access" ); |
| 694 | assert(NewVReg == 0 && "Register has already been created" ); |
| 695 | // The new registers are always bound to scalar with the right size. |
| 696 | // The actual type has to be set when the target does the mapping |
| 697 | // of the instruction. |
| 698 | // The rationale is that this generic code cannot guess how the |
| 699 | // target plans to split the input type. |
| 700 | NewVReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: PartMap->Length)); |
| 701 | MRI.setRegBank(Reg: NewVReg, RegBank: *PartMap->RegBank); |
| 702 | ++PartMap; |
| 703 | } |
| 704 | } |
| 705 | |
| 706 | void RegisterBankInfo::OperandsMapper::setVRegs(unsigned OpIdx, |
| 707 | unsigned PartialMapIdx, |
| 708 | Register NewVReg) { |
| 709 | assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access" ); |
| 710 | assert(getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns > |
| 711 | PartialMapIdx && |
| 712 | "Out-of-bound access for partial mapping" ); |
| 713 | // Make sure the memory is initialized for that operand. |
| 714 | (void)getVRegsMem(OpIdx); |
| 715 | assert(NewVRegs[OpToNewVRegIdx[OpIdx] + PartialMapIdx] == 0 && |
| 716 | "This value is already set" ); |
| 717 | NewVRegs[OpToNewVRegIdx[OpIdx] + PartialMapIdx] = NewVReg; |
| 718 | } |
| 719 | |
| 720 | iterator_range<SmallVectorImpl<Register>::const_iterator> |
| 721 | RegisterBankInfo::OperandsMapper::getVRegs(unsigned OpIdx, |
| 722 | bool ForDebug) const { |
| 723 | (void)ForDebug; |
| 724 | assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access" ); |
| 725 | int StartIdx = OpToNewVRegIdx[OpIdx]; |
| 726 | |
| 727 | if (StartIdx == OperandsMapper::DontKnowIdx) |
| 728 | return make_range(x: NewVRegs.end(), y: NewVRegs.end()); |
| 729 | |
| 730 | unsigned PartMapSize = |
| 731 | getInstrMapping().getOperandMapping(i: OpIdx).NumBreakDowns; |
| 732 | SmallVectorImpl<Register>::const_iterator End = |
| 733 | getNewVRegsEnd(StartIdx, NumVal: PartMapSize); |
| 734 | iterator_range<SmallVectorImpl<Register>::const_iterator> Res = |
| 735 | make_range(x: &NewVRegs[StartIdx], y: End); |
| 736 | #ifndef NDEBUG |
| 737 | for (Register VReg : Res) |
| 738 | assert((VReg || ForDebug) && "Some registers are uninitialized" ); |
| 739 | #endif |
| 740 | return Res; |
| 741 | } |
| 742 | |
| 743 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| 744 | LLVM_DUMP_METHOD void RegisterBankInfo::OperandsMapper::dump() const { |
| 745 | print(dbgs(), true); |
| 746 | dbgs() << '\n'; |
| 747 | } |
| 748 | #endif |
| 749 | |
| 750 | void RegisterBankInfo::OperandsMapper::print(raw_ostream &OS, |
| 751 | bool ForDebug) const { |
| 752 | unsigned NumOpds = getInstrMapping().getNumOperands(); |
| 753 | if (ForDebug) { |
| 754 | OS << "Mapping for " << getMI() << "\nwith " << getInstrMapping() << '\n'; |
| 755 | // Print out the internal state of the index table. |
| 756 | OS << "Populated indices (CellNumber, IndexInNewVRegs): " ; |
| 757 | bool IsFirst = true; |
| 758 | for (unsigned Idx = 0; Idx != NumOpds; ++Idx) { |
| 759 | if (OpToNewVRegIdx[Idx] != DontKnowIdx) { |
| 760 | if (!IsFirst) |
| 761 | OS << ", " ; |
| 762 | OS << '(' << Idx << ", " << OpToNewVRegIdx[Idx] << ')'; |
| 763 | IsFirst = false; |
| 764 | } |
| 765 | } |
| 766 | OS << '\n'; |
| 767 | } else |
| 768 | OS << "Mapping ID: " << getInstrMapping().getID() << ' '; |
| 769 | |
| 770 | OS << "Operand Mapping: " ; |
| 771 | // If we have a function, we can pretty print the name of the registers. |
| 772 | // Otherwise we will print the raw numbers. |
| 773 | const TargetRegisterInfo *TRI = |
| 774 | getMI().getParent() && getMI().getMF() |
| 775 | ? getMI().getMF()->getSubtarget().getRegisterInfo() |
| 776 | : nullptr; |
| 777 | bool IsFirst = true; |
| 778 | for (unsigned Idx = 0; Idx != NumOpds; ++Idx) { |
| 779 | if (OpToNewVRegIdx[Idx] == DontKnowIdx) |
| 780 | continue; |
| 781 | if (!IsFirst) |
| 782 | OS << ", " ; |
| 783 | IsFirst = false; |
| 784 | OS << '(' << printReg(Reg: getMI().getOperand(i: Idx).getReg(), TRI) << ", [" ; |
| 785 | bool IsFirstNewVReg = true; |
| 786 | for (Register VReg : getVRegs(OpIdx: Idx)) { |
| 787 | if (!IsFirstNewVReg) |
| 788 | OS << ", " ; |
| 789 | IsFirstNewVReg = false; |
| 790 | OS << printReg(Reg: VReg, TRI); |
| 791 | } |
| 792 | OS << "])" ; |
| 793 | } |
| 794 | } |
| 795 | |