1//===- llvm/CodeGen/GlobalISel/RegisterBankInfo.cpp --------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the RegisterBankInfo class.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/RegisterBankInfo.h"
13#include "llvm/ADT/APInt.h"
14#include "llvm/ADT/SmallVector.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/ADT/iterator_range.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineRegisterInfo.h"
19#include "llvm/CodeGen/RegisterBank.h"
20#include "llvm/CodeGen/TargetOpcodes.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/Config/llvm-config.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/raw_ostream.h"
26
27#include <algorithm> // For std::max.
28
29#define DEBUG_TYPE "registerbankinfo"
30
31using namespace llvm;
32
33STATISTIC(NumPartialMappingsCreated,
34 "Number of partial mappings dynamically created");
35STATISTIC(NumPartialMappingsAccessed,
36 "Number of partial mappings dynamically accessed");
37STATISTIC(NumValueMappingsCreated,
38 "Number of value mappings dynamically created");
39STATISTIC(NumValueMappingsAccessed,
40 "Number of value mappings dynamically accessed");
41STATISTIC(NumOperandsMappingsCreated,
42 "Number of operands mappings dynamically created");
43STATISTIC(NumOperandsMappingsAccessed,
44 "Number of operands mappings dynamically accessed");
45STATISTIC(NumInstructionMappingsCreated,
46 "Number of instruction mappings dynamically created");
47STATISTIC(NumInstructionMappingsAccessed,
48 "Number of instruction mappings dynamically accessed");
49
50const unsigned RegisterBankInfo::DefaultMappingID = UINT_MAX;
51const unsigned RegisterBankInfo::InvalidMappingID = UINT_MAX - 1;
52
53//------------------------------------------------------------------------------
54// RegisterBankInfo implementation.
55//------------------------------------------------------------------------------
56RegisterBankInfo::RegisterBankInfo(const RegisterBank **RegBanks,
57 unsigned NumRegBanks, const unsigned *Sizes,
58 unsigned HwMode)
59 : RegBanks(RegBanks), NumRegBanks(NumRegBanks), Sizes(Sizes),
60 HwMode(HwMode) {
61#ifndef NDEBUG
62 for (unsigned Idx = 0, End = getNumRegBanks(); Idx != End; ++Idx) {
63 assert(RegBanks[Idx] != nullptr && "Invalid RegisterBank");
64 assert(RegBanks[Idx]->getID() == Idx &&
65 "RegisterBank ID should match index");
66 }
67#endif // NDEBUG
68}
69
70bool RegisterBankInfo::verify(const TargetRegisterInfo &TRI) const {
71#ifndef NDEBUG
72 for (unsigned Idx = 0, End = getNumRegBanks(); Idx != End; ++Idx) {
73 const RegisterBank &RegBank = getRegBank(Idx);
74 assert(Idx == RegBank.getID() &&
75 "ID does not match the index in the array");
76 LLVM_DEBUG(dbgs() << "Verify " << RegBank << '\n');
77 assert(RegBank.verify(*this, TRI) && "RegBank is invalid");
78 }
79#endif // NDEBUG
80 return true;
81}
82
83const RegisterBank *
84RegisterBankInfo::getRegBank(Register Reg, const MachineRegisterInfo &MRI,
85 const TargetRegisterInfo &TRI) const {
86 if (!Reg.isVirtual()) {
87 // FIXME: This was probably a copy to a virtual register that does have a
88 // type we could use.
89 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
90 return RC ? &getRegBankFromRegClass(RC: *RC, Ty: LLT()) : nullptr;
91 }
92
93 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
94 if (auto *RB = dyn_cast_if_present<const RegisterBank *>(Val: RegClassOrBank))
95 return RB;
96 if (auto *RC =
97 dyn_cast_if_present<const TargetRegisterClass *>(Val: RegClassOrBank))
98 return &getRegBankFromRegClass(RC: *RC, Ty: MRI.getType(Reg));
99 return nullptr;
100}
101
102const RegisterBank *RegisterBankInfo::getRegBankFromConstraints(
103 const MachineInstr &MI, unsigned OpIdx, const TargetInstrInfo &TII,
104 const MachineRegisterInfo &MRI) const {
105 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
106
107 // The mapping of the registers may be available via the
108 // register class constraints.
109 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx, TII: &TII, TRI);
110
111 if (!RC)
112 return nullptr;
113
114 Register Reg = MI.getOperand(i: OpIdx).getReg();
115 const RegisterBank &RegBank = getRegBankFromRegClass(RC: *RC, Ty: MRI.getType(Reg));
116 // Check that the target properly implemented getRegBankFromRegClass.
117 assert(RegBank.covers(*RC) &&
118 "The mapping of the register bank does not make sense");
119 return &RegBank;
120}
121
122const TargetRegisterClass *RegisterBankInfo::constrainGenericRegister(
123 Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI) {
124
125 // If the register already has a class, fallback to MRI::constrainRegClass.
126 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
127 if (isa<const TargetRegisterClass *>(Val: RegClassOrBank))
128 return MRI.constrainRegClass(Reg, RC: &RC);
129
130 const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank);
131 // Otherwise, all we can do is ensure the bank covers the class, and set it.
132 if (RB && !RB->covers(RC))
133 return nullptr;
134
135 // If nothing was set or the class is simply compatible, set it.
136 MRI.setRegClass(Reg, RC: &RC);
137 return &RC;
138}
139
140/// Check whether or not \p MI should be treated like a copy
141/// for the mappings.
142/// Copy like instruction are special for mapping because
143/// they don't have actual register constraints. Moreover,
144/// they sometimes have register classes assigned and we can
145/// just use that instead of failing to provide a generic mapping.
146static bool isCopyLike(const MachineInstr &MI) {
147 return MI.isCopy() || MI.isPHI() ||
148 MI.getOpcode() == TargetOpcode::REG_SEQUENCE;
149}
150
151const RegisterBankInfo::InstructionMapping &
152RegisterBankInfo::getInstrMappingImpl(const MachineInstr &MI) const {
153 // For copies we want to walk over the operands and try to find one
154 // that has a register bank since the instruction itself will not get
155 // us any constraint.
156 bool IsCopyLike = isCopyLike(MI);
157 // For copy like instruction, only the mapping of the definition
158 // is important. The rest is not constrained.
159 unsigned NumOperandsForMapping = IsCopyLike ? 1 : MI.getNumOperands();
160
161 const MachineFunction &MF = *MI.getMF();
162 const TargetSubtargetInfo &STI = MF.getSubtarget();
163 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
164 const MachineRegisterInfo &MRI = MF.getRegInfo();
165 // We may need to query the instruction encoding to guess the mapping.
166 const TargetInstrInfo &TII = *STI.getInstrInfo();
167
168 // Before doing anything complicated check if the mapping is not
169 // directly available.
170 bool CompleteMapping = true;
171
172 SmallVector<const ValueMapping *, 8> OperandsMapping(NumOperandsForMapping);
173 for (unsigned OpIdx = 0, EndIdx = MI.getNumOperands(); OpIdx != EndIdx;
174 ++OpIdx) {
175 const MachineOperand &MO = MI.getOperand(i: OpIdx);
176 if (!MO.isReg())
177 continue;
178 Register Reg = MO.getReg();
179 if (!Reg)
180 continue;
181 // The register bank of Reg is just a side effect of the current
182 // excution and in particular, there is no reason to believe this
183 // is the best default mapping for the current instruction. Keep
184 // it as an alternative register bank if we cannot figure out
185 // something.
186 const RegisterBank *AltRegBank = getRegBank(Reg, MRI, TRI);
187 // For copy-like instruction, we want to reuse the register bank
188 // that is already set on Reg, if any, since those instructions do
189 // not have any constraints.
190 const RegisterBank *CurRegBank = IsCopyLike ? AltRegBank : nullptr;
191 if (!CurRegBank) {
192 // If this is a target specific instruction, we can deduce
193 // the register bank from the encoding constraints.
194 CurRegBank = getRegBankFromConstraints(MI, OpIdx, TII, MRI);
195 if (!CurRegBank) {
196 // All our attempts failed, give up.
197 CompleteMapping = false;
198
199 if (!IsCopyLike)
200 // MI does not carry enough information to guess the mapping.
201 return getInvalidInstructionMapping();
202 continue;
203 }
204 }
205
206 TypeSize Size = TRI.getRegSizeInBits(Reg, MRI);
207 const ValueMapping *ValMapping =
208 &getValueMapping(StartIdx: 0, Length: Size.getKnownMinValue(), RegBank: *CurRegBank);
209 if (IsCopyLike) {
210 if (!OperandsMapping[0]) {
211 if (MI.isRegSequence()) {
212 // For reg_sequence, the result size does not match the input.
213 unsigned ResultSize =
214 TRI.getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
215 OperandsMapping[0] = &getValueMapping(StartIdx: 0, Length: ResultSize, RegBank: *CurRegBank);
216 } else {
217 OperandsMapping[0] = ValMapping;
218 }
219 }
220
221 // The default handling assumes any register bank can be copied to any
222 // other. If this isn't the case, the target should specially deal with
223 // reg_sequence/phi. There may also be unsatisfiable copies.
224 for (; OpIdx != EndIdx; ++OpIdx) {
225 const MachineOperand &MO = MI.getOperand(i: OpIdx);
226 if (!MO.isReg())
227 continue;
228 Register Reg = MO.getReg();
229 if (!Reg)
230 continue;
231
232 const RegisterBank *AltRegBank = getRegBank(Reg, MRI, TRI);
233 if (AltRegBank && cannotCopy(Dst: *CurRegBank, Src: *AltRegBank,
234 Size: TRI.getRegSizeInBits(Reg, MRI)))
235 return getInvalidInstructionMapping();
236 }
237
238 CompleteMapping = true;
239 break;
240 }
241
242 OperandsMapping[OpIdx] = ValMapping;
243 }
244
245 if (IsCopyLike && !CompleteMapping) {
246 // No way to deduce the type from what we have.
247 return getInvalidInstructionMapping();
248 }
249
250 assert(CompleteMapping && "Setting an uncomplete mapping");
251 return getInstructionMapping(
252 ID: DefaultMappingID, /*Cost*/ 1,
253 /*OperandsMapping*/ getOperandsMapping(OpdsMapping: OperandsMapping),
254 NumOperands: NumOperandsForMapping);
255}
256
257/// Hashing function for PartialMapping.
258static hash_code hashPartialMapping(unsigned StartIdx, unsigned Length,
259 const RegisterBank *RegBank) {
260 return hash_combine(args: StartIdx, args: Length, args: RegBank ? RegBank->getID() : 0);
261}
262
263/// Overloaded version of hash_value for a PartialMapping.
264hash_code
265llvm::hash_value(const RegisterBankInfo::PartialMapping &PartMapping) {
266 return hashPartialMapping(StartIdx: PartMapping.StartIdx, Length: PartMapping.Length,
267 RegBank: PartMapping.RegBank);
268}
269
270const RegisterBankInfo::PartialMapping &
271RegisterBankInfo::getPartialMapping(unsigned StartIdx, unsigned Length,
272 const RegisterBank &RegBank) const {
273 ++NumPartialMappingsAccessed;
274
275 hash_code Hash = hashPartialMapping(StartIdx, Length, RegBank: &RegBank);
276 auto [It, Inserted] = MapOfPartialMappings.try_emplace(Key: Hash);
277 if (!Inserted)
278 return *It->second;
279
280 ++NumPartialMappingsCreated;
281
282 auto &PartMapping = It->second;
283 PartMapping = std::make_unique<PartialMapping>(args&: StartIdx, args&: Length, args: RegBank);
284 return *PartMapping;
285}
286
287const RegisterBankInfo::ValueMapping &
288RegisterBankInfo::getValueMapping(unsigned StartIdx, unsigned Length,
289 const RegisterBank &RegBank) const {
290 return getValueMapping(BreakDown: &getPartialMapping(StartIdx, Length, RegBank), NumBreakDowns: 1);
291}
292
293static hash_code
294hashValueMapping(const RegisterBankInfo::PartialMapping *BreakDown,
295 unsigned NumBreakDowns) {
296 if (LLVM_LIKELY(NumBreakDowns == 1))
297 return hash_value(PartMapping: *BreakDown);
298 SmallVector<size_t, 8> Hashes(NumBreakDowns);
299 for (unsigned Idx = 0; Idx != NumBreakDowns; ++Idx)
300 Hashes.push_back(Elt: hash_value(PartMapping: BreakDown[Idx]));
301 return hash_combine_range(R&: Hashes);
302}
303
304const RegisterBankInfo::ValueMapping &
305RegisterBankInfo::getValueMapping(const PartialMapping *BreakDown,
306 unsigned NumBreakDowns) const {
307 ++NumValueMappingsAccessed;
308
309 hash_code Hash = hashValueMapping(BreakDown, NumBreakDowns);
310 auto [It, Inserted] = MapOfValueMappings.try_emplace(Key: Hash);
311 if (!Inserted)
312 return *It->second;
313
314 ++NumValueMappingsCreated;
315
316 auto &ValMapping = It->second;
317 ValMapping = std::make_unique<ValueMapping>(args&: BreakDown, args&: NumBreakDowns);
318 return *ValMapping;
319}
320
321template <typename Iterator>
322const RegisterBankInfo::ValueMapping *
323RegisterBankInfo::getOperandsMapping(Iterator Begin, Iterator End) const {
324
325 ++NumOperandsMappingsAccessed;
326
327 // The addresses of the value mapping are unique.
328 // Therefore, we can use them directly to hash the operand mapping.
329 hash_code Hash = hash_combine_range(Begin, End);
330 auto &Res = MapOfOperandsMappings[Hash];
331 if (Res)
332 return Res.get();
333
334 ++NumOperandsMappingsCreated;
335
336 // Create the array of ValueMapping.
337 // Note: this array will not hash to this instance of operands
338 // mapping, because we use the pointer of the ValueMapping
339 // to hash and we expect them to uniquely identify an instance
340 // of value mapping.
341 Res = std::make_unique<ValueMapping[]>(std::distance(Begin, End));
342 unsigned Idx = 0;
343 for (Iterator It = Begin; It != End; ++It, ++Idx) {
344 const ValueMapping *ValMap = *It;
345 if (!ValMap)
346 continue;
347 Res[Idx] = *ValMap;
348 }
349 return Res.get();
350}
351
352const RegisterBankInfo::ValueMapping *RegisterBankInfo::getOperandsMapping(
353 const SmallVectorImpl<const RegisterBankInfo::ValueMapping *> &OpdsMapping)
354 const {
355 return getOperandsMapping(Begin: OpdsMapping.begin(), End: OpdsMapping.end());
356}
357
358const RegisterBankInfo::ValueMapping *RegisterBankInfo::getOperandsMapping(
359 std::initializer_list<const RegisterBankInfo::ValueMapping *> OpdsMapping)
360 const {
361 return getOperandsMapping(Begin: OpdsMapping.begin(), End: OpdsMapping.end());
362}
363
364static hash_code
365hashInstructionMapping(unsigned ID, unsigned Cost,
366 const RegisterBankInfo::ValueMapping *OperandsMapping,
367 unsigned NumOperands) {
368 return hash_combine(args: ID, args: Cost, args: OperandsMapping, args: NumOperands);
369}
370
371const RegisterBankInfo::InstructionMapping &
372RegisterBankInfo::getInstructionMappingImpl(
373 bool IsInvalid, unsigned ID, unsigned Cost,
374 const RegisterBankInfo::ValueMapping *OperandsMapping,
375 unsigned NumOperands) const {
376 assert(((IsInvalid && ID == InvalidMappingID && Cost == 0 &&
377 OperandsMapping == nullptr && NumOperands == 0) ||
378 !IsInvalid) &&
379 "Mismatch argument for invalid input");
380 ++NumInstructionMappingsAccessed;
381
382 hash_code Hash =
383 hashInstructionMapping(ID, Cost, OperandsMapping, NumOperands);
384 auto [It, Inserted] = MapOfInstructionMappings.try_emplace(Key: Hash);
385 if (!Inserted)
386 return *It->second;
387
388 ++NumInstructionMappingsCreated;
389
390 auto &InstrMapping = It->second;
391 InstrMapping = std::make_unique<InstructionMapping>(
392 args&: ID, args&: Cost, args&: OperandsMapping, args&: NumOperands);
393 return *InstrMapping;
394}
395
396const RegisterBankInfo::InstructionMapping &
397RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
398 const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI);
399 if (Mapping.isValid())
400 return Mapping;
401 llvm_unreachable("The target must implement this");
402}
403
404RegisterBankInfo::InstructionMappings
405RegisterBankInfo::getInstrPossibleMappings(const MachineInstr &MI) const {
406 InstructionMappings PossibleMappings;
407 const auto &Mapping = getInstrMapping(MI);
408 if (Mapping.isValid()) {
409 // Put the default mapping first.
410 PossibleMappings.push_back(Elt: &Mapping);
411 }
412
413 // Then the alternative mapping, if any.
414 InstructionMappings AltMappings = getInstrAlternativeMappings(MI);
415 append_range(C&: PossibleMappings, R&: AltMappings);
416#ifndef NDEBUG
417 for (const InstructionMapping *Mapping : PossibleMappings)
418 assert(Mapping->verify(MI) && "Mapping is invalid");
419#endif
420 return PossibleMappings;
421}
422
423RegisterBankInfo::InstructionMappings
424RegisterBankInfo::getInstrAlternativeMappings(const MachineInstr &MI) const {
425 // No alternative for MI.
426 return InstructionMappings();
427}
428
429void RegisterBankInfo::applyDefaultMapping(const OperandsMapper &OpdMapper) {
430 MachineInstr &MI = OpdMapper.getMI();
431 MachineRegisterInfo &MRI = OpdMapper.getMRI();
432 LLVM_DEBUG(dbgs() << "Applying default-like mapping\n");
433 for (unsigned OpIdx = 0,
434 EndIdx = OpdMapper.getInstrMapping().getNumOperands();
435 OpIdx != EndIdx; ++OpIdx) {
436 LLVM_DEBUG(dbgs() << "OpIdx " << OpIdx);
437 MachineOperand &MO = MI.getOperand(i: OpIdx);
438 if (!MO.isReg()) {
439 LLVM_DEBUG(dbgs() << " is not a register, nothing to be done\n");
440 continue;
441 }
442 if (!MO.getReg()) {
443 LLVM_DEBUG(dbgs() << " is $noreg, nothing to be done\n");
444 continue;
445 }
446 LLT Ty = MRI.getType(Reg: MO.getReg());
447 if (!Ty.isValid())
448 continue;
449 assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns !=
450 0 &&
451 "Invalid mapping");
452 assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns ==
453 1 &&
454 "This mapping is too complex for this function");
455 iterator_range<SmallVectorImpl<Register>::const_iterator> NewRegs =
456 OpdMapper.getVRegs(OpIdx);
457 if (NewRegs.empty()) {
458 LLVM_DEBUG(dbgs() << " has not been repaired, nothing to be done\n");
459 continue;
460 }
461 Register OrigReg = MO.getReg();
462 Register NewReg = *NewRegs.begin();
463 LLVM_DEBUG(dbgs() << " changed, replace " << printReg(OrigReg, nullptr));
464 MO.setReg(NewReg);
465 LLVM_DEBUG(dbgs() << " with " << printReg(NewReg, nullptr));
466
467 // The OperandsMapper creates plain scalar, we may have to fix that.
468 // Check if the types match and if not, fix that.
469 LLT OrigTy = MRI.getType(Reg: OrigReg);
470 LLT NewTy = MRI.getType(Reg: NewReg);
471 if (OrigTy != NewTy) {
472 // The default mapping is not supposed to change the size of
473 // the storage. However, right now we don't necessarily bump all
474 // the types to storage size. For instance, we can consider
475 // s16 G_AND legal whereas the storage size is going to be 32.
476 assert(
477 TypeSize::isKnownLE(OrigTy.getSizeInBits(), NewTy.getSizeInBits()) &&
478 "Types with difference size cannot be handled by the default "
479 "mapping");
480 LLVM_DEBUG(dbgs() << "\nChange type of new opd from " << NewTy << " to "
481 << OrigTy);
482 MRI.setType(VReg: NewReg, Ty: OrigTy);
483 }
484 LLVM_DEBUG(dbgs() << '\n');
485 }
486}
487
488//------------------------------------------------------------------------------
489// Helper classes implementation.
490//------------------------------------------------------------------------------
491#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
492LLVM_DUMP_METHOD void RegisterBankInfo::PartialMapping::dump() const {
493 print(dbgs());
494 dbgs() << '\n';
495}
496#endif
497
498bool RegisterBankInfo::PartialMapping::verify(
499 const RegisterBankInfo &RBI) const {
500 assert(RegBank && "Register bank not set");
501 assert(Length && "Empty mapping");
502 assert((StartIdx <= getHighBitIdx()) && "Overflow, switch to APInt?");
503 // Check if the minimum width fits into RegBank.
504 assert(RBI.getMaximumSize(RegBank->getID()) >= Length &&
505 "Register bank too small for Mask");
506 return true;
507}
508
509void RegisterBankInfo::PartialMapping::print(raw_ostream &OS) const {
510 OS << "[" << StartIdx << ", " << getHighBitIdx() << "], RegBank = ";
511 if (RegBank)
512 OS << *RegBank;
513 else
514 OS << "nullptr";
515}
516
517bool RegisterBankInfo::ValueMapping::partsAllUniform() const {
518 if (NumBreakDowns < 2)
519 return true;
520
521 const PartialMapping *First = begin();
522 for (const PartialMapping *Part = First + 1; Part != end(); ++Part) {
523 if (Part->Length != First->Length || Part->RegBank != First->RegBank)
524 return false;
525 }
526
527 return true;
528}
529
530bool RegisterBankInfo::ValueMapping::verify(const RegisterBankInfo &RBI,
531 TypeSize MeaningfulBitWidth) const {
532 assert(NumBreakDowns && "Value mapped nowhere?!");
533 unsigned OrigValueBitWidth = 0;
534 for (const RegisterBankInfo::PartialMapping &PartMap : *this) {
535 // Check that each register bank is big enough to hold the partial value:
536 // this check is done by PartialMapping::verify
537 assert(PartMap.verify(RBI) && "Partial mapping is invalid");
538 // The original value should completely be mapped.
539 // Thus the maximum accessed index + 1 is the size of the original value.
540 OrigValueBitWidth =
541 std::max(a: OrigValueBitWidth, b: PartMap.getHighBitIdx() + 1);
542 }
543 assert((MeaningfulBitWidth.isScalable() ||
544 OrigValueBitWidth >= MeaningfulBitWidth) &&
545 "Meaningful bits not covered by the mapping");
546 APInt ValueMask(OrigValueBitWidth, 0);
547 for (const RegisterBankInfo::PartialMapping &PartMap : *this) {
548 // Check that the union of the partial mappings covers the whole value,
549 // without overlaps.
550 // The high bit is exclusive in the APInt API, thus getHighBitIdx + 1.
551 APInt PartMapMask = APInt::getBitsSet(numBits: OrigValueBitWidth, loBit: PartMap.StartIdx,
552 hiBit: PartMap.getHighBitIdx() + 1);
553 ValueMask ^= PartMapMask;
554 assert((ValueMask & PartMapMask) == PartMapMask &&
555 "Some partial mappings overlap");
556 }
557 assert(ValueMask.isAllOnes() && "Value is not fully mapped");
558 return true;
559}
560
561#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
562LLVM_DUMP_METHOD void RegisterBankInfo::ValueMapping::dump() const {
563 print(dbgs());
564 dbgs() << '\n';
565}
566#endif
567
568void RegisterBankInfo::ValueMapping::print(raw_ostream &OS) const {
569 OS << "#BreakDown: " << NumBreakDowns << " ";
570 bool IsFirst = true;
571 for (const PartialMapping &PartMap : *this) {
572 if (!IsFirst)
573 OS << ", ";
574 OS << '[' << PartMap << ']';
575 IsFirst = false;
576 }
577}
578
579bool RegisterBankInfo::InstructionMapping::verify(
580 const MachineInstr &MI) const {
581 // Check that all the register operands are properly mapped.
582 // Check the constructor invariant.
583 // For PHI, we only care about mapping the definition.
584 assert(NumOperands == (isCopyLike(MI) ? 1 : MI.getNumOperands()) &&
585 "NumOperands must match, see constructor");
586 assert(MI.getParent() && MI.getMF() &&
587 "MI must be connected to a MachineFunction");
588 const MachineFunction &MF = *MI.getMF();
589 const RegisterBankInfo *RBI = MF.getSubtarget().getRegBankInfo();
590 (void)RBI;
591 const MachineRegisterInfo &MRI = MF.getRegInfo();
592
593 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
594 const MachineOperand &MO = MI.getOperand(i: Idx);
595 if (!MO.isReg()) {
596 assert(!getOperandMapping(Idx).isValid() &&
597 "We should not care about non-reg mapping");
598 continue;
599 }
600 Register Reg = MO.getReg();
601 if (!Reg)
602 continue;
603 LLT Ty = MRI.getType(Reg);
604 if (!Ty.isValid())
605 continue;
606 assert(getOperandMapping(Idx).isValid() &&
607 "We must have a mapping for reg operands");
608 const RegisterBankInfo::ValueMapping &MOMapping = getOperandMapping(i: Idx);
609 (void)MOMapping;
610 // Register size in bits.
611 // This size must match what the mapping expects.
612 assert(MOMapping.verify(
613 *RBI, MF.getSubtarget().getRegisterInfo()->getRegSizeInBits(
614 Reg, MRI)) &&
615 "Value mapping is invalid");
616 }
617 return true;
618}
619
620#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
621LLVM_DUMP_METHOD void RegisterBankInfo::InstructionMapping::dump() const {
622 print(dbgs());
623 dbgs() << '\n';
624}
625#endif
626
627void RegisterBankInfo::InstructionMapping::print(raw_ostream &OS) const {
628 OS << "ID: " << getID() << " Cost: " << getCost() << " Mapping: ";
629
630 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
631 const ValueMapping &ValMapping = getOperandMapping(i: OpIdx);
632 if (OpIdx)
633 OS << ", ";
634 OS << "{ Idx: " << OpIdx << " Map: " << ValMapping << '}';
635 }
636}
637
638const int RegisterBankInfo::OperandsMapper::DontKnowIdx = -1;
639
640RegisterBankInfo::OperandsMapper::OperandsMapper(
641 MachineInstr &MI, const InstructionMapping &InstrMapping,
642 MachineRegisterInfo &MRI)
643 : MRI(MRI), MI(MI), InstrMapping(InstrMapping) {
644 unsigned NumOpds = InstrMapping.getNumOperands();
645 OpToNewVRegIdx.resize(N: NumOpds, NV: OperandsMapper::DontKnowIdx);
646 assert(InstrMapping.verify(MI) && "Invalid mapping for MI");
647}
648
649iterator_range<SmallVectorImpl<Register>::iterator>
650RegisterBankInfo::OperandsMapper::getVRegsMem(unsigned OpIdx) {
651 assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access");
652 unsigned NumPartialVal =
653 getInstrMapping().getOperandMapping(i: OpIdx).NumBreakDowns;
654 int StartIdx = OpToNewVRegIdx[OpIdx];
655
656 if (StartIdx == OperandsMapper::DontKnowIdx) {
657 // This is the first time we try to access OpIdx.
658 // Create the cells that will hold all the partial values at the
659 // end of the list of NewVReg.
660 StartIdx = NewVRegs.size();
661 OpToNewVRegIdx[OpIdx] = StartIdx;
662 for (unsigned i = 0; i < NumPartialVal; ++i)
663 NewVRegs.push_back(Elt: 0);
664 }
665 SmallVectorImpl<Register>::iterator End =
666 getNewVRegsEnd(StartIdx, NumVal: NumPartialVal);
667
668 return make_range(x: &NewVRegs[StartIdx], y: End);
669}
670
671SmallVectorImpl<Register>::const_iterator
672RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx,
673 unsigned NumVal) const {
674 return const_cast<OperandsMapper *>(this)->getNewVRegsEnd(StartIdx, NumVal);
675}
676SmallVectorImpl<Register>::iterator
677RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx,
678 unsigned NumVal) {
679 assert((NewVRegs.size() == StartIdx + NumVal ||
680 NewVRegs.size() > StartIdx + NumVal) &&
681 "NewVRegs too small to contain all the partial mapping");
682 return NewVRegs.size() <= StartIdx + NumVal ? NewVRegs.end()
683 : &NewVRegs[StartIdx + NumVal];
684}
685
686void RegisterBankInfo::OperandsMapper::createVRegs(unsigned OpIdx) {
687 assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access");
688 iterator_range<SmallVectorImpl<Register>::iterator> NewVRegsForOpIdx =
689 getVRegsMem(OpIdx);
690 const ValueMapping &ValMapping = getInstrMapping().getOperandMapping(i: OpIdx);
691 const PartialMapping *PartMap = ValMapping.begin();
692 for (Register &NewVReg : NewVRegsForOpIdx) {
693 assert(PartMap != ValMapping.end() && "Out-of-bound access");
694 assert(NewVReg == 0 && "Register has already been created");
695 // The new registers are always bound to scalar with the right size.
696 // The actual type has to be set when the target does the mapping
697 // of the instruction.
698 // The rationale is that this generic code cannot guess how the
699 // target plans to split the input type.
700 NewVReg = MRI.createGenericVirtualRegister(Ty: LLT::integer(SizeInBits: PartMap->Length));
701 MRI.setRegBank(Reg: NewVReg, RegBank: *PartMap->RegBank);
702 ++PartMap;
703 }
704}
705
706void RegisterBankInfo::OperandsMapper::setVRegs(unsigned OpIdx,
707 unsigned PartialMapIdx,
708 Register NewVReg) {
709 assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access");
710 assert(getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns >
711 PartialMapIdx &&
712 "Out-of-bound access for partial mapping");
713 // Make sure the memory is initialized for that operand.
714 (void)getVRegsMem(OpIdx);
715 assert(NewVRegs[OpToNewVRegIdx[OpIdx] + PartialMapIdx] == 0 &&
716 "This value is already set");
717 NewVRegs[OpToNewVRegIdx[OpIdx] + PartialMapIdx] = NewVReg;
718}
719
720iterator_range<SmallVectorImpl<Register>::const_iterator>
721RegisterBankInfo::OperandsMapper::getVRegs(unsigned OpIdx,
722 bool ForDebug) const {
723 (void)ForDebug;
724 assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access");
725 int StartIdx = OpToNewVRegIdx[OpIdx];
726
727 if (StartIdx == OperandsMapper::DontKnowIdx)
728 return make_range(x: NewVRegs.end(), y: NewVRegs.end());
729
730 unsigned PartMapSize =
731 getInstrMapping().getOperandMapping(i: OpIdx).NumBreakDowns;
732 SmallVectorImpl<Register>::const_iterator End =
733 getNewVRegsEnd(StartIdx, NumVal: PartMapSize);
734 iterator_range<SmallVectorImpl<Register>::const_iterator> Res =
735 make_range(x: &NewVRegs[StartIdx], y: End);
736#ifndef NDEBUG
737 for (Register VReg : Res)
738 assert((VReg || ForDebug) && "Some registers are uninitialized");
739#endif
740 return Res;
741}
742
743#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
744LLVM_DUMP_METHOD void RegisterBankInfo::OperandsMapper::dump() const {
745 print(dbgs(), true);
746 dbgs() << '\n';
747}
748#endif
749
750void RegisterBankInfo::OperandsMapper::print(raw_ostream &OS,
751 bool ForDebug) const {
752 unsigned NumOpds = getInstrMapping().getNumOperands();
753 if (ForDebug) {
754 OS << "Mapping for " << getMI() << "\nwith " << getInstrMapping() << '\n';
755 // Print out the internal state of the index table.
756 OS << "Populated indices (CellNumber, IndexInNewVRegs): ";
757 bool IsFirst = true;
758 for (unsigned Idx = 0; Idx != NumOpds; ++Idx) {
759 if (OpToNewVRegIdx[Idx] != DontKnowIdx) {
760 if (!IsFirst)
761 OS << ", ";
762 OS << '(' << Idx << ", " << OpToNewVRegIdx[Idx] << ')';
763 IsFirst = false;
764 }
765 }
766 OS << '\n';
767 } else
768 OS << "Mapping ID: " << getInstrMapping().getID() << ' ';
769
770 OS << "Operand Mapping: ";
771 // If we have a function, we can pretty print the name of the registers.
772 // Otherwise we will print the raw numbers.
773 const TargetRegisterInfo *TRI =
774 getMI().getParent() && getMI().getMF()
775 ? getMI().getMF()->getSubtarget().getRegisterInfo()
776 : nullptr;
777 bool IsFirst = true;
778 for (unsigned Idx = 0; Idx != NumOpds; ++Idx) {
779 if (OpToNewVRegIdx[Idx] == DontKnowIdx)
780 continue;
781 if (!IsFirst)
782 OS << ", ";
783 IsFirst = false;
784 OS << '(' << printReg(Reg: getMI().getOperand(i: Idx).getReg(), TRI) << ", [";
785 bool IsFirstNewVReg = true;
786 for (Register VReg : getVRegs(OpIdx: Idx)) {
787 if (!IsFirstNewVReg)
788 OS << ", ";
789 IsFirstNewVReg = false;
790 OS << printReg(Reg: VReg, TRI);
791 }
792 OS << "])";
793 }
794}
795