1//===--- X86DomainReassignment.cpp - Selectively switch register classes---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass attempts to find instruction chains (closures) in one domain,
10// and convert them to equivalent instructions in a different domain,
11// if profitable.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86InstrInfo.h"
17#include "X86Subtarget.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/Printable.h"
29#include <bitset>
30
31using namespace llvm;
32
33#define DEBUG_TYPE "x86-domain-reassignment"
34
35STATISTIC(NumClosuresConverted, "Number of closures converted by the pass");
36STATISTIC(NumClosuresBuilt, "Number of closures built by the pass");
37
38static cl::opt<bool> DisableX86DomainReassignment(
39 "disable-x86-domain-reassignment", cl::Hidden,
40 cl::desc("X86: Disable Virtual Register Reassignment."), cl::init(Val: false));
41
42namespace {
43enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
44
45static bool isMask(const TargetRegisterClass *RC,
46 const TargetRegisterInfo *TRI) {
47 return X86::VK16RegClass.hasSubClassEq(RC);
48}
49
50static RegDomain getDomain(const TargetRegisterClass *RC,
51 const TargetRegisterInfo *TRI) {
52 if (TRI->isGeneralPurposeRegisterClass(RC))
53 return GPRDomain;
54 if (isMask(RC, TRI))
55 return MaskDomain;
56 return OtherDomain;
57}
58
59/// Return a register class equivalent to \p SrcRC, in \p Domain.
60static const TargetRegisterClass *getDstRC(const TargetRegisterClass *SrcRC,
61 RegDomain Domain) {
62 assert(Domain == MaskDomain && "add domain");
63 if (X86::GR8RegClass.hasSubClassEq(RC: SrcRC))
64 return &X86::VK8RegClass;
65 if (X86::GR16RegClass.hasSubClassEq(RC: SrcRC))
66 return &X86::VK16RegClass;
67 if (X86::GR32RegClass.hasSubClassEq(RC: SrcRC))
68 return &X86::VK32RegClass;
69 if (X86::GR64RegClass.hasSubClassEq(RC: SrcRC))
70 return &X86::VK64RegClass;
71 llvm_unreachable("add register class");
72 return nullptr;
73}
74
75/// Abstract Instruction Converter class.
76class InstrConverterBase {
77protected:
78 unsigned SrcOpcode;
79
80public:
81 InstrConverterBase(unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
82
83 virtual ~InstrConverterBase() = default;
84
85 /// \returns true if \p MI is legal to convert.
86 virtual bool isLegal(const MachineInstr *MI,
87 const TargetInstrInfo *TII) const {
88 assert(MI->getOpcode() == SrcOpcode &&
89 "Wrong instruction passed to converter");
90 return true;
91 }
92
93 /// Applies conversion to \p MI.
94 ///
95 /// \returns true if \p MI is no longer need, and can be deleted.
96 virtual bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
97 MachineRegisterInfo *MRI) const = 0;
98
99 /// \returns the cost increment incurred by converting \p MI.
100 virtual double getExtraCost(const MachineInstr *MI,
101 MachineRegisterInfo *MRI) const = 0;
102};
103
104/// An Instruction Converter which ignores the given instruction.
105/// For example, PHI instructions can be safely ignored since only the registers
106/// need to change.
107class InstrIgnore : public InstrConverterBase {
108public:
109 InstrIgnore(unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
110
111 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
112 MachineRegisterInfo *MRI) const override {
113 assert(isLegal(MI, TII) && "Cannot convert instruction");
114 return false;
115 }
116
117 double getExtraCost(const MachineInstr *MI,
118 MachineRegisterInfo *MRI) const override {
119 return 0;
120 }
121};
122
123/// An Instruction Converter which replaces an instruction with another.
124class InstrReplacer : public InstrConverterBase {
125public:
126 /// Opcode of the destination instruction.
127 unsigned DstOpcode;
128
129 InstrReplacer(unsigned SrcOpcode, unsigned DstOpcode)
130 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
131
132 bool isLegal(const MachineInstr *MI,
133 const TargetInstrInfo *TII) const override {
134 if (!InstrConverterBase::isLegal(MI, TII))
135 return false;
136 // It's illegal to replace an instruction that implicitly defines a register
137 // with an instruction that doesn't, unless that register dead.
138 for (const auto &MO : MI->implicit_operands())
139 if (MO.isReg() && MO.isDef() && !MO.isDead() &&
140 !TII->get(Opcode: DstOpcode).hasImplicitDefOfPhysReg(Reg: MO.getReg()))
141 return false;
142 return true;
143 }
144
145 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
146 MachineRegisterInfo *MRI) const override {
147 assert(isLegal(MI, TII) && "Cannot convert instruction");
148 MachineInstrBuilder Bld =
149 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: DstOpcode));
150 // Transfer explicit operands from original instruction. Implicit operands
151 // are handled by BuildMI.
152 for (auto &Op : MI->explicit_operands())
153 Bld.add(MO: Op);
154 return true;
155 }
156
157 double getExtraCost(const MachineInstr *MI,
158 MachineRegisterInfo *MRI) const override {
159 // Assuming instructions have the same cost.
160 return 0;
161 }
162};
163
164/// An Instruction Converter which replaces an instruction with another, and
165/// adds a COPY from the new instruction's destination to the old one's.
166class InstrReplacerDstCOPY : public InstrConverterBase {
167public:
168 unsigned DstOpcode;
169
170 InstrReplacerDstCOPY(unsigned SrcOpcode, unsigned DstOpcode)
171 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
172
173 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
174 MachineRegisterInfo *MRI) const override {
175 assert(isLegal(MI, TII) && "Cannot convert instruction");
176 MachineBasicBlock *MBB = MI->getParent();
177 const DebugLoc &DL = MI->getDebugLoc();
178
179 Register Reg =
180 MRI->createVirtualRegister(RegClass: TII->getRegClass(MCID: TII->get(Opcode: DstOpcode), OpNum: 0));
181 MachineInstrBuilder Bld = BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: DstOpcode), DestReg: Reg);
182 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
183 Bld.add(MO);
184
185 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
186 .add(MO: MI->getOperand(i: 0))
187 .addReg(RegNo: Reg);
188
189 return true;
190 }
191
192 double getExtraCost(const MachineInstr *MI,
193 MachineRegisterInfo *MRI) const override {
194 // Assuming instructions have the same cost, and that COPY is in the same
195 // domain so it will be eliminated.
196 return 0;
197 }
198};
199
200/// An Instruction Converter for replacing COPY instructions.
201class InstrCOPYReplacer : public InstrReplacer {
202public:
203 RegDomain DstDomain;
204
205 InstrCOPYReplacer(unsigned SrcOpcode, RegDomain DstDomain, unsigned DstOpcode)
206 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
207
208 bool isLegal(const MachineInstr *MI,
209 const TargetInstrInfo *TII) const override {
210 if (!InstrConverterBase::isLegal(MI, TII))
211 return false;
212
213 // Don't allow copies to/flow GR8/GR16 physical registers.
214 // FIXME: Is there some better way to support this?
215 Register DstReg = MI->getOperand(i: 0).getReg();
216 if (DstReg.isPhysical() && (X86::GR8RegClass.contains(Reg: DstReg) ||
217 X86::GR16RegClass.contains(Reg: DstReg)))
218 return false;
219 Register SrcReg = MI->getOperand(i: 1).getReg();
220 if (SrcReg.isPhysical() && (X86::GR8RegClass.contains(Reg: SrcReg) ||
221 X86::GR16RegClass.contains(Reg: SrcReg)))
222 return false;
223
224 return true;
225 }
226
227 double getExtraCost(const MachineInstr *MI,
228 MachineRegisterInfo *MRI) const override {
229 assert(MI->getOpcode() == TargetOpcode::COPY && "Expected a COPY");
230
231 for (const auto &MO : MI->operands()) {
232 // Physical registers will not be converted. Assume that converting the
233 // COPY to the destination domain will eventually result in a actual
234 // instruction.
235 if (MO.getReg().isPhysical())
236 return 1;
237
238 RegDomain OpDomain = getDomain(RC: MRI->getRegClass(Reg: MO.getReg()),
239 TRI: MRI->getTargetRegisterInfo());
240 // Converting a cross domain COPY to a same domain COPY should eliminate
241 // an insturction
242 if (OpDomain == DstDomain)
243 return -1;
244 }
245 return 0;
246 }
247};
248
249/// An Instruction Converter which replaces an instruction with a COPY.
250class InstrReplaceWithCopy : public InstrConverterBase {
251public:
252 // Source instruction operand Index, to be used as the COPY source.
253 unsigned SrcOpIdx;
254
255 InstrReplaceWithCopy(unsigned SrcOpcode, unsigned SrcOpIdx)
256 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
257
258 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
259 MachineRegisterInfo *MRI) const override {
260 assert(isLegal(MI, TII) && "Cannot convert instruction");
261 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
262 MCID: TII->get(Opcode: TargetOpcode::COPY))
263 .add(MOs: {MI->getOperand(i: 0), MI->getOperand(i: SrcOpIdx)});
264 return true;
265 }
266
267 double getExtraCost(const MachineInstr *MI,
268 MachineRegisterInfo *MRI) const override {
269 return 0;
270 }
271};
272
273// Key type to be used by the Instruction Converters map.
274// A converter is identified by <destination domain, source opcode>
275typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
276
277typedef DenseMap<InstrConverterBaseKeyTy, std::unique_ptr<InstrConverterBase>>
278 InstrConverterBaseMap;
279
280/// A closure is a set of virtual register representing all of the edges in
281/// the closure, as well as all of the instructions connected by those edges.
282///
283/// A closure may encompass virtual registers in the same register bank that
284/// have different widths. For example, it may contain 32-bit GPRs as well as
285/// 64-bit GPRs.
286///
287/// A closure that computes an address (i.e. defines a virtual register that is
288/// used in a memory operand) excludes the instructions that contain memory
289/// operands using the address. Such an instruction will be included in a
290/// different closure that manipulates the loaded or stored value.
291class Closure {
292private:
293 /// Virtual registers in the closure.
294 DenseSet<Register> Edges;
295
296 /// Instructions in the closure.
297 SmallVector<MachineInstr *, 8> Instrs;
298
299 /// Domains which this closure can legally be reassigned to.
300 std::bitset<NumDomains> LegalDstDomains;
301
302 /// An ID to uniquely identify this closure, even when it gets
303 /// moved around
304 unsigned ID;
305
306public:
307 Closure(unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) : ID(ID) {
308 for (RegDomain D : LegalDstDomainList)
309 LegalDstDomains.set(position: D);
310 }
311
312 /// Mark this closure as illegal for reassignment to all domains.
313 void setAllIllegal() { LegalDstDomains.reset(); }
314
315 /// \returns true if this closure has domains which are legal to reassign to.
316 bool hasLegalDstDomain() const { return LegalDstDomains.any(); }
317
318 /// \returns true if is legal to reassign this closure to domain \p RD.
319 bool isLegal(RegDomain RD) const { return LegalDstDomains[RD]; }
320
321 /// Mark this closure as illegal for reassignment to domain \p RD.
322 void setIllegal(RegDomain RD) { LegalDstDomains[RD] = false; }
323
324 bool empty() const { return Edges.empty(); }
325
326 bool insertEdge(Register Reg) { return Edges.insert(V: Reg).second; }
327
328 using const_edge_iterator = DenseSet<Register>::const_iterator;
329 iterator_range<const_edge_iterator> edges() const { return Edges; }
330
331 void addInstruction(MachineInstr *I) {
332 Instrs.push_back(Elt: I);
333 }
334
335 ArrayRef<MachineInstr *> instructions() const {
336 return Instrs;
337 }
338
339 LLVM_DUMP_METHOD void dump(const MachineRegisterInfo *MRI) const {
340 dbgs() << "Registers: ";
341 ListSeparator LS;
342 for (Register Reg : Edges)
343 dbgs() << LS << printReg(Reg, TRI: MRI->getTargetRegisterInfo(), SubIdx: 0, MRI);
344 dbgs() << "\n" << "Instructions:";
345 for (MachineInstr *MI : Instrs) {
346 dbgs() << "\n ";
347 MI->print(OS&: dbgs());
348 }
349 dbgs() << "\n";
350 }
351
352 unsigned getID() const {
353 return ID;
354 }
355
356};
357
358class X86DomainReassignmentImpl {
359public:
360 bool runOnMachineFunction(MachineFunction &MF);
361
362private:
363 const X86Subtarget *STI = nullptr;
364 MachineRegisterInfo *MRI = nullptr;
365 const X86InstrInfo *TII = nullptr;
366
367 /// All edges that are included in some closure
368 DenseMap<Register, unsigned> EnclosedEdges;
369
370 /// All instructions that are included in some closure.
371 DenseMap<MachineInstr *, unsigned> EnclosedInstrs;
372
373 /// A map of available Instruction Converters.
374 InstrConverterBaseMap Converters;
375
376 /// Initialize Converters map.
377 void initConverters();
378
379 /// Starting from \Reg, expand the closure as much as possible.
380 void buildClosure(Closure &, Register Reg);
381
382 /// Enqueue \p Reg to be considered for addition to the closure.
383 /// Return false if the closure becomes invalid.
384 bool visitRegister(Closure &, Register Reg, RegDomain &Domain,
385 SmallVectorImpl<Register> &Worklist);
386
387 /// Reassign the closure to \p Domain.
388 void reassign(const Closure &C, RegDomain Domain) const;
389
390 /// Add \p MI to the closure.
391 /// Return false if the closure becomes invalid.
392 bool encloseInstr(Closure &C, MachineInstr *MI);
393
394 /// /returns true if it is profitable to reassign the closure to \p Domain.
395 bool isReassignmentProfitable(const Closure &C, RegDomain Domain) const;
396
397 /// Calculate the total cost of reassigning the closure to \p Domain.
398 double calculateCost(const Closure &C, RegDomain Domain) const;
399};
400
401class X86DomainReassignmentLegacy : public MachineFunctionPass {
402public:
403 static char ID;
404
405 X86DomainReassignmentLegacy() : MachineFunctionPass(ID) {}
406
407 bool runOnMachineFunction(MachineFunction &MF) override;
408
409 void getAnalysisUsage(AnalysisUsage &AU) const override {
410 AU.setPreservesCFG();
411 MachineFunctionPass::getAnalysisUsage(AU);
412 }
413
414 StringRef getPassName() const override {
415 return "X86 Domain Reassignment Pass";
416 }
417};
418
419char X86DomainReassignmentLegacy::ID = 0;
420
421} // End anonymous namespace.
422
423bool X86DomainReassignmentImpl::visitRegister(
424 Closure &C, Register Reg, RegDomain &Domain,
425 SmallVectorImpl<Register> &Worklist) {
426 if (!Reg.isVirtual())
427 return true;
428
429 auto I = EnclosedEdges.find(Val: Reg);
430 if (I != EnclosedEdges.end()) {
431 if (I->second != C.getID()) {
432 C.setAllIllegal();
433 return false;
434 }
435 return true;
436 }
437
438 if (!MRI->hasOneDef(RegNo: Reg))
439 return true;
440
441 RegDomain RD = getDomain(RC: MRI->getRegClass(Reg), TRI: MRI->getTargetRegisterInfo());
442 // First edge in closure sets the domain.
443 if (Domain == NoDomain)
444 Domain = RD;
445
446 if (Domain != RD)
447 return true;
448
449 Worklist.push_back(Elt: Reg);
450 return true;
451}
452
453bool X86DomainReassignmentImpl::encloseInstr(Closure &C, MachineInstr *MI) {
454 auto [I, Inserted] = EnclosedInstrs.try_emplace(Key: MI, Args: C.getID());
455 if (!Inserted) {
456 if (I->second != C.getID()) {
457 // Instruction already belongs to another closure, avoid conflicts between
458 // closure and mark this closure as illegal.
459 C.setAllIllegal();
460 return false;
461 }
462 return true;
463 }
464
465 C.addInstruction(I: MI);
466
467 // Mark closure as illegal for reassignment to domains, if there is no
468 // converter for the instruction or if the converter cannot convert the
469 // instruction.
470 for (int i = 0; i != NumDomains; ++i) {
471 if (C.isLegal(RD: (RegDomain)i)) {
472 auto I = Converters.find(Val: {i, MI->getOpcode()});
473 if (I == Converters.end() || !I->second->isLegal(MI, TII))
474 C.setIllegal((RegDomain)i);
475 }
476 }
477 return C.hasLegalDstDomain();
478}
479
480double X86DomainReassignmentImpl::calculateCost(const Closure &C,
481 RegDomain DstDomain) const {
482 assert(C.isLegal(DstDomain) && "Cannot calculate cost for illegal closure");
483
484 double Cost = 0.0;
485 for (auto *MI : C.instructions())
486 Cost += Converters.find(Val: {DstDomain, MI->getOpcode()})
487 ->second->getExtraCost(MI, MRI);
488 return Cost;
489}
490
491bool X86DomainReassignmentImpl::isReassignmentProfitable(
492 const Closure &C, RegDomain Domain) const {
493 return calculateCost(C, DstDomain: Domain) < 0.0;
494}
495
496void X86DomainReassignmentImpl::reassign(const Closure &C,
497 RegDomain Domain) const {
498 assert(C.isLegal(Domain) && "Cannot convert illegal closure");
499
500 // Iterate all instructions in the closure, convert each one using the
501 // appropriate converter.
502 SmallVector<MachineInstr *, 8> ToErase;
503 for (auto *MI : C.instructions())
504 if (Converters.find(Val: {Domain, MI->getOpcode()})
505 ->second->convertInstr(MI, TII, MRI))
506 ToErase.push_back(Elt: MI);
507
508 // Iterate all registers in the closure, replace them with registers in the
509 // destination domain.
510 for (Register Reg : C.edges()) {
511 MRI->setRegClass(Reg, RC: getDstRC(SrcRC: MRI->getRegClass(Reg), Domain));
512 for (auto &MO : MRI->use_operands(Reg)) {
513 if (MO.isReg())
514 // Remove all subregister references as they are not valid in the
515 // destination domain.
516 MO.setSubReg(0);
517 }
518 }
519
520 for (auto *MI : ToErase)
521 MI->eraseFromParent();
522}
523
524/// \returns true when \p Reg is used as part of an address calculation in \p
525/// MI.
526static bool usedAsAddr(const MachineInstr &MI, Register Reg,
527 const TargetInstrInfo *TII) {
528 if (!MI.mayLoadOrStore())
529 return false;
530
531 const MCInstrDesc &Desc = TII->get(Opcode: MI.getOpcode());
532 int MemOpStart = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
533 if (MemOpStart == -1)
534 return false;
535
536 MemOpStart += X86II::getOperandBias(Desc);
537 for (unsigned MemOpIdx = MemOpStart,
538 MemOpEnd = MemOpStart + X86::AddrNumOperands;
539 MemOpIdx < MemOpEnd; ++MemOpIdx) {
540 const MachineOperand &Op = MI.getOperand(i: MemOpIdx);
541 if (Op.isReg() && Op.getReg() == Reg)
542 return true;
543 }
544 return false;
545}
546
547void X86DomainReassignmentImpl::buildClosure(Closure &C, Register Reg) {
548 SmallVector<Register, 4> Worklist;
549 RegDomain Domain = NoDomain;
550 visitRegister(C, Reg, Domain, Worklist);
551 while (!Worklist.empty()) {
552 Register CurReg = Worklist.pop_back_val();
553
554 // Register already in this closure.
555 if (!C.insertEdge(Reg: CurReg))
556 continue;
557 EnclosedEdges[CurReg] = C.getID();
558
559 MachineInstr *DefMI = MRI->getVRegDef(Reg: CurReg);
560 if (!encloseInstr(C, MI: DefMI))
561 return;
562
563 // Add register used by the defining MI to the worklist.
564 // Do not add registers which are used in address calculation, they will be
565 // added to a different closure.
566 int OpEnd = DefMI->getNumOperands();
567 const MCInstrDesc &Desc = DefMI->getDesc();
568 int MemOp = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
569 if (MemOp != -1)
570 MemOp += X86II::getOperandBias(Desc);
571 for (int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {
572 if (OpIdx == MemOp) {
573 // skip address calculation.
574 OpIdx += (X86::AddrNumOperands - 1);
575 continue;
576 }
577 auto &Op = DefMI->getOperand(i: OpIdx);
578 if (!Op.isReg() || !Op.isUse())
579 continue;
580 if (!visitRegister(C, Reg: Op.getReg(), Domain, Worklist))
581 return;
582 }
583
584 // Expand closure through register uses.
585 for (auto &UseMI : MRI->use_nodbg_instructions(Reg: CurReg)) {
586 // We would like to avoid converting closures which calculare addresses,
587 // as this should remain in GPRs.
588 if (usedAsAddr(MI: UseMI, Reg: CurReg, TII)) {
589 C.setAllIllegal();
590 return;
591 }
592 if (!encloseInstr(C, MI: &UseMI))
593 return;
594
595 for (auto &DefOp : UseMI.defs()) {
596 if (!DefOp.isReg())
597 continue;
598
599 Register DefReg = DefOp.getReg();
600 if (!DefReg.isVirtual()) {
601 C.setAllIllegal();
602 return;
603 }
604 if (!visitRegister(C, Reg: DefReg, Domain, Worklist))
605 return;
606 }
607 }
608 }
609}
610
611void X86DomainReassignmentImpl::initConverters() {
612 Converters[{MaskDomain, TargetOpcode::PHI}] =
613 std::make_unique<InstrIgnore>(args: TargetOpcode::PHI);
614
615 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
616 std::make_unique<InstrIgnore>(args: TargetOpcode::IMPLICIT_DEF);
617
618 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
619 std::make_unique<InstrReplaceWithCopy>(args: TargetOpcode::INSERT_SUBREG, args: 2);
620
621 Converters[{MaskDomain, TargetOpcode::COPY}] =
622 std::make_unique<InstrCOPYReplacer>(args: TargetOpcode::COPY, args: MaskDomain,
623 args: TargetOpcode::COPY);
624
625 auto createReplacerDstCOPY = [&](unsigned From, unsigned To) {
626 Converters[{MaskDomain, From}] =
627 std::make_unique<InstrReplacerDstCOPY>(args&: From, args&: To);
628 };
629
630#define GET_EGPR_IF_ENABLED(OPC) STI->hasEGPR() ? OPC##_EVEX : OPC
631 createReplacerDstCOPY(X86::MOVZX32rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
632 createReplacerDstCOPY(X86::MOVZX64rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
633
634 createReplacerDstCOPY(X86::MOVZX32rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
635 createReplacerDstCOPY(X86::MOVZX64rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
636
637 if (STI->hasDQI()) {
638 createReplacerDstCOPY(X86::MOVZX16rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
639 createReplacerDstCOPY(X86::MOVZX32rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
640 createReplacerDstCOPY(X86::MOVZX64rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
641
642 createReplacerDstCOPY(X86::MOVZX16rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
643 createReplacerDstCOPY(X86::MOVZX32rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
644 createReplacerDstCOPY(X86::MOVZX64rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
645 }
646
647 auto createReplacer = [&](unsigned From, unsigned To) {
648 Converters[{MaskDomain, From}] = std::make_unique<InstrReplacer>(args&: From, args&: To);
649 };
650
651 createReplacer(X86::MOV16rm, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
652 createReplacer(X86::MOV16mr, GET_EGPR_IF_ENABLED(X86::KMOVWmk));
653 createReplacer(X86::MOV16rr, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
654 createReplacer(X86::SHR16ri, X86::KSHIFTRWki);
655 createReplacer(X86::SHL16ri, X86::KSHIFTLWki);
656 createReplacer(X86::NOT16r, X86::KNOTWkk);
657 createReplacer(X86::OR16rr, X86::KORWkk);
658 createReplacer(X86::AND16rr, X86::KANDWkk);
659 createReplacer(X86::XOR16rr, X86::KXORWkk);
660
661 bool HasNDD = STI->hasNDD();
662 if (HasNDD) {
663 createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWki);
664 createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWki);
665 createReplacer(X86::NOT16r_ND, X86::KNOTWkk);
666 createReplacer(X86::OR16rr_ND, X86::KORWkk);
667 createReplacer(X86::AND16rr_ND, X86::KANDWkk);
668 createReplacer(X86::XOR16rr_ND, X86::KXORWkk);
669 }
670
671 if (STI->hasBWI()) {
672 createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm));
673 createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm));
674
675 createReplacer(X86::MOV32mr, GET_EGPR_IF_ENABLED(X86::KMOVDmk));
676 createReplacer(X86::MOV64mr, GET_EGPR_IF_ENABLED(X86::KMOVQmk));
677
678 createReplacer(X86::MOV32rr, GET_EGPR_IF_ENABLED(X86::KMOVDkk));
679 createReplacer(X86::MOV64rr, GET_EGPR_IF_ENABLED(X86::KMOVQkk));
680
681 createReplacer(X86::SHR32ri, X86::KSHIFTRDki);
682 createReplacer(X86::SHR64ri, X86::KSHIFTRQki);
683
684 createReplacer(X86::SHL32ri, X86::KSHIFTLDki);
685 createReplacer(X86::SHL64ri, X86::KSHIFTLQki);
686
687 createReplacer(X86::ADD32rr, X86::KADDDkk);
688 createReplacer(X86::ADD64rr, X86::KADDQkk);
689
690 createReplacer(X86::NOT32r, X86::KNOTDkk);
691 createReplacer(X86::NOT64r, X86::KNOTQkk);
692
693 createReplacer(X86::OR32rr, X86::KORDkk);
694 createReplacer(X86::OR64rr, X86::KORQkk);
695
696 createReplacer(X86::AND32rr, X86::KANDDkk);
697 createReplacer(X86::AND64rr, X86::KANDQkk);
698
699 createReplacer(X86::ANDN32rr, X86::KANDNDkk);
700 createReplacer(X86::ANDN64rr, X86::KANDNQkk);
701
702 createReplacer(X86::XOR32rr, X86::KXORDkk);
703 createReplacer(X86::XOR64rr, X86::KXORQkk);
704
705 if (HasNDD) {
706 createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDki);
707 createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDki);
708 createReplacer(X86::ADD32rr_ND, X86::KADDDkk);
709 createReplacer(X86::NOT32r_ND, X86::KNOTDkk);
710 createReplacer(X86::OR32rr_ND, X86::KORDkk);
711 createReplacer(X86::AND32rr_ND, X86::KANDDkk);
712 createReplacer(X86::XOR32rr_ND, X86::KXORDkk);
713 createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQki);
714 createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQki);
715 createReplacer(X86::ADD64rr_ND, X86::KADDQkk);
716 createReplacer(X86::NOT64r_ND, X86::KNOTQkk);
717 createReplacer(X86::OR64rr_ND, X86::KORQkk);
718 createReplacer(X86::AND64rr_ND, X86::KANDQkk);
719 createReplacer(X86::XOR64rr_ND, X86::KXORQkk);
720 }
721
722 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
723 // to prove only Z flag is used.
724 // createReplacer(X86::TEST32rr, X86::KTESTDkk);
725 // createReplacer(X86::TEST64rr, X86::KTESTQkk);
726 }
727
728 if (STI->hasDQI()) {
729 createReplacer(X86::ADD8rr, X86::KADDBkk);
730 createReplacer(X86::ADD16rr, X86::KADDWkk);
731
732 createReplacer(X86::AND8rr, X86::KANDBkk);
733
734 createReplacer(X86::MOV8rm, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
735 createReplacer(X86::MOV8mr, GET_EGPR_IF_ENABLED(X86::KMOVBmk));
736 createReplacer(X86::MOV8rr, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
737
738 createReplacer(X86::NOT8r, X86::KNOTBkk);
739
740 createReplacer(X86::OR8rr, X86::KORBkk);
741
742 createReplacer(X86::SHR8ri, X86::KSHIFTRBki);
743 createReplacer(X86::SHL8ri, X86::KSHIFTLBki);
744
745 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
746 // to prove only Z flag is used.
747 // createReplacer(X86::TEST8rr, X86::KTESTBkk);
748 // createReplacer(X86::TEST16rr, X86::KTESTWkk);
749
750 createReplacer(X86::XOR8rr, X86::KXORBkk);
751
752 if (HasNDD) {
753 createReplacer(X86::ADD8rr_ND, X86::KADDBkk);
754 createReplacer(X86::ADD16rr_ND, X86::KADDWkk);
755 createReplacer(X86::AND8rr_ND, X86::KANDBkk);
756 createReplacer(X86::NOT8r_ND, X86::KNOTBkk);
757 createReplacer(X86::OR8rr_ND, X86::KORBkk);
758 createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBki);
759 createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBki);
760 createReplacer(X86::XOR8rr_ND, X86::KXORBkk);
761 }
762 }
763#undef GET_EGPR_IF_ENABLED
764}
765
766bool X86DomainReassignmentImpl::runOnMachineFunction(MachineFunction &MF) {
767 if (DisableX86DomainReassignment)
768 return false;
769
770 LLVM_DEBUG(
771 dbgs() << "***** Machine Function before Domain Reassignment *****\n");
772 LLVM_DEBUG(MF.print(dbgs()));
773
774 STI = &MF.getSubtarget<X86Subtarget>();
775 // GPR->K is the only transformation currently supported, bail out early if no
776 // AVX512.
777 // TODO: We're also bailing of AVX512BW isn't supported since we use VK32 and
778 // VK64 for GR32/GR64, but those aren't legal classes on KNL. If the register
779 // coalescer doesn't clean it up and we generate a spill we will crash.
780 if (!STI->hasAVX512() || !STI->hasBWI())
781 return false;
782
783 MRI = &MF.getRegInfo();
784 assert(MRI->isSSA() && "Expected MIR to be in SSA form");
785
786 TII = STI->getInstrInfo();
787 initConverters();
788 bool Changed = false;
789
790 EnclosedEdges.clear();
791 EnclosedInstrs.clear();
792
793 std::vector<Closure> Closures;
794
795 // Go over all virtual registers and calculate a closure.
796 unsigned ClosureID = 0;
797 for (unsigned Idx = 0; Idx < MRI->getNumVirtRegs(); ++Idx) {
798 Register Reg = Register::index2VirtReg(Index: Idx);
799
800 // Skip unused VRegs.
801 if (MRI->reg_nodbg_empty(RegNo: Reg))
802 continue;
803
804 // GPR only current source domain supported.
805 if (!MRI->getTargetRegisterInfo()->isGeneralPurposeRegisterClass(
806 RC: MRI->getRegClass(Reg)))
807 continue;
808
809 // Register already in closure.
810 if (EnclosedEdges.contains(Val: Reg))
811 continue;
812
813 // Calculate closure starting with Reg.
814 Closure C(ClosureID++, {MaskDomain});
815 buildClosure(C, Reg);
816 ++NumClosuresBuilt;
817
818 // Collect all closures that can potentially be converted.
819 if (!C.empty() && C.isLegal(RD: MaskDomain))
820 Closures.push_back(x: std::move(C));
821 }
822
823 for (Closure &C : Closures) {
824 LLVM_DEBUG(C.dump(MRI));
825 if (isReassignmentProfitable(C, Domain: MaskDomain)) {
826 reassign(C, Domain: MaskDomain);
827 ++NumClosuresConverted;
828 Changed = true;
829 }
830 }
831
832 LLVM_DEBUG(
833 dbgs() << "***** Machine Function after Domain Reassignment *****\n");
834 LLVM_DEBUG(MF.print(dbgs()));
835
836 return Changed;
837}
838
839bool X86DomainReassignmentLegacy::runOnMachineFunction(MachineFunction &MF) {
840 if (skipFunction(F: MF.getFunction()))
841 return false;
842 X86DomainReassignmentImpl Impl;
843 return Impl.runOnMachineFunction(MF);
844}
845
846INITIALIZE_PASS(X86DomainReassignmentLegacy, "x86-domain-reassignment",
847 "X86 Domain Reassignment Pass", false, false)
848
849/// Returns an instance of the Domain Reassignment pass.
850FunctionPass *llvm::createX86DomainReassignmentLegacyPass() {
851 return new X86DomainReassignmentLegacy();
852}
853
854PreservedAnalyses
855X86DomainReassignmentPass::run(MachineFunction &MF,
856 MachineFunctionAnalysisManager &MFAM) {
857 X86DomainReassignmentImpl Impl;
858 bool Changed = Impl.runOnMachineFunction(MF);
859 if (!Changed)
860 return PreservedAnalyses::all();
861 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
862 PA.preserveSet<CFGAnalyses>();
863 return PA;
864}
865