1//===--- X86DomainReassignment.cpp - Selectively switch register classes---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass attempts to find instruction chains (closures) in one domain,
10// and convert them to equivalent instructions in a different domain,
11// if profitable.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86InstrInfo.h"
17#include "X86Subtarget.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/Printable.h"
29#include <bitset>
30
31using namespace llvm;
32
33#define DEBUG_TYPE "x86-domain-reassignment"
34
35STATISTIC(NumClosuresConverted, "Number of closures converted by the pass");
36
37static cl::opt<bool> DisableX86DomainReassignment(
38 "disable-x86-domain-reassignment", cl::Hidden,
39 cl::desc("X86: Disable Virtual Register Reassignment."), cl::init(Val: false));
40
41namespace {
42enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
43
44static bool isMask(const TargetRegisterClass *RC,
45 const TargetRegisterInfo *TRI) {
46 return X86::VK16RegClass.hasSubClassEq(RC);
47}
48
49static RegDomain getDomain(const TargetRegisterClass *RC,
50 const TargetRegisterInfo *TRI) {
51 if (TRI->isGeneralPurposeRegisterClass(RC))
52 return GPRDomain;
53 if (isMask(RC, TRI))
54 return MaskDomain;
55 return OtherDomain;
56}
57
58/// Return a register class equivalent to \p SrcRC, in \p Domain.
59static const TargetRegisterClass *getDstRC(const TargetRegisterClass *SrcRC,
60 RegDomain Domain) {
61 assert(Domain == MaskDomain && "add domain");
62 if (X86::GR8RegClass.hasSubClassEq(RC: SrcRC))
63 return &X86::VK8RegClass;
64 if (X86::GR16RegClass.hasSubClassEq(RC: SrcRC))
65 return &X86::VK16RegClass;
66 if (X86::GR32RegClass.hasSubClassEq(RC: SrcRC))
67 return &X86::VK32RegClass;
68 if (X86::GR64RegClass.hasSubClassEq(RC: SrcRC))
69 return &X86::VK64RegClass;
70 llvm_unreachable("add register class");
71 return nullptr;
72}
73
74/// Abstract Instruction Converter class.
75class InstrConverterBase {
76protected:
77 unsigned SrcOpcode;
78
79public:
80 InstrConverterBase(unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
81
82 virtual ~InstrConverterBase() = default;
83
84 /// \returns true if \p MI is legal to convert.
85 virtual bool isLegal(const MachineInstr *MI,
86 const TargetInstrInfo *TII) const {
87 assert(MI->getOpcode() == SrcOpcode &&
88 "Wrong instruction passed to converter");
89 return true;
90 }
91
92 /// Applies conversion to \p MI.
93 ///
94 /// \returns true if \p MI is no longer need, and can be deleted.
95 virtual bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
96 MachineRegisterInfo *MRI) const = 0;
97
98 /// \returns the cost increment incurred by converting \p MI.
99 virtual double getExtraCost(const MachineInstr *MI,
100 MachineRegisterInfo *MRI) const = 0;
101};
102
103/// An Instruction Converter which ignores the given instruction.
104/// For example, PHI instructions can be safely ignored since only the registers
105/// need to change.
106class InstrIgnore : public InstrConverterBase {
107public:
108 InstrIgnore(unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
109
110 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
111 MachineRegisterInfo *MRI) const override {
112 assert(isLegal(MI, TII) && "Cannot convert instruction");
113 return false;
114 }
115
116 double getExtraCost(const MachineInstr *MI,
117 MachineRegisterInfo *MRI) const override {
118 return 0;
119 }
120};
121
122/// An Instruction Converter which replaces an instruction with another.
123class InstrReplacer : public InstrConverterBase {
124public:
125 /// Opcode of the destination instruction.
126 unsigned DstOpcode;
127
128 InstrReplacer(unsigned SrcOpcode, unsigned DstOpcode)
129 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
130
131 bool isLegal(const MachineInstr *MI,
132 const TargetInstrInfo *TII) const override {
133 if (!InstrConverterBase::isLegal(MI, TII))
134 return false;
135 // It's illegal to replace an instruction that implicitly defines a register
136 // with an instruction that doesn't, unless that register dead.
137 for (const auto &MO : MI->implicit_operands())
138 if (MO.isReg() && MO.isDef() && !MO.isDead() &&
139 !TII->get(Opcode: DstOpcode).hasImplicitDefOfPhysReg(Reg: MO.getReg()))
140 return false;
141 return true;
142 }
143
144 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
145 MachineRegisterInfo *MRI) const override {
146 assert(isLegal(MI, TII) && "Cannot convert instruction");
147 MachineInstrBuilder Bld =
148 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: DstOpcode));
149 // Transfer explicit operands from original instruction. Implicit operands
150 // are handled by BuildMI.
151 for (auto &Op : MI->explicit_operands())
152 Bld.add(MO: Op);
153 return true;
154 }
155
156 double getExtraCost(const MachineInstr *MI,
157 MachineRegisterInfo *MRI) const override {
158 // Assuming instructions have the same cost.
159 return 0;
160 }
161};
162
163/// An Instruction Converter which replaces an instruction with another, and
164/// adds a COPY from the new instruction's destination to the old one's.
165class InstrReplacerDstCOPY : public InstrConverterBase {
166public:
167 unsigned DstOpcode;
168
169 InstrReplacerDstCOPY(unsigned SrcOpcode, unsigned DstOpcode)
170 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
171
172 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
173 MachineRegisterInfo *MRI) const override {
174 assert(isLegal(MI, TII) && "Cannot convert instruction");
175 MachineBasicBlock *MBB = MI->getParent();
176 const DebugLoc &DL = MI->getDebugLoc();
177
178 Register Reg =
179 MRI->createVirtualRegister(RegClass: TII->getRegClass(MCID: TII->get(Opcode: DstOpcode), OpNum: 0));
180 MachineInstrBuilder Bld = BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: DstOpcode), DestReg: Reg);
181 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
182 Bld.add(MO);
183
184 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
185 .add(MO: MI->getOperand(i: 0))
186 .addReg(RegNo: Reg);
187
188 return true;
189 }
190
191 double getExtraCost(const MachineInstr *MI,
192 MachineRegisterInfo *MRI) const override {
193 // Assuming instructions have the same cost, and that COPY is in the same
194 // domain so it will be eliminated.
195 return 0;
196 }
197};
198
199/// An Instruction Converter for replacing COPY instructions.
200class InstrCOPYReplacer : public InstrReplacer {
201public:
202 RegDomain DstDomain;
203
204 InstrCOPYReplacer(unsigned SrcOpcode, RegDomain DstDomain, unsigned DstOpcode)
205 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
206
207 bool isLegal(const MachineInstr *MI,
208 const TargetInstrInfo *TII) const override {
209 if (!InstrConverterBase::isLegal(MI, TII))
210 return false;
211
212 // Don't allow copies to/flow GR8/GR16 physical registers.
213 // FIXME: Is there some better way to support this?
214 Register DstReg = MI->getOperand(i: 0).getReg();
215 if (DstReg.isPhysical() && (X86::GR8RegClass.contains(Reg: DstReg) ||
216 X86::GR16RegClass.contains(Reg: DstReg)))
217 return false;
218 Register SrcReg = MI->getOperand(i: 1).getReg();
219 if (SrcReg.isPhysical() && (X86::GR8RegClass.contains(Reg: SrcReg) ||
220 X86::GR16RegClass.contains(Reg: SrcReg)))
221 return false;
222
223 return true;
224 }
225
226 double getExtraCost(const MachineInstr *MI,
227 MachineRegisterInfo *MRI) const override {
228 assert(MI->getOpcode() == TargetOpcode::COPY && "Expected a COPY");
229
230 for (const auto &MO : MI->operands()) {
231 // Physical registers will not be converted. Assume that converting the
232 // COPY to the destination domain will eventually result in a actual
233 // instruction.
234 if (MO.getReg().isPhysical())
235 return 1;
236
237 RegDomain OpDomain = getDomain(RC: MRI->getRegClass(Reg: MO.getReg()),
238 TRI: MRI->getTargetRegisterInfo());
239 // Converting a cross domain COPY to a same domain COPY should eliminate
240 // an insturction
241 if (OpDomain == DstDomain)
242 return -1;
243 }
244 return 0;
245 }
246};
247
248/// An Instruction Converter which replaces an instruction with a COPY.
249class InstrReplaceWithCopy : public InstrConverterBase {
250public:
251 // Source instruction operand Index, to be used as the COPY source.
252 unsigned SrcOpIdx;
253
254 InstrReplaceWithCopy(unsigned SrcOpcode, unsigned SrcOpIdx)
255 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
256
257 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
258 MachineRegisterInfo *MRI) const override {
259 assert(isLegal(MI, TII) && "Cannot convert instruction");
260 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
261 MCID: TII->get(Opcode: TargetOpcode::COPY))
262 .add(MOs: {MI->getOperand(i: 0), MI->getOperand(i: SrcOpIdx)});
263 return true;
264 }
265
266 double getExtraCost(const MachineInstr *MI,
267 MachineRegisterInfo *MRI) const override {
268 return 0;
269 }
270};
271
272// Key type to be used by the Instruction Converters map.
273// A converter is identified by <destination domain, source opcode>
274typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
275
276typedef DenseMap<InstrConverterBaseKeyTy, std::unique_ptr<InstrConverterBase>>
277 InstrConverterBaseMap;
278
279/// A closure is a set of virtual register representing all of the edges in
280/// the closure, as well as all of the instructions connected by those edges.
281///
282/// A closure may encompass virtual registers in the same register bank that
283/// have different widths. For example, it may contain 32-bit GPRs as well as
284/// 64-bit GPRs.
285///
286/// A closure that computes an address (i.e. defines a virtual register that is
287/// used in a memory operand) excludes the instructions that contain memory
288/// operands using the address. Such an instruction will be included in a
289/// different closure that manipulates the loaded or stored value.
290class Closure {
291private:
292 /// Virtual registers in the closure.
293 DenseSet<Register> Edges;
294
295 /// Instructions in the closure.
296 SmallVector<MachineInstr *, 8> Instrs;
297
298 /// Domains which this closure can legally be reassigned to.
299 std::bitset<NumDomains> LegalDstDomains;
300
301 /// An ID to uniquely identify this closure, even when it gets
302 /// moved around
303 unsigned ID;
304
305public:
306 Closure(unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) : ID(ID) {
307 for (RegDomain D : LegalDstDomainList)
308 LegalDstDomains.set(position: D);
309 }
310
311 /// Mark this closure as illegal for reassignment to all domains.
312 void setAllIllegal() { LegalDstDomains.reset(); }
313
314 /// \returns true if this closure has domains which are legal to reassign to.
315 bool hasLegalDstDomain() const { return LegalDstDomains.any(); }
316
317 /// \returns true if is legal to reassign this closure to domain \p RD.
318 bool isLegal(RegDomain RD) const { return LegalDstDomains[RD]; }
319
320 /// Mark this closure as illegal for reassignment to domain \p RD.
321 void setIllegal(RegDomain RD) { LegalDstDomains[RD] = false; }
322
323 bool empty() const { return Edges.empty(); }
324
325 bool insertEdge(Register Reg) { return Edges.insert(V: Reg).second; }
326
327 using const_edge_iterator = DenseSet<Register>::const_iterator;
328 iterator_range<const_edge_iterator> edges() const { return Edges; }
329
330 void addInstruction(MachineInstr *I) {
331 Instrs.push_back(Elt: I);
332 }
333
334 ArrayRef<MachineInstr *> instructions() const {
335 return Instrs;
336 }
337
338 LLVM_DUMP_METHOD void dump(const MachineRegisterInfo *MRI) const {
339 dbgs() << "Registers: ";
340 ListSeparator LS;
341 for (Register Reg : Edges)
342 dbgs() << LS << printReg(Reg, TRI: MRI->getTargetRegisterInfo(), SubIdx: 0, MRI);
343 dbgs() << "\n" << "Instructions:";
344 for (MachineInstr *MI : Instrs) {
345 dbgs() << "\n ";
346 MI->print(OS&: dbgs());
347 }
348 dbgs() << "\n";
349 }
350
351 unsigned getID() const {
352 return ID;
353 }
354
355};
356
357class X86DomainReassignmentImpl {
358public:
359 bool runOnMachineFunction(MachineFunction &MF);
360
361private:
362 const X86Subtarget *STI = nullptr;
363 MachineRegisterInfo *MRI = nullptr;
364 const X86InstrInfo *TII = nullptr;
365
366 /// All edges that are included in some closure
367 DenseMap<Register, unsigned> EnclosedEdges;
368
369 /// All instructions that are included in some closure.
370 DenseMap<MachineInstr *, unsigned> EnclosedInstrs;
371
372 /// A map of available Instruction Converters.
373 InstrConverterBaseMap Converters;
374
375 /// Initialize Converters map.
376 void initConverters();
377
378 /// Starting from \Reg, expand the closure as much as possible.
379 void buildClosure(Closure &, Register Reg);
380
381 /// Enqueue \p Reg to be considered for addition to the closure.
382 /// Return false if the closure becomes invalid.
383 bool visitRegister(Closure &, Register Reg, RegDomain &Domain,
384 SmallVectorImpl<Register> &Worklist);
385
386 /// Reassign the closure to \p Domain.
387 void reassign(const Closure &C, RegDomain Domain) const;
388
389 /// Add \p MI to the closure.
390 /// Return false if the closure becomes invalid.
391 bool encloseInstr(Closure &C, MachineInstr *MI);
392
393 /// /returns true if it is profitable to reassign the closure to \p Domain.
394 bool isReassignmentProfitable(const Closure &C, RegDomain Domain) const;
395
396 /// Calculate the total cost of reassigning the closure to \p Domain.
397 double calculateCost(const Closure &C, RegDomain Domain) const;
398};
399
400class X86DomainReassignmentLegacy : public MachineFunctionPass {
401public:
402 static char ID;
403
404 X86DomainReassignmentLegacy() : MachineFunctionPass(ID) {}
405
406 bool runOnMachineFunction(MachineFunction &MF) override;
407
408 void getAnalysisUsage(AnalysisUsage &AU) const override {
409 AU.setPreservesCFG();
410 MachineFunctionPass::getAnalysisUsage(AU);
411 }
412
413 StringRef getPassName() const override {
414 return "X86 Domain Reassignment Pass";
415 }
416};
417
418char X86DomainReassignmentLegacy::ID = 0;
419
420} // End anonymous namespace.
421
422bool X86DomainReassignmentImpl::visitRegister(
423 Closure &C, Register Reg, RegDomain &Domain,
424 SmallVectorImpl<Register> &Worklist) {
425 if (!Reg.isVirtual())
426 return true;
427
428 auto I = EnclosedEdges.find(Val: Reg);
429 if (I != EnclosedEdges.end()) {
430 if (I->second != C.getID()) {
431 C.setAllIllegal();
432 return false;
433 }
434 return true;
435 }
436
437 if (!MRI->hasOneDef(RegNo: Reg))
438 return true;
439
440 RegDomain RD = getDomain(RC: MRI->getRegClass(Reg), TRI: MRI->getTargetRegisterInfo());
441 // First edge in closure sets the domain.
442 if (Domain == NoDomain)
443 Domain = RD;
444
445 if (Domain != RD)
446 return true;
447
448 Worklist.push_back(Elt: Reg);
449 return true;
450}
451
452bool X86DomainReassignmentImpl::encloseInstr(Closure &C, MachineInstr *MI) {
453 auto [I, Inserted] = EnclosedInstrs.try_emplace(Key: MI, Args: C.getID());
454 if (!Inserted) {
455 if (I->second != C.getID()) {
456 // Instruction already belongs to another closure, avoid conflicts between
457 // closure and mark this closure as illegal.
458 C.setAllIllegal();
459 return false;
460 }
461 return true;
462 }
463
464 C.addInstruction(I: MI);
465
466 // Mark closure as illegal for reassignment to domains, if there is no
467 // converter for the instruction or if the converter cannot convert the
468 // instruction.
469 for (int i = 0; i != NumDomains; ++i) {
470 if (C.isLegal(RD: (RegDomain)i)) {
471 auto I = Converters.find(Val: {i, MI->getOpcode()});
472 if (I == Converters.end() || !I->second->isLegal(MI, TII))
473 C.setIllegal((RegDomain)i);
474 }
475 }
476 return C.hasLegalDstDomain();
477}
478
479double X86DomainReassignmentImpl::calculateCost(const Closure &C,
480 RegDomain DstDomain) const {
481 assert(C.isLegal(DstDomain) && "Cannot calculate cost for illegal closure");
482
483 double Cost = 0.0;
484 for (auto *MI : C.instructions())
485 Cost += Converters.find(Val: {DstDomain, MI->getOpcode()})
486 ->second->getExtraCost(MI, MRI);
487 return Cost;
488}
489
490bool X86DomainReassignmentImpl::isReassignmentProfitable(
491 const Closure &C, RegDomain Domain) const {
492 return calculateCost(C, DstDomain: Domain) < 0.0;
493}
494
495void X86DomainReassignmentImpl::reassign(const Closure &C,
496 RegDomain Domain) const {
497 assert(C.isLegal(Domain) && "Cannot convert illegal closure");
498
499 // Iterate all instructions in the closure, convert each one using the
500 // appropriate converter.
501 SmallVector<MachineInstr *, 8> ToErase;
502 for (auto *MI : C.instructions())
503 if (Converters.find(Val: {Domain, MI->getOpcode()})
504 ->second->convertInstr(MI, TII, MRI))
505 ToErase.push_back(Elt: MI);
506
507 // Iterate all registers in the closure, replace them with registers in the
508 // destination domain.
509 for (Register Reg : C.edges()) {
510 MRI->setRegClass(Reg, RC: getDstRC(SrcRC: MRI->getRegClass(Reg), Domain));
511 for (auto &MO : MRI->use_operands(Reg)) {
512 if (MO.isReg())
513 // Remove all subregister references as they are not valid in the
514 // destination domain.
515 MO.setSubReg(0);
516 }
517 }
518
519 for (auto *MI : ToErase)
520 MI->eraseFromParent();
521}
522
523/// \returns true when \p Reg is used as part of an address calculation in \p
524/// MI.
525static bool usedAsAddr(const MachineInstr &MI, Register Reg,
526 const TargetInstrInfo *TII) {
527 if (!MI.mayLoadOrStore())
528 return false;
529
530 const MCInstrDesc &Desc = TII->get(Opcode: MI.getOpcode());
531 int MemOpStart = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
532 if (MemOpStart == -1)
533 return false;
534
535 MemOpStart += X86II::getOperandBias(Desc);
536 for (unsigned MemOpIdx = MemOpStart,
537 MemOpEnd = MemOpStart + X86::AddrNumOperands;
538 MemOpIdx < MemOpEnd; ++MemOpIdx) {
539 const MachineOperand &Op = MI.getOperand(i: MemOpIdx);
540 if (Op.isReg() && Op.getReg() == Reg)
541 return true;
542 }
543 return false;
544}
545
546void X86DomainReassignmentImpl::buildClosure(Closure &C, Register Reg) {
547 SmallVector<Register, 4> Worklist;
548 RegDomain Domain = NoDomain;
549 visitRegister(C, Reg, Domain, Worklist);
550 while (!Worklist.empty()) {
551 Register CurReg = Worklist.pop_back_val();
552
553 // Register already in this closure.
554 if (!C.insertEdge(Reg: CurReg))
555 continue;
556 EnclosedEdges[Reg] = C.getID();
557
558 MachineInstr *DefMI = MRI->getVRegDef(Reg: CurReg);
559 if (!encloseInstr(C, MI: DefMI))
560 return;
561
562 // Add register used by the defining MI to the worklist.
563 // Do not add registers which are used in address calculation, they will be
564 // added to a different closure.
565 int OpEnd = DefMI->getNumOperands();
566 const MCInstrDesc &Desc = DefMI->getDesc();
567 int MemOp = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
568 if (MemOp != -1)
569 MemOp += X86II::getOperandBias(Desc);
570 for (int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {
571 if (OpIdx == MemOp) {
572 // skip address calculation.
573 OpIdx += (X86::AddrNumOperands - 1);
574 continue;
575 }
576 auto &Op = DefMI->getOperand(i: OpIdx);
577 if (!Op.isReg() || !Op.isUse())
578 continue;
579 if (!visitRegister(C, Reg: Op.getReg(), Domain, Worklist))
580 return;
581 }
582
583 // Expand closure through register uses.
584 for (auto &UseMI : MRI->use_nodbg_instructions(Reg: CurReg)) {
585 // We would like to avoid converting closures which calculare addresses,
586 // as this should remain in GPRs.
587 if (usedAsAddr(MI: UseMI, Reg: CurReg, TII)) {
588 C.setAllIllegal();
589 return;
590 }
591 if (!encloseInstr(C, MI: &UseMI))
592 return;
593
594 for (auto &DefOp : UseMI.defs()) {
595 if (!DefOp.isReg())
596 continue;
597
598 Register DefReg = DefOp.getReg();
599 if (!DefReg.isVirtual()) {
600 C.setAllIllegal();
601 return;
602 }
603 if (!visitRegister(C, Reg: DefReg, Domain, Worklist))
604 return;
605 }
606 }
607 }
608}
609
610void X86DomainReassignmentImpl::initConverters() {
611 Converters[{MaskDomain, TargetOpcode::PHI}] =
612 std::make_unique<InstrIgnore>(args: TargetOpcode::PHI);
613
614 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
615 std::make_unique<InstrIgnore>(args: TargetOpcode::IMPLICIT_DEF);
616
617 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
618 std::make_unique<InstrReplaceWithCopy>(args: TargetOpcode::INSERT_SUBREG, args: 2);
619
620 Converters[{MaskDomain, TargetOpcode::COPY}] =
621 std::make_unique<InstrCOPYReplacer>(args: TargetOpcode::COPY, args: MaskDomain,
622 args: TargetOpcode::COPY);
623
624 auto createReplacerDstCOPY = [&](unsigned From, unsigned To) {
625 Converters[{MaskDomain, From}] =
626 std::make_unique<InstrReplacerDstCOPY>(args&: From, args&: To);
627 };
628
629#define GET_EGPR_IF_ENABLED(OPC) STI->hasEGPR() ? OPC##_EVEX : OPC
630 createReplacerDstCOPY(X86::MOVZX32rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
631 createReplacerDstCOPY(X86::MOVZX64rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
632
633 createReplacerDstCOPY(X86::MOVZX32rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
634 createReplacerDstCOPY(X86::MOVZX64rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
635
636 if (STI->hasDQI()) {
637 createReplacerDstCOPY(X86::MOVZX16rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
638 createReplacerDstCOPY(X86::MOVZX32rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
639 createReplacerDstCOPY(X86::MOVZX64rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
640
641 createReplacerDstCOPY(X86::MOVZX16rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
642 createReplacerDstCOPY(X86::MOVZX32rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
643 createReplacerDstCOPY(X86::MOVZX64rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
644 }
645
646 auto createReplacer = [&](unsigned From, unsigned To) {
647 Converters[{MaskDomain, From}] = std::make_unique<InstrReplacer>(args&: From, args&: To);
648 };
649
650 createReplacer(X86::MOV16rm, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
651 createReplacer(X86::MOV16mr, GET_EGPR_IF_ENABLED(X86::KMOVWmk));
652 createReplacer(X86::MOV16rr, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
653 createReplacer(X86::SHR16ri, X86::KSHIFTRWki);
654 createReplacer(X86::SHL16ri, X86::KSHIFTLWki);
655 createReplacer(X86::NOT16r, X86::KNOTWkk);
656 createReplacer(X86::OR16rr, X86::KORWkk);
657 createReplacer(X86::AND16rr, X86::KANDWkk);
658 createReplacer(X86::XOR16rr, X86::KXORWkk);
659
660 bool HasNDD = STI->hasNDD();
661 if (HasNDD) {
662 createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWki);
663 createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWki);
664 createReplacer(X86::NOT16r_ND, X86::KNOTWkk);
665 createReplacer(X86::OR16rr_ND, X86::KORWkk);
666 createReplacer(X86::AND16rr_ND, X86::KANDWkk);
667 createReplacer(X86::XOR16rr_ND, X86::KXORWkk);
668 }
669
670 if (STI->hasBWI()) {
671 createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm));
672 createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm));
673
674 createReplacer(X86::MOV32mr, GET_EGPR_IF_ENABLED(X86::KMOVDmk));
675 createReplacer(X86::MOV64mr, GET_EGPR_IF_ENABLED(X86::KMOVQmk));
676
677 createReplacer(X86::MOV32rr, GET_EGPR_IF_ENABLED(X86::KMOVDkk));
678 createReplacer(X86::MOV64rr, GET_EGPR_IF_ENABLED(X86::KMOVQkk));
679
680 createReplacer(X86::SHR32ri, X86::KSHIFTRDki);
681 createReplacer(X86::SHR64ri, X86::KSHIFTRQki);
682
683 createReplacer(X86::SHL32ri, X86::KSHIFTLDki);
684 createReplacer(X86::SHL64ri, X86::KSHIFTLQki);
685
686 createReplacer(X86::ADD32rr, X86::KADDDkk);
687 createReplacer(X86::ADD64rr, X86::KADDQkk);
688
689 createReplacer(X86::NOT32r, X86::KNOTDkk);
690 createReplacer(X86::NOT64r, X86::KNOTQkk);
691
692 createReplacer(X86::OR32rr, X86::KORDkk);
693 createReplacer(X86::OR64rr, X86::KORQkk);
694
695 createReplacer(X86::AND32rr, X86::KANDDkk);
696 createReplacer(X86::AND64rr, X86::KANDQkk);
697
698 createReplacer(X86::ANDN32rr, X86::KANDNDkk);
699 createReplacer(X86::ANDN64rr, X86::KANDNQkk);
700
701 createReplacer(X86::XOR32rr, X86::KXORDkk);
702 createReplacer(X86::XOR64rr, X86::KXORQkk);
703
704 if (HasNDD) {
705 createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDki);
706 createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDki);
707 createReplacer(X86::ADD32rr_ND, X86::KADDDkk);
708 createReplacer(X86::NOT32r_ND, X86::KNOTDkk);
709 createReplacer(X86::OR32rr_ND, X86::KORDkk);
710 createReplacer(X86::AND32rr_ND, X86::KANDDkk);
711 createReplacer(X86::XOR32rr_ND, X86::KXORDkk);
712 createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQki);
713 createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQki);
714 createReplacer(X86::ADD64rr_ND, X86::KADDQkk);
715 createReplacer(X86::NOT64r_ND, X86::KNOTQkk);
716 createReplacer(X86::OR64rr_ND, X86::KORQkk);
717 createReplacer(X86::AND64rr_ND, X86::KANDQkk);
718 createReplacer(X86::XOR64rr_ND, X86::KXORQkk);
719 }
720
721 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
722 // to prove only Z flag is used.
723 // createReplacer(X86::TEST32rr, X86::KTESTDkk);
724 // createReplacer(X86::TEST64rr, X86::KTESTQkk);
725 }
726
727 if (STI->hasDQI()) {
728 createReplacer(X86::ADD8rr, X86::KADDBkk);
729 createReplacer(X86::ADD16rr, X86::KADDWkk);
730
731 createReplacer(X86::AND8rr, X86::KANDBkk);
732
733 createReplacer(X86::MOV8rm, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
734 createReplacer(X86::MOV8mr, GET_EGPR_IF_ENABLED(X86::KMOVBmk));
735 createReplacer(X86::MOV8rr, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
736
737 createReplacer(X86::NOT8r, X86::KNOTBkk);
738
739 createReplacer(X86::OR8rr, X86::KORBkk);
740
741 createReplacer(X86::SHR8ri, X86::KSHIFTRBki);
742 createReplacer(X86::SHL8ri, X86::KSHIFTLBki);
743
744 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
745 // to prove only Z flag is used.
746 // createReplacer(X86::TEST8rr, X86::KTESTBkk);
747 // createReplacer(X86::TEST16rr, X86::KTESTWkk);
748
749 createReplacer(X86::XOR8rr, X86::KXORBkk);
750
751 if (HasNDD) {
752 createReplacer(X86::ADD8rr_ND, X86::KADDBkk);
753 createReplacer(X86::ADD16rr_ND, X86::KADDWkk);
754 createReplacer(X86::AND8rr_ND, X86::KANDBkk);
755 createReplacer(X86::NOT8r_ND, X86::KNOTBkk);
756 createReplacer(X86::OR8rr_ND, X86::KORBkk);
757 createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBki);
758 createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBki);
759 createReplacer(X86::XOR8rr_ND, X86::KXORBkk);
760 }
761 }
762#undef GET_EGPR_IF_ENABLED
763}
764
765bool X86DomainReassignmentImpl::runOnMachineFunction(MachineFunction &MF) {
766 if (DisableX86DomainReassignment)
767 return false;
768
769 LLVM_DEBUG(
770 dbgs() << "***** Machine Function before Domain Reassignment *****\n");
771 LLVM_DEBUG(MF.print(dbgs()));
772
773 STI = &MF.getSubtarget<X86Subtarget>();
774 // GPR->K is the only transformation currently supported, bail out early if no
775 // AVX512.
776 // TODO: We're also bailing of AVX512BW isn't supported since we use VK32 and
777 // VK64 for GR32/GR64, but those aren't legal classes on KNL. If the register
778 // coalescer doesn't clean it up and we generate a spill we will crash.
779 if (!STI->hasAVX512() || !STI->hasBWI())
780 return false;
781
782 MRI = &MF.getRegInfo();
783 assert(MRI->isSSA() && "Expected MIR to be in SSA form");
784
785 TII = STI->getInstrInfo();
786 initConverters();
787 bool Changed = false;
788
789 EnclosedEdges.clear();
790 EnclosedInstrs.clear();
791
792 std::vector<Closure> Closures;
793
794 // Go over all virtual registers and calculate a closure.
795 unsigned ClosureID = 0;
796 for (unsigned Idx = 0; Idx < MRI->getNumVirtRegs(); ++Idx) {
797 Register Reg = Register::index2VirtReg(Index: Idx);
798
799 // Skip unused VRegs.
800 if (MRI->reg_nodbg_empty(RegNo: Reg))
801 continue;
802
803 // GPR only current source domain supported.
804 if (!MRI->getTargetRegisterInfo()->isGeneralPurposeRegisterClass(
805 RC: MRI->getRegClass(Reg)))
806 continue;
807
808 // Register already in closure.
809 if (EnclosedEdges.contains(Val: Reg))
810 continue;
811
812 // Calculate closure starting with Reg.
813 Closure C(ClosureID++, {MaskDomain});
814 buildClosure(C, Reg);
815
816 // Collect all closures that can potentially be converted.
817 if (!C.empty() && C.isLegal(RD: MaskDomain))
818 Closures.push_back(x: std::move(C));
819 }
820
821 for (Closure &C : Closures) {
822 LLVM_DEBUG(C.dump(MRI));
823 if (isReassignmentProfitable(C, Domain: MaskDomain)) {
824 reassign(C, Domain: MaskDomain);
825 ++NumClosuresConverted;
826 Changed = true;
827 }
828 }
829
830 LLVM_DEBUG(
831 dbgs() << "***** Machine Function after Domain Reassignment *****\n");
832 LLVM_DEBUG(MF.print(dbgs()));
833
834 return Changed;
835}
836
837bool X86DomainReassignmentLegacy::runOnMachineFunction(MachineFunction &MF) {
838 if (skipFunction(F: MF.getFunction()))
839 return false;
840 X86DomainReassignmentImpl Impl;
841 return Impl.runOnMachineFunction(MF);
842}
843
844INITIALIZE_PASS(X86DomainReassignmentLegacy, "x86-domain-reassignment",
845 "X86 Domain Reassignment Pass", false, false)
846
847/// Returns an instance of the Domain Reassignment pass.
848FunctionPass *llvm::createX86DomainReassignmentLegacyPass() {
849 return new X86DomainReassignmentLegacy();
850}
851
852PreservedAnalyses
853X86DomainReassignmentPass::run(MachineFunction &MF,
854 MachineFunctionAnalysisManager &MFAM) {
855 X86DomainReassignmentImpl Impl;
856 bool Changed = Impl.runOnMachineFunction(MF);
857 if (!Changed)
858 return PreservedAnalyses::all();
859 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
860 PA.preserveSet<CFGAnalyses>();
861 return PA;
862}
863