1//===--- X86DomainReassignment.cpp - Selectively switch register classes---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass attempts to find instruction chains (closures) in one domain,
10// and convert them to equivalent instructions in a different domain,
11// if profitable.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86InstrInfo.h"
17#include "X86Subtarget.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/Printable.h"
29#include <bitset>
30
31using namespace llvm;
32
33#define DEBUG_TYPE "x86-domain-reassignment"
34
35STATISTIC(NumClosuresConverted, "Number of closures converted by the pass");
36STATISTIC(NumClosuresBuilt, "Number of closures built by the pass");
37
38static cl::opt<bool> DisableX86DomainReassignment(
39 "disable-x86-domain-reassignment", cl::Hidden,
40 cl::desc("X86: Disable Virtual Register Reassignment."), cl::init(Val: false));
41
42namespace {
43enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
44
45static bool isMask(const TargetRegisterClass *RC,
46 const TargetRegisterInfo *TRI) {
47 return X86::VK16RegClass.hasSubClassEq(RC);
48}
49
50static RegDomain getDomain(const TargetRegisterClass *RC,
51 const TargetRegisterInfo *TRI) {
52 if (TRI->isGeneralPurposeRegisterClass(RC))
53 return GPRDomain;
54 if (isMask(RC, TRI))
55 return MaskDomain;
56 return OtherDomain;
57}
58
59/// Return a register class equivalent to \p SrcRC, in \p Domain.
60static const TargetRegisterClass *getDstRC(const TargetRegisterClass *SrcRC,
61 RegDomain Domain) {
62 assert(Domain == MaskDomain && "add domain");
63 if (X86::GR8RegClass.hasSubClassEq(RC: SrcRC))
64 return &X86::VK8RegClass;
65 if (X86::GR16RegClass.hasSubClassEq(RC: SrcRC))
66 return &X86::VK16RegClass;
67 if (X86::GR32RegClass.hasSubClassEq(RC: SrcRC))
68 return &X86::VK32RegClass;
69 if (X86::GR64RegClass.hasSubClassEq(RC: SrcRC))
70 return &X86::VK64RegClass;
71 llvm_unreachable("add register class");
72 return nullptr;
73}
74
75/// Abstract Instruction Converter class.
76class InstrConverterBase {
77protected:
78 unsigned SrcOpcode;
79
80public:
81 InstrConverterBase(unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
82
83 virtual ~InstrConverterBase() = default;
84
85 /// \returns true if \p MI is legal to convert.
86 virtual bool isLegal(const MachineInstr *MI,
87 const TargetInstrInfo *TII) const {
88 assert(MI->getOpcode() == SrcOpcode &&
89 "Wrong instruction passed to converter");
90 return true;
91 }
92
93 /// Applies conversion to \p MI.
94 ///
95 /// \returns true if \p MI is no longer need, and can be deleted.
96 virtual bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
97 MachineRegisterInfo *MRI) const = 0;
98
99 /// \returns the cost increment incurred by converting \p MI.
100 virtual double getExtraCost(const MachineInstr *MI,
101 MachineRegisterInfo *MRI) const = 0;
102};
103
104/// An Instruction Converter which ignores the given instruction.
105/// For example, PHI instructions can be safely ignored since only the registers
106/// need to change.
107class InstrIgnore : public InstrConverterBase {
108public:
109 InstrIgnore(unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
110
111 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
112 MachineRegisterInfo *MRI) const override {
113 assert(isLegal(MI, TII) && "Cannot convert instruction");
114 return false;
115 }
116
117 double getExtraCost(const MachineInstr *MI,
118 MachineRegisterInfo *MRI) const override {
119 return 0;
120 }
121};
122
123/// An Instruction Converter which replaces an instruction with another.
124class InstrReplacer : public InstrConverterBase {
125public:
126 /// Opcode of the destination instruction.
127 unsigned DstOpcode;
128
129 InstrReplacer(unsigned SrcOpcode, unsigned DstOpcode)
130 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
131
132 bool isLegal(const MachineInstr *MI,
133 const TargetInstrInfo *TII) const override {
134 if (!InstrConverterBase::isLegal(MI, TII))
135 return false;
136 // It's illegal to replace an instruction that implicitly defines a register
137 // with an instruction that doesn't, unless that register dead.
138 for (const auto &MO : MI->implicit_operands())
139 if (MO.isReg() && MO.isDef() && !MO.isDead() &&
140 !TII->get(Opcode: DstOpcode).hasImplicitDefOfPhysReg(Reg: MO.getReg()))
141 return false;
142 return true;
143 }
144
145 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
146 MachineRegisterInfo *MRI) const override {
147 assert(isLegal(MI, TII) && "Cannot convert instruction");
148 MachineInstrBuilder Bld =
149 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: DstOpcode));
150 // Transfer explicit operands from original instruction. Implicit operands
151 // are handled by BuildMI.
152 for (auto &Op : MI->explicit_operands())
153 Bld.add(MO: Op);
154 return true;
155 }
156
157 double getExtraCost(const MachineInstr *MI,
158 MachineRegisterInfo *MRI) const override {
159 // Assuming instructions have the same cost.
160 return 0;
161 }
162};
163
164/// An Instruction Converter which replaces an instruction with another, and
165/// adds a COPY from the new instruction's destination to the old one's.
166class InstrReplacerDstCOPY : public InstrConverterBase {
167public:
168 unsigned DstOpcode;
169
170 InstrReplacerDstCOPY(unsigned SrcOpcode, unsigned DstOpcode)
171 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
172
173 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
174 MachineRegisterInfo *MRI) const override {
175 assert(isLegal(MI, TII) && "Cannot convert instruction");
176 MachineBasicBlock *MBB = MI->getParent();
177 const DebugLoc &DL = MI->getDebugLoc();
178
179 Register Reg =
180 MRI->createVirtualRegister(RegClass: TII->getRegClass(MCID: TII->get(Opcode: DstOpcode), OpNum: 0));
181 MachineInstrBuilder Bld = BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: DstOpcode), DestReg: Reg);
182 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
183 Bld.add(MO);
184
185 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
186 .add(MO: MI->getOperand(i: 0))
187 .addReg(RegNo: Reg);
188
189 return true;
190 }
191
192 double getExtraCost(const MachineInstr *MI,
193 MachineRegisterInfo *MRI) const override {
194 // Assuming instructions have the same cost, and that COPY is in the same
195 // domain so it will be eliminated.
196 return 0;
197 }
198};
199
200/// An Instruction Converter for replacing COPY instructions.
201class InstrCOPYReplacer : public InstrReplacer {
202public:
203 RegDomain DstDomain;
204
205 InstrCOPYReplacer(unsigned SrcOpcode, RegDomain DstDomain, unsigned DstOpcode)
206 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
207
208 bool isLegal(const MachineInstr *MI,
209 const TargetInstrInfo *TII) const override {
210 if (!InstrConverterBase::isLegal(MI, TII))
211 return false;
212
213 // Don't allow copies to/flow GR8/GR16 physical registers.
214 // FIXME: Is there some better way to support this?
215 Register DstReg = MI->getOperand(i: 0).getReg();
216 if (DstReg.isPhysical() && (X86::GR8RegClass.contains(Reg: DstReg) ||
217 X86::GR16RegClass.contains(Reg: DstReg)))
218 return false;
219 Register SrcReg = MI->getOperand(i: 1).getReg();
220 if (SrcReg.isPhysical() && (X86::GR8RegClass.contains(Reg: SrcReg) ||
221 X86::GR16RegClass.contains(Reg: SrcReg)))
222 return false;
223
224 return true;
225 }
226
227 double getExtraCost(const MachineInstr *MI,
228 MachineRegisterInfo *MRI) const override {
229 assert(MI->getOpcode() == TargetOpcode::COPY && "Expected a COPY");
230
231 for (const auto &MO : MI->operands()) {
232 // Physical registers will not be converted. Assume that converting the
233 // COPY to the destination domain will eventually result in a actual
234 // instruction.
235 if (MO.getReg().isPhysical())
236 return 1;
237
238 RegDomain OpDomain = getDomain(RC: MRI->getRegClass(Reg: MO.getReg()),
239 TRI: MRI->getTargetRegisterInfo());
240 // Converting a cross domain COPY to a same domain COPY should eliminate
241 // an insturction
242 if (OpDomain == DstDomain)
243 return -1;
244 }
245 return 0;
246 }
247};
248
249/// An Instruction Converter which replaces an instruction with a COPY.
250class InstrReplaceWithCopy : public InstrConverterBase {
251public:
252 // Source instruction operand Index, to be used as the COPY source.
253 unsigned SrcOpIdx;
254
255 InstrReplaceWithCopy(unsigned SrcOpcode, unsigned SrcOpIdx)
256 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
257
258 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
259 MachineRegisterInfo *MRI) const override {
260 assert(isLegal(MI, TII) && "Cannot convert instruction");
261 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
262 MCID: TII->get(Opcode: TargetOpcode::COPY))
263 .add(MOs: {MI->getOperand(i: 0), MI->getOperand(i: SrcOpIdx)});
264 return true;
265 }
266
267 double getExtraCost(const MachineInstr *MI,
268 MachineRegisterInfo *MRI) const override {
269 return 0;
270 }
271};
272
273// Key type to be used by the Instruction Converters map.
274// A converter is identified by <destination domain, source opcode>
275typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
276
277typedef DenseMap<InstrConverterBaseKeyTy, std::unique_ptr<InstrConverterBase>>
278 InstrConverterBaseMap;
279
280/// A closure is a set of virtual register representing all of the edges in
281/// the closure, as well as all of the instructions connected by those edges.
282///
283/// A closure may encompass virtual registers in the same register bank that
284/// have different widths. For example, it may contain 32-bit GPRs as well as
285/// 64-bit GPRs.
286///
287/// A closure that computes an address (i.e. defines a virtual register that is
288/// used in a memory operand) excludes the instructions that contain memory
289/// operands using the address. Such an instruction will be included in a
290/// different closure that manipulates the loaded or stored value.
291class Closure {
292private:
293 /// Virtual registers in the closure.
294 DenseSet<Register> Edges;
295
296 /// Instructions in the closure.
297 SmallVector<MachineInstr *, 8> Instrs;
298
299 /// Domains which this closure can legally be reassigned to.
300 std::bitset<NumDomains> LegalDstDomains;
301
302 /// An ID to uniquely identify this closure, even when it gets
303 /// moved around
304 unsigned ID;
305
306public:
307 Closure(unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) : ID(ID) {
308 for (RegDomain D : LegalDstDomainList)
309 LegalDstDomains.set(position: D);
310 }
311
312 /// Mark this closure as illegal for reassignment to all domains.
313 void setAllIllegal() { LegalDstDomains.reset(); }
314
315 /// \returns true if this closure has domains which are legal to reassign to.
316 bool hasLegalDstDomain() const { return LegalDstDomains.any(); }
317
318 /// \returns true if is legal to reassign this closure to domain \p RD.
319 bool isLegal(RegDomain RD) const { return LegalDstDomains[RD]; }
320
321 /// Mark this closure as illegal for reassignment to domain \p RD.
322 void setIllegal(RegDomain RD) { LegalDstDomains[RD] = false; }
323
324 bool empty() const { return Edges.empty(); }
325
326 bool insertEdge(Register Reg) { return Edges.insert(V: Reg).second; }
327
328 using const_edge_iterator = DenseSet<Register>::const_iterator;
329 iterator_range<const_edge_iterator> edges() const { return Edges; }
330
331 void addInstruction(MachineInstr *I) {
332 Instrs.push_back(Elt: I);
333 }
334
335 ArrayRef<MachineInstr *> instructions() const {
336 return Instrs;
337 }
338
339 LLVM_DUMP_METHOD void dump(const MachineRegisterInfo *MRI) const {
340 dbgs() << "Registers: ";
341 ListSeparator LS;
342 for (Register Reg : Edges)
343 dbgs() << LS << printReg(Reg, TRI: MRI->getTargetRegisterInfo(), SubIdx: 0, MRI);
344 dbgs() << "\n" << "Instructions:";
345 for (MachineInstr *MI : Instrs) {
346 dbgs() << "\n ";
347 MI->print(OS&: dbgs());
348 }
349 dbgs() << "\n";
350 }
351
352 unsigned getID() const {
353 return ID;
354 }
355
356};
357
358class X86DomainReassignmentImpl {
359public:
360 bool runOnMachineFunction(MachineFunction &MF);
361
362private:
363 const X86Subtarget *STI = nullptr;
364 MachineRegisterInfo *MRI = nullptr;
365 const X86InstrInfo *TII = nullptr;
366
367 /// All edges that are included in some closure
368 DenseMap<Register, unsigned> EnclosedEdges;
369
370 /// All instructions that are included in some closure.
371 DenseMap<MachineInstr *, unsigned> EnclosedInstrs;
372
373 /// A map of available Instruction Converters.
374 InstrConverterBaseMap Converters;
375
376 /// Initialize Converters map.
377 void initConverters();
378
379 /// Starting from \Reg, expand the closure as much as possible.
380 void buildClosure(Closure &, Register Reg);
381
382 /// Enqueue \p Reg to be considered for addition to the closure.
383 /// Return false if the closure becomes invalid.
384 bool visitRegister(Closure &, Register Reg, RegDomain &Domain,
385 SmallVectorImpl<Register> &Worklist);
386
387 /// Reassign the closure to \p Domain.
388 void reassign(const Closure &C, RegDomain Domain) const;
389
390 /// Add \p MI to the closure.
391 /// Return false if the closure becomes invalid.
392 bool encloseInstr(Closure &C, MachineInstr *MI);
393
394 /// /returns true if it is profitable to reassign the closure to \p Domain.
395 bool isReassignmentProfitable(const Closure &C, RegDomain Domain) const;
396
397 /// Calculate the total cost of reassigning the closure to \p Domain.
398 double calculateCost(const Closure &C, RegDomain Domain) const;
399};
400
401class X86DomainReassignmentLegacy : public MachineFunctionPass {
402public:
403 static char ID;
404
405 X86DomainReassignmentLegacy() : MachineFunctionPass(ID) {}
406
407 bool runOnMachineFunction(MachineFunction &MF) override;
408
409 void getAnalysisUsage(AnalysisUsage &AU) const override {
410 AU.setPreservesCFG();
411 MachineFunctionPass::getAnalysisUsage(AU);
412 }
413
414 StringRef getPassName() const override {
415 return "X86 Domain Reassignment Pass";
416 }
417};
418
419char X86DomainReassignmentLegacy::ID = 0;
420
421} // End anonymous namespace.
422
423bool X86DomainReassignmentImpl::visitRegister(
424 Closure &C, Register Reg, RegDomain &Domain,
425 SmallVectorImpl<Register> &Worklist) {
426 if (!Reg.isVirtual())
427 return true;
428
429 auto I = EnclosedEdges.find(Val: Reg);
430 if (I != EnclosedEdges.end()) {
431 if (I->second != C.getID()) {
432 C.setAllIllegal();
433 return false;
434 }
435 return true;
436 }
437
438 if (!MRI->hasOneDef(RegNo: Reg))
439 return true;
440
441 RegDomain RD = getDomain(RC: MRI->getRegClass(Reg), TRI: MRI->getTargetRegisterInfo());
442 // First edge in closure sets the domain.
443 if (Domain == NoDomain)
444 Domain = RD;
445
446 if (Domain != RD)
447 return true;
448
449 Worklist.push_back(Elt: Reg);
450 return true;
451}
452
453bool X86DomainReassignmentImpl::encloseInstr(Closure &C, MachineInstr *MI) {
454 auto [I, Inserted] = EnclosedInstrs.try_emplace(Key: MI, Args: C.getID());
455 if (!Inserted) {
456 if (I->second != C.getID()) {
457 // Instruction already belongs to another closure, avoid conflicts between
458 // closure and mark this closure as illegal.
459 C.setAllIllegal();
460 return false;
461 }
462 return true;
463 }
464
465 C.addInstruction(I: MI);
466
467 // Mark closure as illegal for reassignment to domains, if there is no
468 // converter for the instruction or if the converter cannot convert the
469 // instruction.
470 for (int i = 0; i != NumDomains; ++i) {
471 if (C.isLegal(RD: (RegDomain)i)) {
472 auto I = Converters.find(Val: {i, MI->getOpcode()});
473 if (I == Converters.end() || !I->second->isLegal(MI, TII))
474 C.setIllegal((RegDomain)i);
475 }
476 }
477 return C.hasLegalDstDomain();
478}
479
480double X86DomainReassignmentImpl::calculateCost(const Closure &C,
481 RegDomain DstDomain) const {
482 assert(C.isLegal(DstDomain) && "Cannot calculate cost for illegal closure");
483
484 double Cost = 0.0;
485 for (auto *MI : C.instructions())
486 Cost += Converters.find(Val: {DstDomain, MI->getOpcode()})
487 ->second->getExtraCost(MI, MRI);
488 return Cost;
489}
490
491bool X86DomainReassignmentImpl::isReassignmentProfitable(
492 const Closure &C, RegDomain Domain) const {
493 return calculateCost(C, DstDomain: Domain) < 0.0;
494}
495
496void X86DomainReassignmentImpl::reassign(const Closure &C,
497 RegDomain Domain) const {
498 assert(C.isLegal(Domain) && "Cannot convert illegal closure");
499
500 // Iterate all instructions in the closure, convert each one using the
501 // appropriate converter.
502 SmallVector<MachineInstr *, 8> ToErase;
503 for (auto *MI : C.instructions())
504 if (Converters.find(Val: {Domain, MI->getOpcode()})
505 ->second->convertInstr(MI, TII, MRI))
506 ToErase.push_back(Elt: MI);
507
508 // Iterate all registers in the closure, replace them with registers in the
509 // destination domain.
510 for (Register Reg : C.edges()) {
511 MRI->setRegClass(Reg, RC: getDstRC(SrcRC: MRI->getRegClass(Reg), Domain));
512 for (auto &MO : MRI->use_operands(Reg)) {
513 if (MO.isReg())
514 // Remove all subregister references as they are not valid in the
515 // destination domain.
516 MO.setSubReg(0);
517 }
518 }
519
520 for (auto *MI : ToErase)
521 MI->eraseFromParent();
522}
523
524/// \returns true when \p Reg is used as part of an address calculation in \p
525/// MI.
526static bool usedAsAddr(const MachineInstr &MI, Register Reg,
527 const TargetInstrInfo *TII) {
528 if (!MI.mayLoadOrStore())
529 return false;
530
531 int MemOpStart = X86II::getMemoryOperandIdx(Desc: TII->get(Opcode: MI.getOpcode()));
532 if (MemOpStart == -1)
533 return false;
534
535 for (unsigned MemOpIdx = MemOpStart,
536 MemOpEnd = MemOpStart + X86::AddrNumOperands;
537 MemOpIdx < MemOpEnd; ++MemOpIdx) {
538 const MachineOperand &Op = MI.getOperand(i: MemOpIdx);
539 if (Op.isReg() && Op.getReg() == Reg)
540 return true;
541 }
542 return false;
543}
544
545void X86DomainReassignmentImpl::buildClosure(Closure &C, Register Reg) {
546 SmallVector<Register, 4> Worklist;
547 RegDomain Domain = NoDomain;
548 visitRegister(C, Reg, Domain, Worklist);
549 while (!Worklist.empty()) {
550 Register CurReg = Worklist.pop_back_val();
551
552 // Register already in this closure.
553 if (!C.insertEdge(Reg: CurReg))
554 continue;
555 EnclosedEdges[CurReg] = C.getID();
556
557 MachineInstr *DefMI = MRI->getVRegDef(Reg: CurReg);
558 if (!encloseInstr(C, MI: DefMI))
559 return;
560
561 // Add register used by the defining MI to the worklist.
562 // Do not add registers which are used in address calculation, they will be
563 // added to a different closure.
564 int OpEnd = DefMI->getNumOperands();
565 int MemOp = X86II::getMemoryOperandIdx(Desc: DefMI->getDesc());
566 for (int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {
567 if (OpIdx == MemOp) {
568 // skip address calculation.
569 OpIdx += (X86::AddrNumOperands - 1);
570 continue;
571 }
572 auto &Op = DefMI->getOperand(i: OpIdx);
573 if (!Op.isReg() || !Op.isUse())
574 continue;
575 if (!visitRegister(C, Reg: Op.getReg(), Domain, Worklist))
576 return;
577 }
578
579 // Expand closure through register uses.
580 for (auto &UseMI : MRI->use_nodbg_instructions(Reg: CurReg)) {
581 // We would like to avoid converting closures which calculare addresses,
582 // as this should remain in GPRs.
583 if (usedAsAddr(MI: UseMI, Reg: CurReg, TII)) {
584 C.setAllIllegal();
585 return;
586 }
587 if (!encloseInstr(C, MI: &UseMI))
588 return;
589
590 for (auto &DefOp : UseMI.defs()) {
591 if (!DefOp.isReg())
592 continue;
593
594 Register DefReg = DefOp.getReg();
595 if (!DefReg.isVirtual()) {
596 C.setAllIllegal();
597 return;
598 }
599 if (!visitRegister(C, Reg: DefReg, Domain, Worklist))
600 return;
601 }
602 }
603 }
604}
605
606void X86DomainReassignmentImpl::initConverters() {
607 Converters[{MaskDomain, TargetOpcode::PHI}] =
608 std::make_unique<InstrIgnore>(args: TargetOpcode::PHI);
609
610 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
611 std::make_unique<InstrIgnore>(args: TargetOpcode::IMPLICIT_DEF);
612
613 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
614 std::make_unique<InstrReplaceWithCopy>(args: TargetOpcode::INSERT_SUBREG, args: 2);
615
616 Converters[{MaskDomain, TargetOpcode::COPY}] =
617 std::make_unique<InstrCOPYReplacer>(args: TargetOpcode::COPY, args: MaskDomain,
618 args: TargetOpcode::COPY);
619
620 auto createReplacerDstCOPY = [&](unsigned From, unsigned To) {
621 Converters[{MaskDomain, From}] =
622 std::make_unique<InstrReplacerDstCOPY>(args&: From, args&: To);
623 };
624
625#define GET_EGPR_IF_ENABLED(OPC) STI->hasEGPR() ? OPC##_EVEX : OPC
626 createReplacerDstCOPY(X86::MOVZX32rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
627 createReplacerDstCOPY(X86::MOVZX64rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
628
629 createReplacerDstCOPY(X86::MOVZX32rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
630 createReplacerDstCOPY(X86::MOVZX64rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
631
632 if (STI->hasDQI()) {
633 createReplacerDstCOPY(X86::MOVZX16rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
634 createReplacerDstCOPY(X86::MOVZX32rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
635 createReplacerDstCOPY(X86::MOVZX64rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
636
637 createReplacerDstCOPY(X86::MOVZX16rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
638 createReplacerDstCOPY(X86::MOVZX32rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
639 createReplacerDstCOPY(X86::MOVZX64rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
640 }
641
642 auto createReplacer = [&](unsigned From, unsigned To) {
643 Converters[{MaskDomain, From}] = std::make_unique<InstrReplacer>(args&: From, args&: To);
644 };
645
646 createReplacer(X86::MOV16rm, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
647 createReplacer(X86::MOV16mr, GET_EGPR_IF_ENABLED(X86::KMOVWmk));
648 createReplacer(X86::MOV16rr, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
649 createReplacer(X86::SHR16ri, X86::KSHIFTRWki);
650 createReplacer(X86::SHL16ri, X86::KSHIFTLWki);
651 createReplacer(X86::NOT16r, X86::KNOTWkk);
652 createReplacer(X86::OR16rr, X86::KORWkk);
653 createReplacer(X86::AND16rr, X86::KANDWkk);
654 createReplacer(X86::XOR16rr, X86::KXORWkk);
655
656 bool HasNDD = STI->hasNDD();
657 if (HasNDD) {
658 createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWki);
659 createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWki);
660 createReplacer(X86::NOT16r_ND, X86::KNOTWkk);
661 createReplacer(X86::OR16rr_ND, X86::KORWkk);
662 createReplacer(X86::AND16rr_ND, X86::KANDWkk);
663 createReplacer(X86::XOR16rr_ND, X86::KXORWkk);
664 }
665
666 if (STI->hasBWI()) {
667 createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm));
668 createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm));
669
670 createReplacer(X86::MOV32mr, GET_EGPR_IF_ENABLED(X86::KMOVDmk));
671 createReplacer(X86::MOV64mr, GET_EGPR_IF_ENABLED(X86::KMOVQmk));
672
673 createReplacer(X86::MOV32rr, GET_EGPR_IF_ENABLED(X86::KMOVDkk));
674 createReplacer(X86::MOV64rr, GET_EGPR_IF_ENABLED(X86::KMOVQkk));
675
676 createReplacer(X86::SHR32ri, X86::KSHIFTRDki);
677 createReplacer(X86::SHR64ri, X86::KSHIFTRQki);
678
679 createReplacer(X86::SHL32ri, X86::KSHIFTLDki);
680 createReplacer(X86::SHL64ri, X86::KSHIFTLQki);
681
682 createReplacer(X86::ADD32rr, X86::KADDDkk);
683 createReplacer(X86::ADD64rr, X86::KADDQkk);
684
685 createReplacer(X86::NOT32r, X86::KNOTDkk);
686 createReplacer(X86::NOT64r, X86::KNOTQkk);
687
688 createReplacer(X86::OR32rr, X86::KORDkk);
689 createReplacer(X86::OR64rr, X86::KORQkk);
690
691 createReplacer(X86::AND32rr, X86::KANDDkk);
692 createReplacer(X86::AND64rr, X86::KANDQkk);
693
694 createReplacer(X86::ANDN32rr, X86::KANDNDkk);
695 createReplacer(X86::ANDN64rr, X86::KANDNQkk);
696
697 createReplacer(X86::XOR32rr, X86::KXORDkk);
698 createReplacer(X86::XOR64rr, X86::KXORQkk);
699
700 if (HasNDD) {
701 createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDki);
702 createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDki);
703 createReplacer(X86::ADD32rr_ND, X86::KADDDkk);
704 createReplacer(X86::NOT32r_ND, X86::KNOTDkk);
705 createReplacer(X86::OR32rr_ND, X86::KORDkk);
706 createReplacer(X86::AND32rr_ND, X86::KANDDkk);
707 createReplacer(X86::XOR32rr_ND, X86::KXORDkk);
708 createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQki);
709 createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQki);
710 createReplacer(X86::ADD64rr_ND, X86::KADDQkk);
711 createReplacer(X86::NOT64r_ND, X86::KNOTQkk);
712 createReplacer(X86::OR64rr_ND, X86::KORQkk);
713 createReplacer(X86::AND64rr_ND, X86::KANDQkk);
714 createReplacer(X86::XOR64rr_ND, X86::KXORQkk);
715 }
716
717 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
718 // to prove only Z flag is used.
719 // createReplacer(X86::TEST32rr, X86::KTESTDkk);
720 // createReplacer(X86::TEST64rr, X86::KTESTQkk);
721 }
722
723 if (STI->hasDQI()) {
724 createReplacer(X86::ADD8rr, X86::KADDBkk);
725 createReplacer(X86::ADD16rr, X86::KADDWkk);
726
727 createReplacer(X86::AND8rr, X86::KANDBkk);
728
729 createReplacer(X86::MOV8rm, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
730 createReplacer(X86::MOV8mr, GET_EGPR_IF_ENABLED(X86::KMOVBmk));
731 createReplacer(X86::MOV8rr, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
732
733 createReplacer(X86::NOT8r, X86::KNOTBkk);
734
735 createReplacer(X86::OR8rr, X86::KORBkk);
736
737 createReplacer(X86::SHR8ri, X86::KSHIFTRBki);
738 createReplacer(X86::SHL8ri, X86::KSHIFTLBki);
739
740 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
741 // to prove only Z flag is used.
742 // createReplacer(X86::TEST8rr, X86::KTESTBkk);
743 // createReplacer(X86::TEST16rr, X86::KTESTWkk);
744
745 createReplacer(X86::XOR8rr, X86::KXORBkk);
746
747 if (HasNDD) {
748 createReplacer(X86::ADD8rr_ND, X86::KADDBkk);
749 createReplacer(X86::ADD16rr_ND, X86::KADDWkk);
750 createReplacer(X86::AND8rr_ND, X86::KANDBkk);
751 createReplacer(X86::NOT8r_ND, X86::KNOTBkk);
752 createReplacer(X86::OR8rr_ND, X86::KORBkk);
753 createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBki);
754 createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBki);
755 createReplacer(X86::XOR8rr_ND, X86::KXORBkk);
756 }
757 }
758#undef GET_EGPR_IF_ENABLED
759}
760
761bool X86DomainReassignmentImpl::runOnMachineFunction(MachineFunction &MF) {
762 if (DisableX86DomainReassignment)
763 return false;
764
765 LLVM_DEBUG(
766 dbgs() << "***** Machine Function before Domain Reassignment *****\n");
767 LLVM_DEBUG(MF.print(dbgs()));
768
769 STI = &MF.getSubtarget<X86Subtarget>();
770 // GPR->K is the only transformation currently supported, bail out early if no
771 // AVX512.
772 // TODO: We're also bailing of AVX512BW isn't supported since we use VK32 and
773 // VK64 for GR32/GR64, but those aren't legal classes on KNL. If the register
774 // coalescer doesn't clean it up and we generate a spill we will crash.
775 if (!STI->hasAVX512() || !STI->hasBWI())
776 return false;
777
778 MRI = &MF.getRegInfo();
779 assert(MRI->isSSA() && "Expected MIR to be in SSA form");
780
781 TII = STI->getInstrInfo();
782 initConverters();
783 bool Changed = false;
784
785 EnclosedEdges.clear();
786 EnclosedInstrs.clear();
787
788 std::vector<Closure> Closures;
789
790 // Go over all virtual registers and calculate a closure.
791 unsigned ClosureID = 0;
792 for (unsigned Idx = 0; Idx < MRI->getNumVirtRegs(); ++Idx) {
793 Register Reg = Register::index2VirtReg(Index: Idx);
794
795 // Skip unused VRegs.
796 if (MRI->reg_nodbg_empty(RegNo: Reg))
797 continue;
798
799 // GPR only current source domain supported.
800 if (!MRI->getTargetRegisterInfo()->isGeneralPurposeRegisterClass(
801 RC: MRI->getRegClass(Reg)))
802 continue;
803
804 // Register already in closure.
805 if (EnclosedEdges.contains(Val: Reg))
806 continue;
807
808 // Calculate closure starting with Reg.
809 Closure C(ClosureID++, {MaskDomain});
810 buildClosure(C, Reg);
811 ++NumClosuresBuilt;
812
813 // Collect all closures that can potentially be converted.
814 if (!C.empty() && C.isLegal(RD: MaskDomain))
815 Closures.push_back(x: std::move(C));
816 }
817
818 for (Closure &C : Closures) {
819 LLVM_DEBUG(C.dump(MRI));
820 if (isReassignmentProfitable(C, Domain: MaskDomain)) {
821 reassign(C, Domain: MaskDomain);
822 ++NumClosuresConverted;
823 Changed = true;
824 }
825 }
826
827 LLVM_DEBUG(
828 dbgs() << "***** Machine Function after Domain Reassignment *****\n");
829 LLVM_DEBUG(MF.print(dbgs()));
830
831 return Changed;
832}
833
834bool X86DomainReassignmentLegacy::runOnMachineFunction(MachineFunction &MF) {
835 if (skipFunction(F: MF.getFunction()))
836 return false;
837 X86DomainReassignmentImpl Impl;
838 return Impl.runOnMachineFunction(MF);
839}
840
841INITIALIZE_PASS(X86DomainReassignmentLegacy, "x86-domain-reassignment",
842 "X86 Domain Reassignment Pass", false, false)
843
844/// Returns an instance of the Domain Reassignment pass.
845FunctionPass *llvm::createX86DomainReassignmentLegacyPass() {
846 return new X86DomainReassignmentLegacy();
847}
848
849PreservedAnalyses
850X86DomainReassignmentPass::run(MachineFunction &MF,
851 MachineFunctionAnalysisManager &MFAM) {
852 X86DomainReassignmentImpl Impl;
853 bool Changed = Impl.runOnMachineFunction(MF);
854 if (!Changed)
855 return PreservedAnalyses::all();
856 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
857 PA.preserveSet<CFGAnalyses>();
858 return PA;
859}
860