1//===--- X86DomainReassignment.cpp - Selectively switch register classes---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass attempts to find instruction chains (closures) in one domain,
10// and convert them to equivalent instructions in a different domain,
11// if profitable.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86InstrInfo.h"
17#include "X86Subtarget.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/Printable.h"
29#include <bitset>
30
31using namespace llvm;
32
33#define DEBUG_TYPE "x86-domain-reassignment"
34
35STATISTIC(NumClosuresConverted, "Number of closures converted by the pass");
36STATISTIC(NumClosuresBuilt, "Number of closures built by the pass");
37
38namespace {
39enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
40
41static bool isMask(const TargetRegisterClass *RC,
42 const TargetRegisterInfo *TRI) {
43 return X86::VK16RegClass.hasSubClassEq(RC);
44}
45
46static RegDomain getDomain(const TargetRegisterClass *RC,
47 const TargetRegisterInfo *TRI) {
48 if (TRI->isGeneralPurposeRegisterClass(RC))
49 return GPRDomain;
50 if (isMask(RC, TRI))
51 return MaskDomain;
52 return OtherDomain;
53}
54
55/// Return a register class equivalent to \p SrcRC, in \p Domain.
56static const TargetRegisterClass *getDstRC(const TargetRegisterClass *SrcRC,
57 RegDomain Domain) {
58 assert(Domain == MaskDomain && "add domain");
59 if (X86::GR8RegClass.hasSubClassEq(RC: SrcRC))
60 return &X86::VK8RegClass;
61 if (X86::GR16RegClass.hasSubClassEq(RC: SrcRC))
62 return &X86::VK16RegClass;
63 if (X86::GR32RegClass.hasSubClassEq(RC: SrcRC))
64 return &X86::VK32RegClass;
65 if (X86::GR64RegClass.hasSubClassEq(RC: SrcRC))
66 return &X86::VK64RegClass;
67 llvm_unreachable("add register class");
68 return nullptr;
69}
70
71/// Abstract Instruction Converter class.
72class InstrConverterBase {
73protected:
74 unsigned SrcOpcode;
75
76public:
77 InstrConverterBase(unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
78
79 virtual ~InstrConverterBase() = default;
80
81 /// \returns true if \p MI is legal to convert.
82 virtual bool isLegal(const MachineInstr *MI,
83 const TargetInstrInfo *TII) const {
84 assert(MI->getOpcode() == SrcOpcode &&
85 "Wrong instruction passed to converter");
86 return true;
87 }
88
89 /// Applies conversion to \p MI.
90 ///
91 /// \returns true if \p MI is no longer need, and can be deleted.
92 virtual bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
93 MachineRegisterInfo *MRI) const = 0;
94
95 /// \returns the cost increment incurred by converting \p MI.
96 virtual double getExtraCost(const MachineInstr *MI,
97 MachineRegisterInfo *MRI) const = 0;
98};
99
100/// An Instruction Converter which ignores the given instruction.
101/// For example, PHI instructions can be safely ignored since only the registers
102/// need to change.
103class InstrIgnore : public InstrConverterBase {
104public:
105 InstrIgnore(unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
106
107 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
108 MachineRegisterInfo *MRI) const override {
109 assert(isLegal(MI, TII) && "Cannot convert instruction");
110 return false;
111 }
112
113 double getExtraCost(const MachineInstr *MI,
114 MachineRegisterInfo *MRI) const override {
115 return 0;
116 }
117};
118
119/// An Instruction Converter which replaces an instruction with another.
120class InstrReplacer : public InstrConverterBase {
121public:
122 /// Opcode of the destination instruction.
123 unsigned DstOpcode;
124
125 InstrReplacer(unsigned SrcOpcode, unsigned DstOpcode)
126 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
127
128 bool isLegal(const MachineInstr *MI,
129 const TargetInstrInfo *TII) const override {
130 if (!InstrConverterBase::isLegal(MI, TII))
131 return false;
132 // It's illegal to replace an instruction that implicitly defines a register
133 // with an instruction that doesn't, unless that register dead.
134 for (const auto &MO : MI->implicit_operands())
135 if (MO.isReg() && MO.isDef() && !MO.isDead() &&
136 !TII->get(Opcode: DstOpcode).hasImplicitDefOfPhysReg(Reg: MO.getReg()))
137 return false;
138 return true;
139 }
140
141 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
142 MachineRegisterInfo *MRI) const override {
143 assert(isLegal(MI, TII) && "Cannot convert instruction");
144 MachineInstrBuilder Bld =
145 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: DstOpcode));
146 // Transfer explicit operands from original instruction. Implicit operands
147 // are handled by BuildMI.
148 for (auto &Op : MI->explicit_operands())
149 Bld.add(MO: Op);
150 return true;
151 }
152
153 double getExtraCost(const MachineInstr *MI,
154 MachineRegisterInfo *MRI) const override {
155 // Assuming instructions have the same cost.
156 return 0;
157 }
158};
159
160/// An Instruction Converter which replaces an instruction with another, and
161/// adds a COPY from the new instruction's destination to the old one's.
162class InstrReplacerDstCOPY : public InstrConverterBase {
163public:
164 unsigned DstOpcode;
165
166 InstrReplacerDstCOPY(unsigned SrcOpcode, unsigned DstOpcode)
167 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
168
169 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
170 MachineRegisterInfo *MRI) const override {
171 assert(isLegal(MI, TII) && "Cannot convert instruction");
172 MachineBasicBlock *MBB = MI->getParent();
173 const DebugLoc &DL = MI->getDebugLoc();
174
175 Register Reg =
176 MRI->createVirtualRegister(RegClass: TII->getRegClass(MCID: TII->get(Opcode: DstOpcode), OpNum: 0));
177 MachineInstrBuilder Bld = BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: DstOpcode), DestReg: Reg);
178 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
179 Bld.add(MO);
180
181 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
182 .add(MO: MI->getOperand(i: 0))
183 .addReg(RegNo: Reg);
184
185 return true;
186 }
187
188 double getExtraCost(const MachineInstr *MI,
189 MachineRegisterInfo *MRI) const override {
190 // Assuming instructions have the same cost, and that COPY is in the same
191 // domain so it will be eliminated.
192 return 0;
193 }
194};
195
196/// An Instruction Converter for replacing COPY instructions.
197class InstrCOPYReplacer : public InstrReplacer {
198public:
199 RegDomain DstDomain;
200
201 InstrCOPYReplacer(unsigned SrcOpcode, RegDomain DstDomain, unsigned DstOpcode)
202 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
203
204 bool isLegal(const MachineInstr *MI,
205 const TargetInstrInfo *TII) const override {
206 if (!InstrConverterBase::isLegal(MI, TII))
207 return false;
208
209 // Don't allow copies to/flow GR8/GR16 physical registers.
210 // FIXME: Is there some better way to support this?
211 Register DstReg = MI->getOperand(i: 0).getReg();
212 if (DstReg.isPhysical() && (X86::GR8RegClass.contains(Reg: DstReg) ||
213 X86::GR16RegClass.contains(Reg: DstReg)))
214 return false;
215 Register SrcReg = MI->getOperand(i: 1).getReg();
216 if (SrcReg.isPhysical() && (X86::GR8RegClass.contains(Reg: SrcReg) ||
217 X86::GR16RegClass.contains(Reg: SrcReg)))
218 return false;
219
220 return true;
221 }
222
223 double getExtraCost(const MachineInstr *MI,
224 MachineRegisterInfo *MRI) const override {
225 assert(MI->getOpcode() == TargetOpcode::COPY && "Expected a COPY");
226
227 for (const auto &MO : MI->operands()) {
228 // Physical registers will not be converted. Assume that converting the
229 // COPY to the destination domain will eventually result in a actual
230 // instruction.
231 if (MO.getReg().isPhysical())
232 return 1;
233
234 RegDomain OpDomain = getDomain(RC: MRI->getRegClass(Reg: MO.getReg()),
235 TRI: MRI->getTargetRegisterInfo());
236 // Converting a cross domain COPY to a same domain COPY should eliminate
237 // an insturction
238 if (OpDomain == DstDomain)
239 return -1;
240 }
241 return 0;
242 }
243};
244
245/// An Instruction Converter which replaces an instruction with a COPY.
246class InstrReplaceWithCopy : public InstrConverterBase {
247public:
248 // Source instruction operand Index, to be used as the COPY source.
249 unsigned SrcOpIdx;
250
251 InstrReplaceWithCopy(unsigned SrcOpcode, unsigned SrcOpIdx)
252 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
253
254 bool convertInstr(MachineInstr *MI, const TargetInstrInfo *TII,
255 MachineRegisterInfo *MRI) const override {
256 assert(isLegal(MI, TII) && "Cannot convert instruction");
257 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
258 MCID: TII->get(Opcode: TargetOpcode::COPY))
259 .add(MOs: {MI->getOperand(i: 0), MI->getOperand(i: SrcOpIdx)});
260 return true;
261 }
262
263 double getExtraCost(const MachineInstr *MI,
264 MachineRegisterInfo *MRI) const override {
265 return 0;
266 }
267};
268
269// Key type to be used by the Instruction Converters map.
270// A converter is identified by <destination domain, source opcode>
271typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
272
273typedef DenseMap<InstrConverterBaseKeyTy, std::unique_ptr<InstrConverterBase>>
274 InstrConverterBaseMap;
275
276/// A closure is a set of virtual register representing all of the edges in
277/// the closure, as well as all of the instructions connected by those edges.
278///
279/// A closure may encompass virtual registers in the same register bank that
280/// have different widths. For example, it may contain 32-bit GPRs as well as
281/// 64-bit GPRs.
282///
283/// A closure that computes an address (i.e. defines a virtual register that is
284/// used in a memory operand) excludes the instructions that contain memory
285/// operands using the address. Such an instruction will be included in a
286/// different closure that manipulates the loaded or stored value.
287class Closure {
288private:
289 /// Virtual registers in the closure.
290 DenseSet<Register> Edges;
291
292 /// Instructions in the closure.
293 SmallVector<MachineInstr *, 8> Instrs;
294
295 /// Domains which this closure can legally be reassigned to.
296 std::bitset<NumDomains> LegalDstDomains;
297
298 /// An ID to uniquely identify this closure, even when it gets
299 /// moved around
300 unsigned ID;
301
302public:
303 Closure(unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) : ID(ID) {
304 for (RegDomain D : LegalDstDomainList)
305 LegalDstDomains.set(position: D);
306 }
307
308 /// Mark this closure as illegal for reassignment to all domains.
309 void setAllIllegal() { LegalDstDomains.reset(); }
310
311 /// \returns true if this closure has domains which are legal to reassign to.
312 bool hasLegalDstDomain() const { return LegalDstDomains.any(); }
313
314 /// \returns true if is legal to reassign this closure to domain \p RD.
315 bool isLegal(RegDomain RD) const { return LegalDstDomains[RD]; }
316
317 /// Mark this closure as illegal for reassignment to domain \p RD.
318 void setIllegal(RegDomain RD) { LegalDstDomains[RD] = false; }
319
320 bool empty() const { return Edges.empty(); }
321
322 bool insertEdge(Register Reg) { return Edges.insert(V: Reg).second; }
323
324 using const_edge_iterator = DenseSet<Register>::const_iterator;
325 iterator_range<const_edge_iterator> edges() const { return Edges; }
326
327 void addInstruction(MachineInstr *I) {
328 Instrs.push_back(Elt: I);
329 }
330
331 ArrayRef<MachineInstr *> instructions() const {
332 return Instrs;
333 }
334
335 LLVM_DUMP_METHOD void dump(const MachineRegisterInfo *MRI) const {
336 dbgs() << "Registers: ";
337 ListSeparator LS;
338 for (Register Reg : Edges)
339 dbgs() << LS << printReg(Reg, TRI: MRI->getTargetRegisterInfo(), SubIdx: 0, MRI);
340 dbgs() << "\n" << "Instructions:";
341 for (MachineInstr *MI : Instrs) {
342 dbgs() << "\n ";
343 MI->print(OS&: dbgs());
344 }
345 dbgs() << "\n";
346 }
347
348 unsigned getID() const {
349 return ID;
350 }
351
352};
353
354class X86DomainReassignmentImpl {
355public:
356 bool runOnMachineFunction(MachineFunction &MF);
357
358private:
359 const X86Subtarget *STI = nullptr;
360 MachineRegisterInfo *MRI = nullptr;
361 const X86InstrInfo *TII = nullptr;
362
363 /// All edges that are included in some closure
364 DenseMap<Register, unsigned> EnclosedEdges;
365
366 /// All instructions that are included in some closure.
367 DenseMap<MachineInstr *, unsigned> EnclosedInstrs;
368
369 /// A map of available Instruction Converters.
370 InstrConverterBaseMap Converters;
371
372 /// Initialize Converters map.
373 void initConverters();
374
375 /// Starting from \Reg, expand the closure as much as possible.
376 void buildClosure(Closure &, Register Reg);
377
378 /// Enqueue \p Reg to be considered for addition to the closure.
379 /// Return false if the closure becomes invalid.
380 bool visitRegister(Closure &, Register Reg, RegDomain &Domain,
381 SmallVectorImpl<Register> &Worklist);
382
383 /// Reassign the closure to \p Domain.
384 void reassign(const Closure &C, RegDomain Domain) const;
385
386 /// Add \p MI to the closure.
387 /// Return false if the closure becomes invalid.
388 bool encloseInstr(Closure &C, MachineInstr *MI);
389
390 /// /returns true if it is profitable to reassign the closure to \p Domain.
391 bool isReassignmentProfitable(const Closure &C, RegDomain Domain) const;
392
393 /// Calculate the total cost of reassigning the closure to \p Domain.
394 double calculateCost(const Closure &C, RegDomain Domain) const;
395};
396
397class X86DomainReassignmentLegacy : public MachineFunctionPass {
398public:
399 static char ID;
400
401 X86DomainReassignmentLegacy() : MachineFunctionPass(ID) {}
402
403 bool runOnMachineFunction(MachineFunction &MF) override;
404
405 void getAnalysisUsage(AnalysisUsage &AU) const override {
406 AU.setPreservesCFG();
407 MachineFunctionPass::getAnalysisUsage(AU);
408 }
409
410 StringRef getPassName() const override {
411 return "X86 Domain Reassignment Pass";
412 }
413};
414
415char X86DomainReassignmentLegacy::ID = 0;
416
417} // End anonymous namespace.
418
419bool X86DomainReassignmentImpl::visitRegister(
420 Closure &C, Register Reg, RegDomain &Domain,
421 SmallVectorImpl<Register> &Worklist) {
422 if (!Reg.isVirtual())
423 return true;
424
425 auto I = EnclosedEdges.find(Val: Reg);
426 if (I != EnclosedEdges.end()) {
427 if (I->second != C.getID()) {
428 C.setAllIllegal();
429 return false;
430 }
431 return true;
432 }
433
434 if (!MRI->hasOneDef(RegNo: Reg))
435 return true;
436
437 RegDomain RD = getDomain(RC: MRI->getRegClass(Reg), TRI: MRI->getTargetRegisterInfo());
438 // First edge in closure sets the domain.
439 if (Domain == NoDomain)
440 Domain = RD;
441
442 if (Domain != RD)
443 return true;
444
445 Worklist.push_back(Elt: Reg);
446 return true;
447}
448
449bool X86DomainReassignmentImpl::encloseInstr(Closure &C, MachineInstr *MI) {
450 auto [I, Inserted] = EnclosedInstrs.try_emplace(Key: MI, Args: C.getID());
451 if (!Inserted) {
452 if (I->second != C.getID()) {
453 // Instruction already belongs to another closure, avoid conflicts between
454 // closure and mark this closure as illegal.
455 C.setAllIllegal();
456 return false;
457 }
458 return true;
459 }
460
461 C.addInstruction(I: MI);
462
463 // Mark closure as illegal for reassignment to domains, if there is no
464 // converter for the instruction or if the converter cannot convert the
465 // instruction.
466 for (int i = 0; i != NumDomains; ++i) {
467 if (C.isLegal(RD: (RegDomain)i)) {
468 auto I = Converters.find(Val: {i, MI->getOpcode()});
469 if (I == Converters.end() || !I->second->isLegal(MI, TII))
470 C.setIllegal((RegDomain)i);
471 }
472 }
473 return C.hasLegalDstDomain();
474}
475
476double X86DomainReassignmentImpl::calculateCost(const Closure &C,
477 RegDomain DstDomain) const {
478 assert(C.isLegal(DstDomain) && "Cannot calculate cost for illegal closure");
479
480 double Cost = 0.0;
481 for (auto *MI : C.instructions())
482 Cost += Converters.find(Val: {DstDomain, MI->getOpcode()})
483 ->second->getExtraCost(MI, MRI);
484 return Cost;
485}
486
487bool X86DomainReassignmentImpl::isReassignmentProfitable(
488 const Closure &C, RegDomain Domain) const {
489 return calculateCost(C, DstDomain: Domain) < 0.0;
490}
491
492void X86DomainReassignmentImpl::reassign(const Closure &C,
493 RegDomain Domain) const {
494 assert(C.isLegal(Domain) && "Cannot convert illegal closure");
495
496 // Iterate all instructions in the closure, convert each one using the
497 // appropriate converter.
498 SmallVector<MachineInstr *, 8> ToErase;
499 for (auto *MI : C.instructions())
500 if (Converters.find(Val: {Domain, MI->getOpcode()})
501 ->second->convertInstr(MI, TII, MRI))
502 ToErase.push_back(Elt: MI);
503
504 // Iterate all registers in the closure, replace them with registers in the
505 // destination domain.
506 for (Register Reg : C.edges()) {
507 MRI->setRegClass(Reg, RC: getDstRC(SrcRC: MRI->getRegClass(Reg), Domain));
508 for (auto &MO : MRI->use_operands(Reg)) {
509 if (MO.isReg())
510 // Remove all subregister references as they are not valid in the
511 // destination domain.
512 MO.setSubReg(0);
513 }
514 }
515
516 for (auto *MI : ToErase)
517 MI->eraseFromParent();
518}
519
520/// \returns true when \p Reg is used as part of an address calculation in \p
521/// MI.
522static bool usedAsAddr(const MachineInstr &MI, Register Reg,
523 const TargetInstrInfo *TII) {
524 if (!MI.mayLoadOrStore())
525 return false;
526
527 int MemOpStart = X86II::getMemoryOperandIdx(Desc: TII->get(Opcode: MI.getOpcode()));
528 if (MemOpStart == -1)
529 return false;
530
531 for (unsigned MemOpIdx = MemOpStart,
532 MemOpEnd = MemOpStart + X86::AddrNumOperands;
533 MemOpIdx < MemOpEnd; ++MemOpIdx) {
534 const MachineOperand &Op = MI.getOperand(i: MemOpIdx);
535 if (Op.isReg() && Op.getReg() == Reg)
536 return true;
537 }
538 return false;
539}
540
541void X86DomainReassignmentImpl::buildClosure(Closure &C, Register Reg) {
542 SmallVector<Register, 4> Worklist;
543 RegDomain Domain = NoDomain;
544 visitRegister(C, Reg, Domain, Worklist);
545 while (!Worklist.empty()) {
546 Register CurReg = Worklist.pop_back_val();
547
548 // Register already in this closure.
549 if (!C.insertEdge(Reg: CurReg))
550 continue;
551 EnclosedEdges[CurReg] = C.getID();
552
553 MachineInstr *DefMI = MRI->getVRegDef(Reg: CurReg);
554 if (!encloseInstr(C, MI: DefMI))
555 return;
556
557 // Add register used by the defining MI to the worklist.
558 // Do not add registers which are used in address calculation, they will be
559 // added to a different closure.
560 int OpEnd = DefMI->getNumOperands();
561 int MemOp = X86II::getMemoryOperandIdx(Desc: DefMI->getDesc());
562 for (int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {
563 if (OpIdx == MemOp) {
564 // skip address calculation.
565 OpIdx += (X86::AddrNumOperands - 1);
566 continue;
567 }
568 auto &Op = DefMI->getOperand(i: OpIdx);
569 if (!Op.isReg() || !Op.isUse())
570 continue;
571 if (!visitRegister(C, Reg: Op.getReg(), Domain, Worklist))
572 return;
573 }
574
575 // Expand closure through register uses.
576 for (auto &UseMI : MRI->use_nodbg_instructions(Reg: CurReg)) {
577 // We would like to avoid converting closures which calculare addresses,
578 // as this should remain in GPRs.
579 if (usedAsAddr(MI: UseMI, Reg: CurReg, TII)) {
580 C.setAllIllegal();
581 return;
582 }
583 if (!encloseInstr(C, MI: &UseMI))
584 return;
585
586 for (auto &DefOp : UseMI.defs()) {
587 if (!DefOp.isReg())
588 continue;
589
590 Register DefReg = DefOp.getReg();
591 if (!DefReg.isVirtual()) {
592 C.setAllIllegal();
593 return;
594 }
595 if (!visitRegister(C, Reg: DefReg, Domain, Worklist))
596 return;
597 }
598 }
599 }
600}
601
602void X86DomainReassignmentImpl::initConverters() {
603 Converters[{MaskDomain, TargetOpcode::PHI}] =
604 std::make_unique<InstrIgnore>(args: TargetOpcode::PHI);
605
606 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
607 std::make_unique<InstrIgnore>(args: TargetOpcode::IMPLICIT_DEF);
608
609 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
610 std::make_unique<InstrReplaceWithCopy>(args: TargetOpcode::INSERT_SUBREG, args: 2);
611
612 Converters[{MaskDomain, TargetOpcode::COPY}] =
613 std::make_unique<InstrCOPYReplacer>(args: TargetOpcode::COPY, args: MaskDomain,
614 args: TargetOpcode::COPY);
615
616 auto createReplacerDstCOPY = [&](unsigned From, unsigned To) {
617 Converters[{MaskDomain, From}] =
618 std::make_unique<InstrReplacerDstCOPY>(args&: From, args&: To);
619 };
620
621#define GET_EGPR_IF_ENABLED(OPC) STI->hasEGPR() ? OPC##_EVEX : OPC
622 createReplacerDstCOPY(X86::MOVZX32rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
623 createReplacerDstCOPY(X86::MOVZX64rm16, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
624
625 createReplacerDstCOPY(X86::MOVZX32rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
626 createReplacerDstCOPY(X86::MOVZX64rr16, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
627
628 if (STI->hasDQI()) {
629 createReplacerDstCOPY(X86::MOVZX16rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
630 createReplacerDstCOPY(X86::MOVZX32rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
631 createReplacerDstCOPY(X86::MOVZX64rm8, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
632
633 createReplacerDstCOPY(X86::MOVZX16rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
634 createReplacerDstCOPY(X86::MOVZX32rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
635 createReplacerDstCOPY(X86::MOVZX64rr8, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
636 }
637
638 auto createReplacer = [&](unsigned From, unsigned To) {
639 Converters[{MaskDomain, From}] = std::make_unique<InstrReplacer>(args&: From, args&: To);
640 };
641
642 createReplacer(X86::MOV16rm, GET_EGPR_IF_ENABLED(X86::KMOVWkm));
643 createReplacer(X86::MOV16mr, GET_EGPR_IF_ENABLED(X86::KMOVWmk));
644 createReplacer(X86::MOV16rr, GET_EGPR_IF_ENABLED(X86::KMOVWkk));
645 createReplacer(X86::SHR16ri, X86::KSHIFTRWki);
646 createReplacer(X86::SHL16ri, X86::KSHIFTLWki);
647 createReplacer(X86::NOT16r, X86::KNOTWkk);
648 createReplacer(X86::OR16rr, X86::KORWkk);
649 createReplacer(X86::AND16rr, X86::KANDWkk);
650 createReplacer(X86::XOR16rr, X86::KXORWkk);
651
652 bool HasNDD = STI->hasNDD();
653 if (HasNDD) {
654 createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWki);
655 createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWki);
656 createReplacer(X86::NOT16r_ND, X86::KNOTWkk);
657 createReplacer(X86::OR16rr_ND, X86::KORWkk);
658 createReplacer(X86::AND16rr_ND, X86::KANDWkk);
659 createReplacer(X86::XOR16rr_ND, X86::KXORWkk);
660 }
661
662 if (STI->hasBWI()) {
663 createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm));
664 createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm));
665
666 createReplacer(X86::MOV32mr, GET_EGPR_IF_ENABLED(X86::KMOVDmk));
667 createReplacer(X86::MOV64mr, GET_EGPR_IF_ENABLED(X86::KMOVQmk));
668
669 createReplacer(X86::MOV32rr, GET_EGPR_IF_ENABLED(X86::KMOVDkk));
670 createReplacer(X86::MOV64rr, GET_EGPR_IF_ENABLED(X86::KMOVQkk));
671
672 createReplacer(X86::SHR32ri, X86::KSHIFTRDki);
673 createReplacer(X86::SHR64ri, X86::KSHIFTRQki);
674
675 createReplacer(X86::SHL32ri, X86::KSHIFTLDki);
676 createReplacer(X86::SHL64ri, X86::KSHIFTLQki);
677
678 createReplacer(X86::ADD32rr, X86::KADDDkk);
679 createReplacer(X86::ADD64rr, X86::KADDQkk);
680
681 createReplacer(X86::NOT32r, X86::KNOTDkk);
682 createReplacer(X86::NOT64r, X86::KNOTQkk);
683
684 createReplacer(X86::OR32rr, X86::KORDkk);
685 createReplacer(X86::OR64rr, X86::KORQkk);
686
687 createReplacer(X86::AND32rr, X86::KANDDkk);
688 createReplacer(X86::AND64rr, X86::KANDQkk);
689
690 createReplacer(X86::ANDN32rr, X86::KANDNDkk);
691 createReplacer(X86::ANDN64rr, X86::KANDNQkk);
692
693 createReplacer(X86::XOR32rr, X86::KXORDkk);
694 createReplacer(X86::XOR64rr, X86::KXORQkk);
695
696 if (HasNDD) {
697 createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDki);
698 createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDki);
699 createReplacer(X86::ADD32rr_ND, X86::KADDDkk);
700 createReplacer(X86::NOT32r_ND, X86::KNOTDkk);
701 createReplacer(X86::OR32rr_ND, X86::KORDkk);
702 createReplacer(X86::AND32rr_ND, X86::KANDDkk);
703 createReplacer(X86::XOR32rr_ND, X86::KXORDkk);
704 createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQki);
705 createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQki);
706 createReplacer(X86::ADD64rr_ND, X86::KADDQkk);
707 createReplacer(X86::NOT64r_ND, X86::KNOTQkk);
708 createReplacer(X86::OR64rr_ND, X86::KORQkk);
709 createReplacer(X86::AND64rr_ND, X86::KANDQkk);
710 createReplacer(X86::XOR64rr_ND, X86::KXORQkk);
711 }
712
713 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
714 // to prove only Z flag is used.
715 // createReplacer(X86::TEST32rr, X86::KTESTDkk);
716 // createReplacer(X86::TEST64rr, X86::KTESTQkk);
717 }
718
719 if (STI->hasDQI()) {
720 createReplacer(X86::ADD8rr, X86::KADDBkk);
721 createReplacer(X86::ADD16rr, X86::KADDWkk);
722
723 createReplacer(X86::AND8rr, X86::KANDBkk);
724
725 createReplacer(X86::MOV8rm, GET_EGPR_IF_ENABLED(X86::KMOVBkm));
726 createReplacer(X86::MOV8mr, GET_EGPR_IF_ENABLED(X86::KMOVBmk));
727 createReplacer(X86::MOV8rr, GET_EGPR_IF_ENABLED(X86::KMOVBkk));
728
729 createReplacer(X86::NOT8r, X86::KNOTBkk);
730
731 createReplacer(X86::OR8rr, X86::KORBkk);
732
733 createReplacer(X86::SHR8ri, X86::KSHIFTRBki);
734 createReplacer(X86::SHL8ri, X86::KSHIFTLBki);
735
736 // TODO: KTEST is not a replacement for TEST due to flag differences. Need
737 // to prove only Z flag is used.
738 // createReplacer(X86::TEST8rr, X86::KTESTBkk);
739 // createReplacer(X86::TEST16rr, X86::KTESTWkk);
740
741 createReplacer(X86::XOR8rr, X86::KXORBkk);
742
743 if (HasNDD) {
744 createReplacer(X86::ADD8rr_ND, X86::KADDBkk);
745 createReplacer(X86::ADD16rr_ND, X86::KADDWkk);
746 createReplacer(X86::AND8rr_ND, X86::KANDBkk);
747 createReplacer(X86::NOT8r_ND, X86::KNOTBkk);
748 createReplacer(X86::OR8rr_ND, X86::KORBkk);
749 createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBki);
750 createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBki);
751 createReplacer(X86::XOR8rr_ND, X86::KXORBkk);
752 }
753 }
754#undef GET_EGPR_IF_ENABLED
755}
756
757bool X86DomainReassignmentImpl::runOnMachineFunction(MachineFunction &MF) {
758 STI = &MF.getSubtarget<X86Subtarget>();
759 if (STI->getCLOpts().disable_x86_domain_reassignment)
760 return false;
761
762 LLVM_DEBUG(
763 dbgs() << "***** Machine Function before Domain Reassignment *****\n");
764 LLVM_DEBUG(MF.print(dbgs()));
765
766 // GPR->K is the only transformation currently supported, bail out early if no
767 // AVX512.
768 // TODO: We're also bailing of AVX512BW isn't supported since we use VK32 and
769 // VK64 for GR32/GR64, but those aren't legal classes on KNL. If the register
770 // coalescer doesn't clean it up and we generate a spill we will crash.
771 if (!STI->hasAVX512() || !STI->hasBWI())
772 return false;
773
774 MRI = &MF.getRegInfo();
775 assert(MRI->isSSA() && "Expected MIR to be in SSA form");
776
777 TII = STI->getInstrInfo();
778 initConverters();
779 bool Changed = false;
780
781 EnclosedEdges.clear();
782 EnclosedInstrs.clear();
783
784 std::vector<Closure> Closures;
785
786 // Go over all virtual registers and calculate a closure.
787 unsigned ClosureID = 0;
788 for (unsigned Idx = 0; Idx < MRI->getNumVirtRegs(); ++Idx) {
789 Register Reg = Register::index2VirtReg(Index: Idx);
790
791 // Skip unused VRegs.
792 if (MRI->reg_nodbg_empty(RegNo: Reg))
793 continue;
794
795 // GPR only current source domain supported.
796 if (!MRI->getTargetRegisterInfo()->isGeneralPurposeRegisterClass(
797 RC: MRI->getRegClass(Reg)))
798 continue;
799
800 // Register already in closure.
801 if (EnclosedEdges.contains(Val: Reg))
802 continue;
803
804 // Calculate closure starting with Reg.
805 Closure C(ClosureID++, {MaskDomain});
806 buildClosure(C, Reg);
807 ++NumClosuresBuilt;
808
809 // Collect all closures that can potentially be converted.
810 if (!C.empty() && C.isLegal(RD: MaskDomain))
811 Closures.push_back(x: std::move(C));
812 }
813
814 for (Closure &C : Closures) {
815 LLVM_DEBUG(C.dump(MRI));
816 if (isReassignmentProfitable(C, Domain: MaskDomain)) {
817 reassign(C, Domain: MaskDomain);
818 ++NumClosuresConverted;
819 Changed = true;
820 }
821 }
822
823 LLVM_DEBUG(
824 dbgs() << "***** Machine Function after Domain Reassignment *****\n");
825 LLVM_DEBUG(MF.print(dbgs()));
826
827 return Changed;
828}
829
830bool X86DomainReassignmentLegacy::runOnMachineFunction(MachineFunction &MF) {
831 if (skipFunction(F: MF.getFunction()))
832 return false;
833 X86DomainReassignmentImpl Impl;
834 return Impl.runOnMachineFunction(MF);
835}
836
837INITIALIZE_PASS(X86DomainReassignmentLegacy, "x86-domain-reassignment",
838 "X86 Domain Reassignment Pass", false, false)
839
840/// Returns an instance of the Domain Reassignment pass.
841FunctionPass *llvm::createX86DomainReassignmentLegacyPass() {
842 return new X86DomainReassignmentLegacy();
843}
844
845PreservedAnalyses
846X86DomainReassignmentPass::run(MachineFunction &MF,
847 MachineFunctionAnalysisManager &MFAM) {
848 X86DomainReassignmentImpl Impl;
849 bool Changed = Impl.runOnMachineFunction(MF);
850 if (!Changed)
851 return PreservedAnalyses::all();
852 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
853 PA.preserveSet<CFGAnalyses>();
854 return PA;
855}
856