1//===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend is responsible for emitting the memory fold tables of
10// the X86 backend instructions.
11//
12//===----------------------------------------------------------------------===//
13
14#include "Common/CodeGenInstruction.h"
15#include "Common/CodeGenTarget.h"
16#include "X86RecognizableInstr.h"
17#include "llvm/ADT/StringSwitch.h"
18#include "llvm/Support/X86FoldTablesUtils.h"
19#include "llvm/TableGen/Error.h"
20#include "llvm/TableGen/Record.h"
21#include "llvm/TableGen/TableGenBackend.h"
22#include <set>
23
24using namespace llvm;
25using namespace X86Disassembler;
26
27namespace {
28// Represents an entry in the manual mapped instructions set.
29struct ManualMapEntry {
30 const char *RegInstStr;
31 const char *MemInstStr;
32 uint16_t Strategy;
33};
34} // namespace
35
36// List of instructions requiring explicitly aligned memory.
37static constexpr const char *ExplicitAlign[] = {
38 "MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", "MOVNTPD", "MOVNTDQ", "MOVNTDQA"};
39
40// List of instructions NOT requiring explicit memory alignment.
41static constexpr const char *ExplicitUnalign[] = {
42 "MOVDQU", "MOVUPS", "MOVUPD", "PCMPESTRM",
43 "PCMPESTRI", "PCMPISTRM", "PCMPISTRI"};
44
45static const ManualMapEntry ManualMapSet[] = {
46#define ENTRY(REG, MEM, FLAGS) {#REG, #MEM, FLAGS},
47#include "X86ManualFoldTables.def"
48};
49
50static const std::set<StringRef> NoFoldSet = {
51#define NOFOLD(INSN) #INSN,
52#include "X86ManualFoldTables.def"
53};
54
55const std::set<StringRef> NoFoldSameMaskPrefixSet = {
56#define NOFOLD_SAME_MASK_PREFIX(PREFIX) #PREFIX,
57#include "X86ManualFoldTables.def"
58};
59
60const std::set<StringRef> NoFoldSameMaskSet = {
61#define NOFOLD_SAME_MASK(INSN) #INSN,
62#include "X86ManualFoldTables.def"
63};
64
65// Check if instruction is unsafe for masked-load folding.
66static bool isNoFoldMaskedInstruction(const CodeGenInstruction *Inst) {
67 StringRef Name = Inst->getName();
68
69 // First check exact instruction name
70 if (NoFoldSameMaskSet.count(x: Name))
71 return true;
72
73 // Then strip suffixes to get base name for prefix matching
74 // Strip k-register suffix: kz or k
75 if (Name.ends_with(Suffix: "kz"))
76 Name = Name.drop_back(N: 2);
77 else if (Name.ends_with(Suffix: "k"))
78 Name = Name.drop_back(N: 1);
79 else
80 return false; // Not a k-register instruction
81
82 // Strip operand form suffix (check longer patterns first)
83 if (Name.ends_with(Suffix: "rri"))
84 Name = Name.drop_back(N: 3);
85 else if (Name.ends_with(Suffix: "rr") || Name.ends_with(Suffix: "ri"))
86 Name = Name.drop_back(N: 2);
87
88 // Strip vector size suffix: Z128, Z256, or Z
89 if (Name.ends_with(Suffix: "Z128") || Name.ends_with(Suffix: "Z256"))
90 Name = Name.drop_back(N: 4);
91 else if (Name.ends_with(Suffix: "Z"))
92 Name = Name.drop_back(N: 1);
93 else
94 return false; // Not a AVX512 instruction
95
96 return NoFoldSameMaskPrefixSet.count(x: Name);
97}
98
99static bool isExplicitAlign(const CodeGenInstruction *Inst) {
100 return any_of(Range: ExplicitAlign, P: [Inst](const char *InstStr) {
101 return Inst->getName().contains(Other: InstStr);
102 });
103}
104
105static bool isExplicitUnalign(const CodeGenInstruction *Inst) {
106 return any_of(Range: ExplicitUnalign, P: [Inst](const char *InstStr) {
107 return Inst->getName().contains(Other: InstStr);
108 });
109}
110
111namespace {
112class X86FoldTablesEmitter {
113 const RecordKeeper &Records;
114 const CodeGenTarget Target;
115
116 // Represents an entry in the folding table
117 class X86FoldTableEntry {
118 public:
119 const CodeGenInstruction *RegInst;
120 const CodeGenInstruction *MemInst;
121
122 bool NoReverse = false;
123 bool NoForward = false;
124 bool FoldLoad = false;
125 bool FoldStore = false;
126 enum BcastType {
127 BCAST_NONE,
128 BCAST_W,
129 BCAST_D,
130 BCAST_Q,
131 BCAST_SS,
132 BCAST_SD,
133 BCAST_SH,
134 };
135 BcastType BroadcastKind = BCAST_NONE;
136
137 Align Alignment;
138 std::optional<unsigned> UnfoldIdx;
139
140 X86FoldTableEntry() = default;
141 X86FoldTableEntry(const CodeGenInstruction *RegInst,
142 const CodeGenInstruction *MemInst)
143 : RegInst(RegInst), MemInst(MemInst) {}
144
145 void print(raw_ostream &OS) const {
146 OS.indent(NumSpaces: 2);
147 OS << "{X86::" << RegInst->getName() << ", ";
148 OS << "X86::" << MemInst->getName() << ", ";
149
150 std::string Attrs;
151 if (UnfoldIdx)
152 Attrs += "TB_INDEX_" + std::to_string(val: *UnfoldIdx) + "|";
153 if (FoldLoad)
154 Attrs += "TB_FOLDED_LOAD|";
155 if (FoldStore)
156 Attrs += "TB_FOLDED_STORE|";
157 if (NoReverse)
158 Attrs += "TB_NO_REVERSE|";
159 if (NoForward)
160 Attrs += "TB_NO_FORWARD|";
161 if (Alignment != Align(1))
162 Attrs += "TB_ALIGN_" + std::to_string(val: Alignment.value()) + "|";
163 switch (BroadcastKind) {
164 case BCAST_NONE:
165 break;
166 case BCAST_W:
167 Attrs += "TB_BCAST_W|";
168 break;
169 case BCAST_D:
170 Attrs += "TB_BCAST_D|";
171 break;
172 case BCAST_Q:
173 Attrs += "TB_BCAST_Q|";
174 break;
175 case BCAST_SS:
176 Attrs += "TB_BCAST_SS|";
177 break;
178 case BCAST_SD:
179 Attrs += "TB_BCAST_SD|";
180 break;
181 case BCAST_SH:
182 Attrs += "TB_BCAST_SH|";
183 break;
184 }
185
186 StringRef SimplifiedAttrs = StringRef(Attrs).rtrim(Chars: "|");
187 if (SimplifiedAttrs.empty())
188 SimplifiedAttrs = "0";
189
190 OS << SimplifiedAttrs << "},\n";
191 }
192
193#ifndef NDEBUG
194 // Check that Uses and Defs are same after memory fold.
195 void checkCorrectness() const {
196 auto &RegInstRec = *RegInst->TheDef;
197 auto &MemInstRec = *MemInst->TheDef;
198 auto ListOfUsesReg = RegInstRec.getValueAsListOfDefs("Uses");
199 auto ListOfUsesMem = MemInstRec.getValueAsListOfDefs("Uses");
200 auto ListOfDefsReg = RegInstRec.getValueAsListOfDefs("Defs");
201 auto ListOfDefsMem = MemInstRec.getValueAsListOfDefs("Defs");
202 if (ListOfUsesReg != ListOfUsesMem || ListOfDefsReg != ListOfDefsMem)
203 report_fatal_error("Uses/Defs couldn't be changed after folding " +
204 RegInstRec.getName() + " to " +
205 MemInstRec.getName());
206 }
207#endif
208 };
209
210 // NOTE: We check the fold tables are sorted in X86InstrFoldTables.cpp by the
211 // enum of the instruction, which is computed in
212 // CodeGenTarget::ComputeInstrsByEnum. So we should use the same comparator
213 // here.
214 // FIXME: Could we share the code with CodeGenTarget::ComputeInstrsByEnum?
215 struct CompareInstrsByEnum {
216 bool operator()(const CodeGenInstruction *LHS,
217 const CodeGenInstruction *RHS) const {
218 assert(LHS && RHS && "LHS and RHS shouldn't be nullptr");
219 const auto &D1 = *LHS->TheDef;
220 const auto &D2 = *RHS->TheDef;
221 return std::tuple(!D1.getValueAsBit(FieldName: "isPseudo"), D1.getName()) <
222 std::tuple(!D2.getValueAsBit(FieldName: "isPseudo"), D2.getName());
223 }
224 };
225
226 using FoldTable = std::map<const CodeGenInstruction *, X86FoldTableEntry,
227 CompareInstrsByEnum>;
228 // Table2Addr - Holds instructions which their memory form performs
229 // load+store.
230 //
231 // Table#i - Holds instructions which the their memory form
232 // performs a load OR a store, and their #i'th operand is folded.
233 //
234 // BroadcastTable#i - Holds instructions which the their memory form performs
235 // a broadcast load and their #i'th operand is folded.
236 FoldTable Table2Addr;
237 FoldTable Table0;
238 FoldTable Table1;
239 FoldTable Table2;
240 FoldTable Table3;
241 FoldTable Table4;
242 FoldTable BroadcastTable1;
243 FoldTable BroadcastTable2;
244 FoldTable BroadcastTable3;
245 FoldTable BroadcastTable4;
246 FoldTable MemUnfoldTable;
247 std::vector<const CodeGenInstruction *> NonFoldableWithSameMaskTable;
248
249public:
250 X86FoldTablesEmitter(const RecordKeeper &R) : Records(R), Target(R) {}
251
252 // run - Generate the 6 X86 memory fold tables.
253 void run(raw_ostream &OS);
254
255private:
256 // Decides to which table to add the entry with the given instructions.
257 // S sets the strategy of adding the TB_NO_REVERSE flag.
258 void updateTables(const CodeGenInstruction *RegInst,
259 const CodeGenInstruction *MemInst, uint16_t S = 0,
260 bool IsManual = false, bool IsBroadcast = false);
261
262 // Generates X86FoldTableEntry with the given instructions and fill it with
263 // the appropriate flags, then adds it to a memory fold table.
264 void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInst,
265 const CodeGenInstruction *MemInst, uint16_t S,
266 unsigned FoldedIdx, bool IsManual);
267 // Generates X86FoldTableEntry with the given instructions and adds it to a
268 // broadcast table.
269 void addBroadcastEntry(FoldTable &Table, const CodeGenInstruction *RegInst,
270 const CodeGenInstruction *MemInst);
271
272 // Adds the reversible entries of Table to MemUnfoldTable, where ExtraFlags
273 // holds the folded operand's TB_INDEX_n and any TB_FOLDED_* implied by Table.
274 void addUnfoldEntries(const FoldTable &Table, uint16_t ExtraFlags);
275
276 // Print the given table as a static const C++ array of type
277 // X86FoldTableEntry.
278 void printTable(const FoldTable &Table, StringRef TableName,
279 raw_ostream &OS) {
280 OS << "static const X86FoldTableEntry " << TableName << "[] = {\n";
281
282 for (auto &E : Table)
283 E.second.print(OS);
284
285 OS << "};\n\n";
286 }
287
288 void printTable(const std::vector<const CodeGenInstruction *> &Instructions,
289 StringRef TableName, raw_ostream &OS) {
290 OS << "static const unsigned " << TableName << "[] = {\n";
291 for (auto Inst : Instructions)
292 OS << " X86::" << Inst->getName() << ",\n";
293 OS << "};\n\n";
294 }
295};
296} // namespace
297
298// Return true if one of the instruction's operands is a RST register class
299static bool hasRSTRegClass(const CodeGenInstruction *Inst) {
300 return any_of(Range: Inst->Operands, P: [](const CGIOperandList::OperandInfo &OpIn) {
301 return OpIn.Rec->getName() == "RST" || OpIn.Rec->getName() == "RSTi";
302 });
303}
304
305// Return true if one of the instruction's operands is a ptr_rc_tailcall
306static bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) {
307 return any_of(Range: Inst->Operands, P: [](const CGIOperandList::OperandInfo &OpIn) {
308 return OpIn.Rec->getName() == "ptr_rc_tailcall";
309 });
310}
311
312static bool mayFoldFromForm(uint8_t Form) {
313 switch (Form) {
314 default:
315 return Form >= X86Local::MRM0r && Form <= X86Local::MRM7r;
316 case X86Local::MRMXr:
317 case X86Local::MRMXrCC:
318 case X86Local::MRMDestReg:
319 case X86Local::MRMSrcReg:
320 case X86Local::MRMSrcReg4VOp3:
321 case X86Local::MRMSrcRegOp4:
322 case X86Local::MRMSrcRegCC:
323 return true;
324 }
325}
326
327static bool mayFoldToForm(uint8_t Form) {
328 switch (Form) {
329 default:
330 return Form >= X86Local::MRM0m && Form <= X86Local::MRM7m;
331 case X86Local::MRMXm:
332 case X86Local::MRMXmCC:
333 case X86Local::MRMDestMem:
334 case X86Local::MRMSrcMem:
335 case X86Local::MRMSrcMem4VOp3:
336 case X86Local::MRMSrcMemOp4:
337 case X86Local::MRMSrcMemCC:
338 return true;
339 }
340}
341
342static bool mayFoldFromLeftToRight(uint8_t LHS, uint8_t RHS) {
343 switch (LHS) {
344 default:
345 llvm_unreachable("Unexpected Form!");
346 case X86Local::MRM0r:
347 return RHS == X86Local::MRM0m;
348 case X86Local::MRM1r:
349 return RHS == X86Local::MRM1m;
350 case X86Local::MRM2r:
351 return RHS == X86Local::MRM2m;
352 case X86Local::MRM3r:
353 return RHS == X86Local::MRM3m;
354 case X86Local::MRM4r:
355 return RHS == X86Local::MRM4m;
356 case X86Local::MRM5r:
357 return RHS == X86Local::MRM5m;
358 case X86Local::MRM6r:
359 return RHS == X86Local::MRM6m;
360 case X86Local::MRM7r:
361 return RHS == X86Local::MRM7m;
362 case X86Local::MRMXr:
363 return RHS == X86Local::MRMXm;
364 case X86Local::MRMXrCC:
365 return RHS == X86Local::MRMXmCC;
366 case X86Local::MRMDestReg:
367 return RHS == X86Local::MRMDestMem;
368 case X86Local::MRMSrcReg:
369 return RHS == X86Local::MRMSrcMem;
370 case X86Local::MRMSrcReg4VOp3:
371 return RHS == X86Local::MRMSrcMem4VOp3;
372 case X86Local::MRMSrcRegOp4:
373 return RHS == X86Local::MRMSrcMemOp4;
374 case X86Local::MRMSrcRegCC:
375 return RHS == X86Local::MRMSrcMemCC;
376 }
377}
378
379static bool isNOREXRegClass(const Record *Op) {
380 return Op->getName().contains(Other: "_NOREX");
381}
382
383// Function object - Operator() returns true if the given Reg instruction
384// matches the Mem instruction of this object.
385namespace {
386class IsMatch {
387 const CodeGenInstruction *MemInst;
388 const X86Disassembler::RecognizableInstrBase MemRI;
389 bool IsBroadcast;
390 const unsigned Variant;
391
392public:
393 IsMatch(const CodeGenInstruction *Inst, bool IsBroadcast, unsigned V)
394 : MemInst(Inst), MemRI(*MemInst), IsBroadcast(IsBroadcast), Variant(V) {}
395
396 bool operator()(const CodeGenInstruction *RegInst) {
397 X86Disassembler::RecognizableInstrBase RegRI(*RegInst);
398 const Record *RegRec = RegInst->TheDef;
399 const Record *MemRec = MemInst->TheDef;
400
401 // EVEX_B means different things for memory and register forms.
402 // register form: rounding control or SAE
403 // memory form: broadcast
404 if (IsBroadcast && (RegRI.HasEVEX_B || !MemRI.HasEVEX_B))
405 return false;
406 // EVEX_B indicates NDD for MAP4 instructions
407 if (!IsBroadcast && (RegRI.HasEVEX_B || MemRI.HasEVEX_B) &&
408 RegRI.OpMap != X86Local::T_MAP4)
409 return false;
410
411 if (!mayFoldFromLeftToRight(LHS: RegRI.Form, RHS: MemRI.Form))
412 return false;
413
414 // X86 encoding is crazy, e.g
415 //
416 // f3 0f c7 30 vmxon (%rax)
417 // f3 0f c7 f0 senduipi %rax
418 //
419 // This two instruction have similiar encoding fields but are unrelated
420 if (X86Disassembler::getMnemonic(I: MemInst, Variant) !=
421 X86Disassembler::getMnemonic(I: RegInst, Variant))
422 return false;
423
424 // Return false if any of the following fields of does not match.
425 if (std::tuple(RegRI.Encoding, RegRI.Opcode, RegRI.OpPrefix, RegRI.OpMap,
426 RegRI.OpSize, RegRI.AdSize, RegRI.HasREX_W, RegRI.HasVEX_4V,
427 RegRI.HasVEX_L, RegRI.IgnoresVEX_L, RegRI.IgnoresW,
428 RegRI.HasEVEX_K, RegRI.HasEVEX_KZ, RegRI.HasEVEX_L2,
429 RegRI.HasEVEX_NF, RegRec->getValueAsBit(FieldName: "hasEVEX_RC"),
430 RegRec->getValueAsBit(FieldName: "hasLockPrefix"),
431 RegRec->getValueAsBit(FieldName: "hasNoTrackPrefix")) !=
432 std::tuple(MemRI.Encoding, MemRI.Opcode, MemRI.OpPrefix, MemRI.OpMap,
433 MemRI.OpSize, MemRI.AdSize, MemRI.HasREX_W, MemRI.HasVEX_4V,
434 MemRI.HasVEX_L, MemRI.IgnoresVEX_L, MemRI.IgnoresW,
435 MemRI.HasEVEX_K, MemRI.HasEVEX_KZ, MemRI.HasEVEX_L2,
436 MemRI.HasEVEX_NF, MemRec->getValueAsBit(FieldName: "hasEVEX_RC"),
437 MemRec->getValueAsBit(FieldName: "hasLockPrefix"),
438 MemRec->getValueAsBit(FieldName: "hasNoTrackPrefix")))
439 return false;
440
441 // Make sure the sizes of the operands of both instructions suit each other.
442 // This is needed for instructions with intrinsic version (_Int).
443 // Where the only difference is the size of the operands.
444 // For example: VUCOMISDZrm and VUCOMISDrm_Int
445 // Also for instructions that their EVEX version was upgraded to work with
446 // k-registers. For example VPCMPEQBrm (xmm output register) and
447 // VPCMPEQBZ128rm (k register output register).
448 unsigned MemOutSize = MemRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
449 unsigned RegOutSize = RegRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
450 unsigned MemInSize = MemRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
451 unsigned RegInSize = RegRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
452
453 // Instructions with one output in their memory form use the memory folded
454 // operand as source and destination (Read-Modify-Write).
455 unsigned RegStartIdx =
456 (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0;
457
458 bool FoundFoldedOp = false;
459 for (unsigned I = 0, E = MemInst->Operands.size(); I != E; I++) {
460 const Record *MemOpRec = MemInst->Operands[I].Rec;
461 const Record *RegOpRec = RegInst->Operands[I + RegStartIdx].Rec;
462
463 if (MemOpRec == RegOpRec)
464 continue;
465
466 if (isRegisterOperand(Rec: MemOpRec) && isRegisterOperand(Rec: RegOpRec) &&
467 ((getRegOperandSize(RegRec: MemOpRec) != getRegOperandSize(RegRec: RegOpRec)) ||
468 (isNOREXRegClass(Op: MemOpRec) != isNOREXRegClass(Op: RegOpRec))))
469 return false;
470
471 if (isMemoryOperand(Rec: MemOpRec) && isMemoryOperand(Rec: RegOpRec) &&
472 (getMemOperandSize(MemRec: MemOpRec) != getMemOperandSize(MemRec: RegOpRec)))
473 return false;
474
475 if (isImmediateOperand(Rec: MemOpRec) && isImmediateOperand(Rec: RegOpRec) &&
476 (MemOpRec->getValueAsDef(FieldName: "Type") != RegOpRec->getValueAsDef(FieldName: "Type")))
477 return false;
478
479 // Only one operand can be folded.
480 if (FoundFoldedOp)
481 return false;
482
483 assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec));
484 FoundFoldedOp = true;
485 }
486
487 return FoundFoldedOp;
488 }
489};
490
491} // end anonymous namespace
492
493void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table,
494 const CodeGenInstruction *RegInst,
495 const CodeGenInstruction *MemInst,
496 uint16_t S, unsigned FoldedIdx,
497 bool IsManual) {
498
499 assert((IsManual || Table.find(RegInst) == Table.end()) &&
500 "Override entry unexpectedly");
501 X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst);
502 const Record *RegRec = RegInst->TheDef;
503 Result.NoReverse = S & TB_NO_REVERSE;
504 Result.NoForward = S & TB_NO_FORWARD;
505 Result.FoldLoad = S & TB_FOLDED_LOAD;
506 Result.FoldStore = S & TB_FOLDED_STORE;
507 Result.Alignment = Align::fromLog2(Log2Value: (S & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT);
508 if (IsManual) {
509 Table[RegInst] = Result;
510 return;
511 }
512
513 const Record *RegOpRec = RegInst->Operands[FoldedIdx].Rec;
514 const Record *MemOpRec = MemInst->Operands[FoldedIdx].Rec;
515
516 // Unfolding code generates a load/store instruction according to the size of
517 // the register in the register form instruction.
518 // If the register's size is greater than the memory's operand size, do not
519 // allow unfolding.
520
521 // the unfolded load size will be based on the register size. If that’s bigger
522 // than the memory operand size, the unfolded load will load more memory and
523 // potentially cause a memory fault.
524 if (getRegOperandSize(RegRec: RegOpRec) > getMemOperandSize(MemRec: MemOpRec))
525 Result.NoReverse = true;
526
527 // Check no-kz version's isMoveReg
528 StringRef RegInstName = RegRec->getName();
529 unsigned DropLen =
530 RegInstName.ends_with(Suffix: "rkz") ? 2 : (RegInstName.ends_with(Suffix: "rk") ? 1 : 0);
531 const Record *BaseDef =
532 DropLen ? Records.getDef(Name: RegInstName.drop_back(N: DropLen)) : nullptr;
533 bool IsMoveReg =
534 BaseDef ? Target.getInstruction(InstRec: BaseDef).isMoveReg : RegInst->isMoveReg;
535 // A masked load can not be unfolded to a full load, otherwise it would access
536 // unexpected memory. A simple store can not be unfolded.
537 if (IsMoveReg && (BaseDef || Result.FoldStore))
538 Result.NoReverse = true;
539
540 uint8_t Enc = byteFromBitsInit(B: RegRec->getValueAsBitsInit(FieldName: "OpEncBits"));
541 if (isExplicitAlign(Inst: RegInst)) {
542 // The instruction require explicitly aligned memory.
543 const BitsInit *VectSize = RegRec->getValueAsBitsInit(FieldName: "VectSize");
544 Result.Alignment = Align(byteFromBitsInit(B: VectSize));
545 } else if (!Enc && !isExplicitUnalign(Inst: RegInst) &&
546 getMemOperandSize(MemRec: MemOpRec) > 64) {
547 // Instructions with XOP/VEX/EVEX encoding do not require alignment while
548 // SSE packed vector instructions require a 16 byte alignment.
549 Result.Alignment = Align(16);
550 }
551 // Expand is only ever created as a masked instruction. It is not safe to
552 // unfold a masked expand because we don't know if it came from an expand load
553 // intrinsic or folding a plain load. If it is from a expand load intrinsic,
554 // Unfolding to plain load would read more elements and could trigger a fault.
555 if (RegRec->getName().contains(Other: "EXPAND"))
556 Result.NoReverse = true;
557
558 Table[RegInst] = Result;
559}
560
561void X86FoldTablesEmitter::addBroadcastEntry(
562 FoldTable &Table, const CodeGenInstruction *RegInst,
563 const CodeGenInstruction *MemInst) {
564
565 assert(Table.find(RegInst) == Table.end() && "Override entry unexpectedly");
566 X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst);
567
568 const DagInit *In = MemInst->TheDef->getValueAsDag(FieldName: "InOperandList");
569 for (unsigned I = 0, E = In->getNumArgs(); I != E; ++I) {
570 Result.BroadcastKind =
571 StringSwitch<X86FoldTableEntry::BcastType>(In->getArg(Num: I)->getAsString())
572 .Case(S: "i16mem", Value: X86FoldTableEntry::BCAST_W)
573 .Case(S: "i32mem", Value: X86FoldTableEntry::BCAST_D)
574 .Case(S: "i64mem", Value: X86FoldTableEntry::BCAST_Q)
575 .Case(S: "f16mem", Value: X86FoldTableEntry::BCAST_SH)
576 .Case(S: "f32mem", Value: X86FoldTableEntry::BCAST_SS)
577 .Case(S: "f64mem", Value: X86FoldTableEntry::BCAST_SD)
578 .Default(Value: X86FoldTableEntry::BCAST_NONE);
579 if (Result.BroadcastKind != X86FoldTableEntry::BCAST_NONE)
580 break;
581 }
582 assert(Result.BroadcastKind != X86FoldTableEntry::BCAST_NONE &&
583 "Unknown memory operand for broadcast");
584
585 Table[RegInst] = Result;
586}
587
588void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInst,
589 const CodeGenInstruction *MemInst,
590 uint16_t S, bool IsManual,
591 bool IsBroadcast) {
592
593 const Record *RegRec = RegInst->TheDef;
594 const Record *MemRec = MemInst->TheDef;
595 unsigned MemOutSize = MemRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
596 unsigned RegOutSize = RegRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
597 unsigned MemInSize = MemRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
598 unsigned RegInSize = RegRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
599
600 // Instructions which Read-Modify-Write should be added to Table2Addr.
601 if (!MemOutSize && RegOutSize == 1 && MemInSize == RegInSize) {
602 assert(!IsBroadcast && "Read-Modify-Write can not be broadcast");
603 // X86 would not unfold Read-Modify-Write instructions so add TB_NO_REVERSE.
604 addEntryWithFlags(Table&: Table2Addr, RegInst, MemInst, S: S | TB_NO_REVERSE, FoldedIdx: 0,
605 IsManual);
606 return;
607 }
608
609 // Only table0 entries should explicitly specify a load or store flag.
610 // If the instruction writes to the folded operand, it will appear as
611 // an output in the register form instruction and as an input in the
612 // memory form instruction. If the instruction reads from the folded
613 // operand, it will appear as in input in both forms.
614 if (MemInSize == RegInSize && MemOutSize == RegOutSize) {
615 // Load-Folding cases.
616 // If the i'th register form operand is a register and the i'th memory form
617 // operand is a memory operand, add instructions to Table#i.
618 for (unsigned I = RegOutSize, E = RegInst->Operands.size(); I < E; I++) {
619 const Record *RegOpRec = RegInst->Operands[I].Rec;
620 const Record *MemOpRec = MemInst->Operands[I].Rec;
621 // RegClassByHwMode: For instructions like TAILJMPr, TAILJMPr64,
622 // TAILJMPr64_REX
623 if ((isRegisterOperand(Rec: RegOpRec) ||
624 (RegOpRec->isSubClassOf(Name: "RegClassByHwMode"))) &&
625 isMemoryOperand(Rec: MemOpRec)) {
626 switch (I) {
627 case 0:
628 assert(!IsBroadcast && "BroadcastTable0 needs to be added");
629 addEntryWithFlags(Table&: Table0, RegInst, MemInst, S: S | TB_FOLDED_LOAD, FoldedIdx: 0,
630 IsManual);
631 return;
632 case 1:
633 IsBroadcast
634 ? addBroadcastEntry(Table&: BroadcastTable1, RegInst, MemInst)
635 : addEntryWithFlags(Table&: Table1, RegInst, MemInst, S, FoldedIdx: 1, IsManual);
636 return;
637 case 2:
638 IsBroadcast
639 ? addBroadcastEntry(Table&: BroadcastTable2, RegInst, MemInst)
640 : addEntryWithFlags(Table&: Table2, RegInst, MemInst, S, FoldedIdx: 2, IsManual);
641 return;
642 case 3:
643 IsBroadcast
644 ? addBroadcastEntry(Table&: BroadcastTable3, RegInst, MemInst)
645 : addEntryWithFlags(Table&: Table3, RegInst, MemInst, S, FoldedIdx: 3, IsManual);
646 return;
647 case 4:
648 IsBroadcast
649 ? addBroadcastEntry(Table&: BroadcastTable4, RegInst, MemInst)
650 : addEntryWithFlags(Table&: Table4, RegInst, MemInst, S, FoldedIdx: 4, IsManual);
651 return;
652 }
653 }
654 }
655 } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) {
656 // Store-Folding cases.
657 // If the memory form instruction performs a store, the *output*
658 // register of the register form instructions disappear and instead a
659 // memory *input* operand appears in the memory form instruction.
660 // For example:
661 // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src)
662 // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src)
663 const Record *RegOpRec = RegInst->Operands[RegOutSize - 1].Rec;
664 const Record *MemOpRec = MemInst->Operands[RegOutSize - 1].Rec;
665 if (isRegisterOperand(Rec: RegOpRec) && isMemoryOperand(Rec: MemOpRec) &&
666 getRegOperandSize(RegRec: RegOpRec) == getMemOperandSize(MemRec: MemOpRec)) {
667 assert(!IsBroadcast && "Store can not be broadcast");
668 addEntryWithFlags(Table&: Table0, RegInst, MemInst, S: S | TB_FOLDED_STORE, FoldedIdx: 0,
669 IsManual);
670 }
671 }
672}
673
674void X86FoldTablesEmitter::run(raw_ostream &OS) {
675 // Holds all memory instructions
676 std::vector<const CodeGenInstruction *> MemInsts;
677 // Holds all register instructions - divided according to opcode.
678 std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts;
679
680 ArrayRef<const CodeGenInstruction *> NumberedInstructions =
681 Target.getInstructions();
682
683 for (const CodeGenInstruction *Inst : NumberedInstructions) {
684 const Record *Rec = Inst->TheDef;
685 if (!Rec->isSubClassOf(Name: "X86Inst") || Rec->getValueAsBit(FieldName: "isAsmParserOnly"))
686 continue;
687
688 if (NoFoldSet.find(x: Rec->getName()) != NoFoldSet.end())
689 continue;
690
691 // Promoted legacy instruction is in EVEX space, and has REX2-encoding
692 // alternative. It's added due to HW design and never emitted by compiler.
693 if (byteFromBitsInit(B: Rec->getValueAsBitsInit(FieldName: "OpMapBits")) ==
694 X86Local::T_MAP4 &&
695 byteFromBitsInit(B: Rec->getValueAsBitsInit(FieldName: "explicitOpPrefixBits")) ==
696 X86Local::ExplicitEVEX)
697 continue;
698
699 // - Instructions including RST register class operands are not relevant
700 // for memory folding (for further details check the explanation in
701 // lib/Target/X86/X86InstrFPStack.td file).
702 // - Some instructions (listed in the manual map above) use the register
703 // class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure
704 // safe mapping of these instruction we manually map them and exclude
705 // them from the automation.
706 if (hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst))
707 continue;
708
709 // Check if this instruction has a prefix in NoFoldSameMaskPrefixSet or is
710 // in NoFoldSameMaskSet (problematic for masked-load folding) and add to
711 // NonFoldableWithSameMaskTable.
712 if (isNoFoldMaskedInstruction(Inst)) {
713 NonFoldableWithSameMaskTable.push_back(x: Inst);
714 }
715
716 // Add all the memory form instructions to MemInsts, and all the register
717 // form instructions to RegInsts[Opc], where Opc is the opcode of each
718 // instructions. this helps reducing the runtime of the backend.
719 const BitsInit *FormBits = Rec->getValueAsBitsInit(FieldName: "FormBits");
720 uint8_t Form = byteFromBitsInit(B: FormBits);
721 if (mayFoldToForm(Form))
722 MemInsts.push_back(x: Inst);
723 else if (mayFoldFromForm(Form)) {
724 uint8_t Opc = byteFromBitsInit(B: Rec->getValueAsBitsInit(FieldName: "Opcode"));
725 RegInsts[Opc].push_back(x: Inst);
726 }
727 }
728
729 // Create a copy b/c the register instruction will removed when a new entry is
730 // added into memory fold tables.
731 auto RegInstsForBroadcast = RegInsts;
732
733 const Record *AsmWriter = Target.getAsmWriter();
734 unsigned Variant = AsmWriter->getValueAsInt(FieldName: "Variant");
735 auto FixUp = [&](const CodeGenInstruction *RegInst) {
736 StringRef RegInstName = RegInst->getName();
737 if (RegInstName.ends_with(Suffix: "_REV") || RegInstName.ends_with(Suffix: "_alt"))
738 if (auto *RegAltRec = Records.getDef(Name: RegInstName.drop_back(N: 4)))
739 RegInst = &Target.getInstruction(InstRec: RegAltRec);
740 return RegInst;
741 };
742 // For each memory form instruction, try to find its register form
743 // instruction.
744 for (const CodeGenInstruction *MemInst : MemInsts) {
745 uint8_t Opc =
746 byteFromBitsInit(B: MemInst->TheDef->getValueAsBitsInit(FieldName: "Opcode"));
747
748 auto RegInstsIt = RegInsts.find(x: Opc);
749 if (RegInstsIt == RegInsts.end())
750 continue;
751
752 // Two forms (memory & register) of the same instruction must have the same
753 // opcode.
754 std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second;
755
756 // Memory fold tables
757 auto Match =
758 find_if(Range&: OpcRegInsts, P: IsMatch(MemInst, /*IsBroadcast=*/false, Variant));
759 if (Match != OpcRegInsts.end()) {
760 updateTables(RegInst: FixUp(*Match), MemInst);
761 OpcRegInsts.erase(position: Match);
762 }
763
764 // Broadcast tables
765 StringRef MemInstName = MemInst->getName();
766 if (!MemInstName.contains(Other: "mb") && !MemInstName.contains(Other: "mib"))
767 continue;
768 RegInstsIt = RegInstsForBroadcast.find(x: Opc);
769 assert(RegInstsIt != RegInstsForBroadcast.end() &&
770 "Unexpected control flow");
771 std::vector<const CodeGenInstruction *> &OpcRegInstsForBroadcast =
772 RegInstsIt->second;
773 Match = find_if(Range&: OpcRegInstsForBroadcast,
774 P: IsMatch(MemInst, /*IsBroadcast=*/true, Variant));
775 if (Match != OpcRegInstsForBroadcast.end()) {
776 updateTables(RegInst: FixUp(*Match), MemInst, S: 0, /*IsManual=*/false,
777 /*IsBroadcast=*/true);
778 OpcRegInstsForBroadcast.erase(position: Match);
779 }
780 }
781
782 // Add the manually mapped instructions listed above.
783 for (const ManualMapEntry &Entry : ManualMapSet) {
784 const Record *RegInstIter = Records.getDef(Name: Entry.RegInstStr);
785 const Record *MemInstIter = Records.getDef(Name: Entry.MemInstStr);
786
787 updateTables(RegInst: &(Target.getInstruction(InstRec: RegInstIter)),
788 MemInst: &(Target.getInstruction(InstRec: MemInstIter)), S: Entry.Strategy, IsManual: true);
789 }
790
791#ifndef NDEBUG
792 auto CheckMemFoldTable = [](const FoldTable &Table) -> void {
793 for (const auto &Record : Table) {
794 auto &FoldEntry = Record.second;
795 FoldEntry.checkCorrectness();
796 }
797 };
798 CheckMemFoldTable(Table2Addr);
799 CheckMemFoldTable(Table0);
800 CheckMemFoldTable(Table1);
801 CheckMemFoldTable(Table2);
802 CheckMemFoldTable(Table3);
803 CheckMemFoldTable(Table4);
804 CheckMemFoldTable(BroadcastTable1);
805 CheckMemFoldTable(BroadcastTable2);
806 CheckMemFoldTable(BroadcastTable3);
807 CheckMemFoldTable(BroadcastTable4);
808#endif
809
810 // Build the memory unfolding table from the reversible entries of all the
811 // fold tables, keyed by the memory instruction.
812 // Index 0, folded load and store, no alignment requirement.
813 addUnfoldEntries(Table: Table2Addr, ExtraFlags: TB_INDEX_0 | TB_FOLDED_LOAD | TB_FOLDED_STORE);
814 // Index 0, mix of loads and stores.
815 addUnfoldEntries(Table: Table0, ExtraFlags: TB_INDEX_0);
816 // Index 1, folded load
817 addUnfoldEntries(Table: Table1, ExtraFlags: TB_INDEX_1 | TB_FOLDED_LOAD);
818 // Index 2, folded load
819 addUnfoldEntries(Table: Table2, ExtraFlags: TB_INDEX_2 | TB_FOLDED_LOAD);
820 // Index 3, folded load
821 addUnfoldEntries(Table: Table3, ExtraFlags: TB_INDEX_3 | TB_FOLDED_LOAD);
822 // Index 4, folded load
823 addUnfoldEntries(Table: Table4, ExtraFlags: TB_INDEX_4 | TB_FOLDED_LOAD);
824 // Broadcast tables.
825 // Index 1, folded broadcast
826 addUnfoldEntries(Table: BroadcastTable1, ExtraFlags: TB_INDEX_1 | TB_FOLDED_LOAD);
827 // Index 2, folded broadcast
828 addUnfoldEntries(Table: BroadcastTable2, ExtraFlags: TB_INDEX_2 | TB_FOLDED_LOAD);
829 // Index 3, folded broadcast
830 addUnfoldEntries(Table: BroadcastTable3, ExtraFlags: TB_INDEX_3 | TB_FOLDED_LOAD);
831 // Index 4, folded broadcast
832 addUnfoldEntries(Table: BroadcastTable4, ExtraFlags: TB_INDEX_4 | TB_FOLDED_LOAD);
833
834#define PRINT_TABLE(TABLE) printTable(TABLE, #TABLE, OS);
835 // Print all tables.
836 PRINT_TABLE(Table2Addr)
837 PRINT_TABLE(Table0)
838 PRINT_TABLE(Table1)
839 PRINT_TABLE(Table2)
840 PRINT_TABLE(Table3)
841 PRINT_TABLE(Table4)
842 PRINT_TABLE(BroadcastTable1)
843 PRINT_TABLE(BroadcastTable2)
844 PRINT_TABLE(BroadcastTable3)
845 PRINT_TABLE(BroadcastTable4)
846 PRINT_TABLE(NonFoldableWithSameMaskTable)
847 PRINT_TABLE(MemUnfoldTable)
848}
849
850void X86FoldTablesEmitter::addUnfoldEntries(const FoldTable &Table,
851 uint16_t ExtraFlags) {
852 for (const auto &[RegInst, Entry] : Table) {
853 if (Entry.NoReverse)
854 continue;
855 X86FoldTableEntry UnfoldEntry = Entry;
856 std::swap(a&: UnfoldEntry.RegInst, b&: UnfoldEntry.MemInst);
857 UnfoldEntry.UnfoldIdx = ExtraFlags & TB_INDEX_MASK;
858 UnfoldEntry.FoldLoad |= ExtraFlags & TB_FOLDED_LOAD;
859 UnfoldEntry.FoldStore |= ExtraFlags & TB_FOLDED_STORE;
860 if (!MemUnfoldTable.try_emplace(k: Entry.MemInst, args&: UnfoldEntry).second)
861 PrintFatalError(Rec: Entry.MemInst->TheDef,
862 Msg: "Memory unfolding table is not unique: " +
863 Entry.MemInst->getName() + " unfolds to both " +
864 MemUnfoldTable.at(k: Entry.MemInst).MemInst->getName() +
865 " and " + RegInst->getName());
866 }
867}
868
869static TableGen::Emitter::OptClass<X86FoldTablesEmitter>
870 X("gen-x86-fold-tables", "Generate X86 fold tables");
871