1//===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend is responsible for emitting the memory fold tables of
10// the X86 backend instructions.
11//
12//===----------------------------------------------------------------------===//
13
14#include "Common/CodeGenInstruction.h"
15#include "Common/CodeGenTarget.h"
16#include "X86RecognizableInstr.h"
17#include "llvm/ADT/StringSwitch.h"
18#include "llvm/Support/X86FoldTablesUtils.h"
19#include "llvm/TableGen/Record.h"
20#include "llvm/TableGen/TableGenBackend.h"
21#include <set>
22
23using namespace llvm;
24using namespace X86Disassembler;
25
26namespace {
27// Represents an entry in the manual mapped instructions set.
28struct ManualMapEntry {
29 const char *RegInstStr;
30 const char *MemInstStr;
31 uint16_t Strategy;
32};
33} // namespace
34
35// List of instructions requiring explicitly aligned memory.
36static constexpr const char *ExplicitAlign[] = {
37 "MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", "MOVNTPD", "MOVNTDQ", "MOVNTDQA"};
38
39// List of instructions NOT requiring explicit memory alignment.
40static constexpr const char *ExplicitUnalign[] = {
41 "MOVDQU", "MOVUPS", "MOVUPD", "PCMPESTRM",
42 "PCMPESTRI", "PCMPISTRM", "PCMPISTRI"};
43
44static const ManualMapEntry ManualMapSet[] = {
45#define ENTRY(REG, MEM, FLAGS) {#REG, #MEM, FLAGS},
46#include "X86ManualFoldTables.def"
47};
48
49static const std::set<StringRef> NoFoldSet = {
50#define NOFOLD(INSN) #INSN,
51#include "X86ManualFoldTables.def"
52};
53
54static bool isExplicitAlign(const CodeGenInstruction *Inst) {
55 return any_of(Range: ExplicitAlign, P: [Inst](const char *InstStr) {
56 return Inst->getName().contains(Other: InstStr);
57 });
58}
59
60static bool isExplicitUnalign(const CodeGenInstruction *Inst) {
61 return any_of(Range: ExplicitUnalign, P: [Inst](const char *InstStr) {
62 return Inst->getName().contains(Other: InstStr);
63 });
64}
65
66namespace {
67class X86FoldTablesEmitter {
68 const RecordKeeper &Records;
69 const CodeGenTarget Target;
70
71 // Represents an entry in the folding table
72 class X86FoldTableEntry {
73 const CodeGenInstruction *RegInst;
74 const CodeGenInstruction *MemInst;
75
76 public:
77 bool NoReverse = false;
78 bool NoForward = false;
79 bool FoldLoad = false;
80 bool FoldStore = false;
81 enum BcastType {
82 BCAST_NONE,
83 BCAST_W,
84 BCAST_D,
85 BCAST_Q,
86 BCAST_SS,
87 BCAST_SD,
88 BCAST_SH,
89 };
90 BcastType BroadcastKind = BCAST_NONE;
91
92 Align Alignment;
93
94 X86FoldTableEntry() = default;
95 X86FoldTableEntry(const CodeGenInstruction *RegInst,
96 const CodeGenInstruction *MemInst)
97 : RegInst(RegInst), MemInst(MemInst) {}
98
99 void print(raw_ostream &OS) const {
100 OS.indent(NumSpaces: 2);
101 OS << "{X86::" << RegInst->getName() << ", ";
102 OS << "X86::" << MemInst->getName() << ", ";
103
104 std::string Attrs;
105 if (FoldLoad)
106 Attrs += "TB_FOLDED_LOAD|";
107 if (FoldStore)
108 Attrs += "TB_FOLDED_STORE|";
109 if (NoReverse)
110 Attrs += "TB_NO_REVERSE|";
111 if (NoForward)
112 Attrs += "TB_NO_FORWARD|";
113 if (Alignment != Align(1))
114 Attrs += "TB_ALIGN_" + std::to_string(val: Alignment.value()) + "|";
115 switch (BroadcastKind) {
116 case BCAST_NONE:
117 break;
118 case BCAST_W:
119 Attrs += "TB_BCAST_W|";
120 break;
121 case BCAST_D:
122 Attrs += "TB_BCAST_D|";
123 break;
124 case BCAST_Q:
125 Attrs += "TB_BCAST_Q|";
126 break;
127 case BCAST_SS:
128 Attrs += "TB_BCAST_SS|";
129 break;
130 case BCAST_SD:
131 Attrs += "TB_BCAST_SD|";
132 break;
133 case BCAST_SH:
134 Attrs += "TB_BCAST_SH|";
135 break;
136 }
137
138 StringRef SimplifiedAttrs = StringRef(Attrs).rtrim(Chars: "|");
139 if (SimplifiedAttrs.empty())
140 SimplifiedAttrs = "0";
141
142 OS << SimplifiedAttrs << "},\n";
143 }
144
145#ifndef NDEBUG
146 // Check that Uses and Defs are same after memory fold.
147 void checkCorrectness() const {
148 auto &RegInstRec = *RegInst->TheDef;
149 auto &MemInstRec = *MemInst->TheDef;
150 auto ListOfUsesReg = RegInstRec.getValueAsListOfDefs("Uses");
151 auto ListOfUsesMem = MemInstRec.getValueAsListOfDefs("Uses");
152 auto ListOfDefsReg = RegInstRec.getValueAsListOfDefs("Defs");
153 auto ListOfDefsMem = MemInstRec.getValueAsListOfDefs("Defs");
154 if (ListOfUsesReg != ListOfUsesMem || ListOfDefsReg != ListOfDefsMem)
155 report_fatal_error("Uses/Defs couldn't be changed after folding " +
156 RegInstRec.getName() + " to " +
157 MemInstRec.getName());
158 }
159#endif
160 };
161
162 // NOTE: We check the fold tables are sorted in X86InstrFoldTables.cpp by the
163 // enum of the instruction, which is computed in
164 // CodeGenTarget::ComputeInstrsByEnum. So we should use the same comparator
165 // here.
166 // FIXME: Could we share the code with CodeGenTarget::ComputeInstrsByEnum?
167 struct CompareInstrsByEnum {
168 bool operator()(const CodeGenInstruction *LHS,
169 const CodeGenInstruction *RHS) const {
170 assert(LHS && RHS && "LHS and RHS shouldn't be nullptr");
171 const auto &D1 = *LHS->TheDef;
172 const auto &D2 = *RHS->TheDef;
173 return std::tuple(!D1.getValueAsBit(FieldName: "isPseudo"), D1.getName()) <
174 std::tuple(!D2.getValueAsBit(FieldName: "isPseudo"), D2.getName());
175 }
176 };
177
178 using FoldTable = std::map<const CodeGenInstruction *, X86FoldTableEntry,
179 CompareInstrsByEnum>;
180 // Table2Addr - Holds instructions which their memory form performs
181 // load+store.
182 //
183 // Table#i - Holds instructions which the their memory form
184 // performs a load OR a store, and their #i'th operand is folded.
185 //
186 // BroadcastTable#i - Holds instructions which the their memory form performs
187 // a broadcast load and their #i'th operand is folded.
188 FoldTable Table2Addr;
189 FoldTable Table0;
190 FoldTable Table1;
191 FoldTable Table2;
192 FoldTable Table3;
193 FoldTable Table4;
194 FoldTable BroadcastTable1;
195 FoldTable BroadcastTable2;
196 FoldTable BroadcastTable3;
197 FoldTable BroadcastTable4;
198
199public:
200 X86FoldTablesEmitter(const RecordKeeper &R) : Records(R), Target(R) {}
201
202 // run - Generate the 6 X86 memory fold tables.
203 void run(raw_ostream &OS);
204
205private:
206 // Decides to which table to add the entry with the given instructions.
207 // S sets the strategy of adding the TB_NO_REVERSE flag.
208 void updateTables(const CodeGenInstruction *RegInst,
209 const CodeGenInstruction *MemInst, uint16_t S = 0,
210 bool IsManual = false, bool IsBroadcast = false);
211
212 // Generates X86FoldTableEntry with the given instructions and fill it with
213 // the appropriate flags, then adds it to a memory fold table.
214 void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInst,
215 const CodeGenInstruction *MemInst, uint16_t S,
216 unsigned FoldedIdx, bool IsManual);
217 // Generates X86FoldTableEntry with the given instructions and adds it to a
218 // broadcast table.
219 void addBroadcastEntry(FoldTable &Table, const CodeGenInstruction *RegInst,
220 const CodeGenInstruction *MemInst);
221
222 // Print the given table as a static const C++ array of type
223 // X86FoldTableEntry.
224 void printTable(const FoldTable &Table, StringRef TableName,
225 raw_ostream &OS) {
226 OS << "static const X86FoldTableEntry " << TableName << "[] = {\n";
227
228 for (auto &E : Table)
229 E.second.print(OS);
230
231 OS << "};\n\n";
232 }
233};
234} // namespace
235
236// Return true if one of the instruction's operands is a RST register class
237static bool hasRSTRegClass(const CodeGenInstruction *Inst) {
238 return any_of(Range: Inst->Operands, P: [](const CGIOperandList::OperandInfo &OpIn) {
239 return OpIn.Rec->getName() == "RST" || OpIn.Rec->getName() == "RSTi";
240 });
241}
242
243// Return true if one of the instruction's operands is a ptr_rc_tailcall
244static bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) {
245 return any_of(Range: Inst->Operands, P: [](const CGIOperandList::OperandInfo &OpIn) {
246 return OpIn.Rec->getName() == "ptr_rc_tailcall";
247 });
248}
249
250static bool mayFoldFromForm(uint8_t Form) {
251 switch (Form) {
252 default:
253 return Form >= X86Local::MRM0r && Form <= X86Local::MRM7r;
254 case X86Local::MRMXr:
255 case X86Local::MRMXrCC:
256 case X86Local::MRMDestReg:
257 case X86Local::MRMSrcReg:
258 case X86Local::MRMSrcReg4VOp3:
259 case X86Local::MRMSrcRegOp4:
260 case X86Local::MRMSrcRegCC:
261 return true;
262 }
263}
264
265static bool mayFoldToForm(uint8_t Form) {
266 switch (Form) {
267 default:
268 return Form >= X86Local::MRM0m && Form <= X86Local::MRM7m;
269 case X86Local::MRMXm:
270 case X86Local::MRMXmCC:
271 case X86Local::MRMDestMem:
272 case X86Local::MRMSrcMem:
273 case X86Local::MRMSrcMem4VOp3:
274 case X86Local::MRMSrcMemOp4:
275 case X86Local::MRMSrcMemCC:
276 return true;
277 }
278}
279
280static bool mayFoldFromLeftToRight(uint8_t LHS, uint8_t RHS) {
281 switch (LHS) {
282 default:
283 llvm_unreachable("Unexpected Form!");
284 case X86Local::MRM0r:
285 return RHS == X86Local::MRM0m;
286 case X86Local::MRM1r:
287 return RHS == X86Local::MRM1m;
288 case X86Local::MRM2r:
289 return RHS == X86Local::MRM2m;
290 case X86Local::MRM3r:
291 return RHS == X86Local::MRM3m;
292 case X86Local::MRM4r:
293 return RHS == X86Local::MRM4m;
294 case X86Local::MRM5r:
295 return RHS == X86Local::MRM5m;
296 case X86Local::MRM6r:
297 return RHS == X86Local::MRM6m;
298 case X86Local::MRM7r:
299 return RHS == X86Local::MRM7m;
300 case X86Local::MRMXr:
301 return RHS == X86Local::MRMXm;
302 case X86Local::MRMXrCC:
303 return RHS == X86Local::MRMXmCC;
304 case X86Local::MRMDestReg:
305 return RHS == X86Local::MRMDestMem;
306 case X86Local::MRMSrcReg:
307 return RHS == X86Local::MRMSrcMem;
308 case X86Local::MRMSrcReg4VOp3:
309 return RHS == X86Local::MRMSrcMem4VOp3;
310 case X86Local::MRMSrcRegOp4:
311 return RHS == X86Local::MRMSrcMemOp4;
312 case X86Local::MRMSrcRegCC:
313 return RHS == X86Local::MRMSrcMemCC;
314 }
315}
316
317static bool isNOREXRegClass(const Record *Op) {
318 return Op->getName().contains(Other: "_NOREX");
319}
320
321// Function object - Operator() returns true if the given Reg instruction
322// matches the Mem instruction of this object.
323namespace {
324class IsMatch {
325 const CodeGenInstruction *MemInst;
326 const X86Disassembler::RecognizableInstrBase MemRI;
327 bool IsBroadcast;
328 const unsigned Variant;
329
330public:
331 IsMatch(const CodeGenInstruction *Inst, bool IsBroadcast, unsigned V)
332 : MemInst(Inst), MemRI(*MemInst), IsBroadcast(IsBroadcast), Variant(V) {}
333
334 bool operator()(const CodeGenInstruction *RegInst) {
335 X86Disassembler::RecognizableInstrBase RegRI(*RegInst);
336 const Record *RegRec = RegInst->TheDef;
337 const Record *MemRec = MemInst->TheDef;
338
339 // EVEX_B means different things for memory and register forms.
340 // register form: rounding control or SAE
341 // memory form: broadcast
342 if (IsBroadcast && (RegRI.HasEVEX_B || !MemRI.HasEVEX_B))
343 return false;
344 // EVEX_B indicates NDD for MAP4 instructions
345 if (!IsBroadcast && (RegRI.HasEVEX_B || MemRI.HasEVEX_B) &&
346 RegRI.OpMap != X86Local::T_MAP4)
347 return false;
348
349 if (!mayFoldFromLeftToRight(LHS: RegRI.Form, RHS: MemRI.Form))
350 return false;
351
352 // X86 encoding is crazy, e.g
353 //
354 // f3 0f c7 30 vmxon (%rax)
355 // f3 0f c7 f0 senduipi %rax
356 //
357 // This two instruction have similiar encoding fields but are unrelated
358 if (X86Disassembler::getMnemonic(I: MemInst, Variant) !=
359 X86Disassembler::getMnemonic(I: RegInst, Variant))
360 return false;
361
362 // Return false if any of the following fields of does not match.
363 if (std::tuple(RegRI.Encoding, RegRI.Opcode, RegRI.OpPrefix, RegRI.OpMap,
364 RegRI.OpSize, RegRI.AdSize, RegRI.HasREX_W, RegRI.HasVEX_4V,
365 RegRI.HasVEX_L, RegRI.IgnoresVEX_L, RegRI.IgnoresW,
366 RegRI.HasEVEX_K, RegRI.HasEVEX_KZ, RegRI.HasEVEX_L2,
367 RegRI.HasEVEX_NF, RegRec->getValueAsBit(FieldName: "hasEVEX_RC"),
368 RegRec->getValueAsBit(FieldName: "hasLockPrefix"),
369 RegRec->getValueAsBit(FieldName: "hasNoTrackPrefix")) !=
370 std::tuple(MemRI.Encoding, MemRI.Opcode, MemRI.OpPrefix, MemRI.OpMap,
371 MemRI.OpSize, MemRI.AdSize, MemRI.HasREX_W, MemRI.HasVEX_4V,
372 MemRI.HasVEX_L, MemRI.IgnoresVEX_L, MemRI.IgnoresW,
373 MemRI.HasEVEX_K, MemRI.HasEVEX_KZ, MemRI.HasEVEX_L2,
374 MemRI.HasEVEX_NF, MemRec->getValueAsBit(FieldName: "hasEVEX_RC"),
375 MemRec->getValueAsBit(FieldName: "hasLockPrefix"),
376 MemRec->getValueAsBit(FieldName: "hasNoTrackPrefix")))
377 return false;
378
379 // Make sure the sizes of the operands of both instructions suit each other.
380 // This is needed for instructions with intrinsic version (_Int).
381 // Where the only difference is the size of the operands.
382 // For example: VUCOMISDZrm and VUCOMISDrm_Int
383 // Also for instructions that their EVEX version was upgraded to work with
384 // k-registers. For example VPCMPEQBrm (xmm output register) and
385 // VPCMPEQBZ128rm (k register output register).
386 unsigned MemOutSize = MemRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
387 unsigned RegOutSize = RegRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
388 unsigned MemInSize = MemRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
389 unsigned RegInSize = RegRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
390
391 // Instructions with one output in their memory form use the memory folded
392 // operand as source and destination (Read-Modify-Write).
393 unsigned RegStartIdx =
394 (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0;
395
396 bool FoundFoldedOp = false;
397 for (unsigned I = 0, E = MemInst->Operands.size(); I != E; I++) {
398 const Record *MemOpRec = MemInst->Operands[I].Rec;
399 const Record *RegOpRec = RegInst->Operands[I + RegStartIdx].Rec;
400
401 if (MemOpRec == RegOpRec)
402 continue;
403
404 if (isRegisterOperand(Rec: MemOpRec) && isRegisterOperand(Rec: RegOpRec) &&
405 ((getRegOperandSize(RegRec: MemOpRec) != getRegOperandSize(RegRec: RegOpRec)) ||
406 (isNOREXRegClass(Op: MemOpRec) != isNOREXRegClass(Op: RegOpRec))))
407 return false;
408
409 if (isMemoryOperand(Rec: MemOpRec) && isMemoryOperand(Rec: RegOpRec) &&
410 (getMemOperandSize(MemRec: MemOpRec) != getMemOperandSize(MemRec: RegOpRec)))
411 return false;
412
413 if (isImmediateOperand(Rec: MemOpRec) && isImmediateOperand(Rec: RegOpRec) &&
414 (MemOpRec->getValueAsDef(FieldName: "Type") != RegOpRec->getValueAsDef(FieldName: "Type")))
415 return false;
416
417 // Only one operand can be folded.
418 if (FoundFoldedOp)
419 return false;
420
421 assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec));
422 FoundFoldedOp = true;
423 }
424
425 return FoundFoldedOp;
426 }
427};
428
429} // end anonymous namespace
430
431void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table,
432 const CodeGenInstruction *RegInst,
433 const CodeGenInstruction *MemInst,
434 uint16_t S, unsigned FoldedIdx,
435 bool IsManual) {
436
437 assert((IsManual || Table.find(RegInst) == Table.end()) &&
438 "Override entry unexpectedly");
439 X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst);
440 const Record *RegRec = RegInst->TheDef;
441 Result.NoReverse = S & TB_NO_REVERSE;
442 Result.NoForward = S & TB_NO_FORWARD;
443 Result.FoldLoad = S & TB_FOLDED_LOAD;
444 Result.FoldStore = S & TB_FOLDED_STORE;
445 Result.Alignment = Align(1ULL << ((S & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT));
446 if (IsManual) {
447 Table[RegInst] = Result;
448 return;
449 }
450
451 const Record *RegOpRec = RegInst->Operands[FoldedIdx].Rec;
452 const Record *MemOpRec = MemInst->Operands[FoldedIdx].Rec;
453
454 // Unfolding code generates a load/store instruction according to the size of
455 // the register in the register form instruction.
456 // If the register's size is greater than the memory's operand size, do not
457 // allow unfolding.
458
459 // the unfolded load size will be based on the register size. If that’s bigger
460 // than the memory operand size, the unfolded load will load more memory and
461 // potentially cause a memory fault.
462 if (getRegOperandSize(RegRec: RegOpRec) > getMemOperandSize(MemRec: MemOpRec))
463 Result.NoReverse = true;
464
465 // Check no-kz version's isMoveReg
466 StringRef RegInstName = RegRec->getName();
467 unsigned DropLen =
468 RegInstName.ends_with(Suffix: "rkz") ? 2 : (RegInstName.ends_with(Suffix: "rk") ? 1 : 0);
469 const Record *BaseDef =
470 DropLen ? Records.getDef(Name: RegInstName.drop_back(N: DropLen)) : nullptr;
471 bool IsMoveReg =
472 BaseDef ? Target.getInstruction(InstRec: BaseDef).isMoveReg : RegInst->isMoveReg;
473 // A masked load can not be unfolded to a full load, otherwise it would access
474 // unexpected memory. A simple store can not be unfolded.
475 if (IsMoveReg && (BaseDef || Result.FoldStore))
476 Result.NoReverse = true;
477
478 uint8_t Enc = byteFromBitsInit(B: RegRec->getValueAsBitsInit(FieldName: "OpEncBits"));
479 if (isExplicitAlign(Inst: RegInst)) {
480 // The instruction require explicitly aligned memory.
481 const BitsInit *VectSize = RegRec->getValueAsBitsInit(FieldName: "VectSize");
482 Result.Alignment = Align(byteFromBitsInit(B: VectSize));
483 } else if (!Enc && !isExplicitUnalign(Inst: RegInst) &&
484 getMemOperandSize(MemRec: MemOpRec) > 64) {
485 // Instructions with XOP/VEX/EVEX encoding do not require alignment while
486 // SSE packed vector instructions require a 16 byte alignment.
487 Result.Alignment = Align(16);
488 }
489 // Expand is only ever created as a masked instruction. It is not safe to
490 // unfold a masked expand because we don't know if it came from an expand load
491 // intrinsic or folding a plain load. If it is from a expand load intrinsic,
492 // Unfolding to plain load would read more elements and could trigger a fault.
493 if (RegRec->getName().contains(Other: "EXPAND"))
494 Result.NoReverse = true;
495
496 Table[RegInst] = Result;
497}
498
499void X86FoldTablesEmitter::addBroadcastEntry(
500 FoldTable &Table, const CodeGenInstruction *RegInst,
501 const CodeGenInstruction *MemInst) {
502
503 assert(Table.find(RegInst) == Table.end() && "Override entry unexpectedly");
504 X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst);
505
506 const DagInit *In = MemInst->TheDef->getValueAsDag(FieldName: "InOperandList");
507 for (unsigned I = 0, E = In->getNumArgs(); I != E; ++I) {
508 Result.BroadcastKind =
509 StringSwitch<X86FoldTableEntry::BcastType>(In->getArg(Num: I)->getAsString())
510 .Case(S: "i16mem", Value: X86FoldTableEntry::BCAST_W)
511 .Case(S: "i32mem", Value: X86FoldTableEntry::BCAST_D)
512 .Case(S: "i64mem", Value: X86FoldTableEntry::BCAST_Q)
513 .Case(S: "f16mem", Value: X86FoldTableEntry::BCAST_SH)
514 .Case(S: "f32mem", Value: X86FoldTableEntry::BCAST_SS)
515 .Case(S: "f64mem", Value: X86FoldTableEntry::BCAST_SD)
516 .Default(Value: X86FoldTableEntry::BCAST_NONE);
517 if (Result.BroadcastKind != X86FoldTableEntry::BCAST_NONE)
518 break;
519 }
520 assert(Result.BroadcastKind != X86FoldTableEntry::BCAST_NONE &&
521 "Unknown memory operand for broadcast");
522
523 Table[RegInst] = Result;
524}
525
526void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInst,
527 const CodeGenInstruction *MemInst,
528 uint16_t S, bool IsManual,
529 bool IsBroadcast) {
530
531 const Record *RegRec = RegInst->TheDef;
532 const Record *MemRec = MemInst->TheDef;
533 unsigned MemOutSize = MemRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
534 unsigned RegOutSize = RegRec->getValueAsDag(FieldName: "OutOperandList")->getNumArgs();
535 unsigned MemInSize = MemRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
536 unsigned RegInSize = RegRec->getValueAsDag(FieldName: "InOperandList")->getNumArgs();
537
538 // Instructions which Read-Modify-Write should be added to Table2Addr.
539 if (!MemOutSize && RegOutSize == 1 && MemInSize == RegInSize) {
540 assert(!IsBroadcast && "Read-Modify-Write can not be broadcast");
541 // X86 would not unfold Read-Modify-Write instructions so add TB_NO_REVERSE.
542 addEntryWithFlags(Table&: Table2Addr, RegInst, MemInst, S: S | TB_NO_REVERSE, FoldedIdx: 0,
543 IsManual);
544 return;
545 }
546
547 // Only table0 entries should explicitly specify a load or store flag.
548 // If the instruction writes to the folded operand, it will appear as
549 // an output in the register form instruction and as an input in the
550 // memory form instruction. If the instruction reads from the folded
551 // operand, it will appear as in input in both forms.
552 if (MemInSize == RegInSize && MemOutSize == RegOutSize) {
553 // Load-Folding cases.
554 // If the i'th register form operand is a register and the i'th memory form
555 // operand is a memory operand, add instructions to Table#i.
556 for (unsigned I = RegOutSize, E = RegInst->Operands.size(); I < E; I++) {
557 const Record *RegOpRec = RegInst->Operands[I].Rec;
558 const Record *MemOpRec = MemInst->Operands[I].Rec;
559 // RegClassByHwMode: For instructions like TAILJMPr, TAILJMPr64,
560 // TAILJMPr64_REX
561 if ((isRegisterOperand(Rec: RegOpRec) ||
562 (RegOpRec->isSubClassOf(Name: "RegClassByHwMode"))) &&
563 isMemoryOperand(Rec: MemOpRec)) {
564 switch (I) {
565 case 0:
566 assert(!IsBroadcast && "BroadcastTable0 needs to be added");
567 addEntryWithFlags(Table&: Table0, RegInst, MemInst, S: S | TB_FOLDED_LOAD, FoldedIdx: 0,
568 IsManual);
569 return;
570 case 1:
571 IsBroadcast
572 ? addBroadcastEntry(Table&: BroadcastTable1, RegInst, MemInst)
573 : addEntryWithFlags(Table&: Table1, RegInst, MemInst, S, FoldedIdx: 1, IsManual);
574 return;
575 case 2:
576 IsBroadcast
577 ? addBroadcastEntry(Table&: BroadcastTable2, RegInst, MemInst)
578 : addEntryWithFlags(Table&: Table2, RegInst, MemInst, S, FoldedIdx: 2, IsManual);
579 return;
580 case 3:
581 IsBroadcast
582 ? addBroadcastEntry(Table&: BroadcastTable3, RegInst, MemInst)
583 : addEntryWithFlags(Table&: Table3, RegInst, MemInst, S, FoldedIdx: 3, IsManual);
584 return;
585 case 4:
586 IsBroadcast
587 ? addBroadcastEntry(Table&: BroadcastTable4, RegInst, MemInst)
588 : addEntryWithFlags(Table&: Table4, RegInst, MemInst, S, FoldedIdx: 4, IsManual);
589 return;
590 }
591 }
592 }
593 } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) {
594 // Store-Folding cases.
595 // If the memory form instruction performs a store, the *output*
596 // register of the register form instructions disappear and instead a
597 // memory *input* operand appears in the memory form instruction.
598 // For example:
599 // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src)
600 // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src)
601 const Record *RegOpRec = RegInst->Operands[RegOutSize - 1].Rec;
602 const Record *MemOpRec = MemInst->Operands[RegOutSize - 1].Rec;
603 if (isRegisterOperand(Rec: RegOpRec) && isMemoryOperand(Rec: MemOpRec) &&
604 getRegOperandSize(RegRec: RegOpRec) == getMemOperandSize(MemRec: MemOpRec)) {
605 assert(!IsBroadcast && "Store can not be broadcast");
606 addEntryWithFlags(Table&: Table0, RegInst, MemInst, S: S | TB_FOLDED_STORE, FoldedIdx: 0,
607 IsManual);
608 }
609 }
610}
611
612void X86FoldTablesEmitter::run(raw_ostream &OS) {
613 // Holds all memory instructions
614 std::vector<const CodeGenInstruction *> MemInsts;
615 // Holds all register instructions - divided according to opcode.
616 std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts;
617
618 ArrayRef<const CodeGenInstruction *> NumberedInstructions =
619 Target.getInstructions();
620
621 for (const CodeGenInstruction *Inst : NumberedInstructions) {
622 const Record *Rec = Inst->TheDef;
623 if (!Rec->isSubClassOf(Name: "X86Inst") || Rec->getValueAsBit(FieldName: "isAsmParserOnly"))
624 continue;
625
626 if (NoFoldSet.find(x: Rec->getName()) != NoFoldSet.end())
627 continue;
628
629 // Promoted legacy instruction is in EVEX space, and has REX2-encoding
630 // alternative. It's added due to HW design and never emitted by compiler.
631 if (byteFromBitsInit(B: Rec->getValueAsBitsInit(FieldName: "OpMapBits")) ==
632 X86Local::T_MAP4 &&
633 byteFromBitsInit(B: Rec->getValueAsBitsInit(FieldName: "explicitOpPrefixBits")) ==
634 X86Local::ExplicitEVEX)
635 continue;
636
637 // - Instructions including RST register class operands are not relevant
638 // for memory folding (for further details check the explanation in
639 // lib/Target/X86/X86InstrFPStack.td file).
640 // - Some instructions (listed in the manual map above) use the register
641 // class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure
642 // safe mapping of these instruction we manually map them and exclude
643 // them from the automation.
644 if (hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst))
645 continue;
646
647 // Add all the memory form instructions to MemInsts, and all the register
648 // form instructions to RegInsts[Opc], where Opc is the opcode of each
649 // instructions. this helps reducing the runtime of the backend.
650 const BitsInit *FormBits = Rec->getValueAsBitsInit(FieldName: "FormBits");
651 uint8_t Form = byteFromBitsInit(B: FormBits);
652 if (mayFoldToForm(Form))
653 MemInsts.push_back(x: Inst);
654 else if (mayFoldFromForm(Form)) {
655 uint8_t Opc = byteFromBitsInit(B: Rec->getValueAsBitsInit(FieldName: "Opcode"));
656 RegInsts[Opc].push_back(x: Inst);
657 }
658 }
659
660 // Create a copy b/c the register instruction will removed when a new entry is
661 // added into memory fold tables.
662 auto RegInstsForBroadcast = RegInsts;
663
664 const Record *AsmWriter = Target.getAsmWriter();
665 unsigned Variant = AsmWriter->getValueAsInt(FieldName: "Variant");
666 auto FixUp = [&](const CodeGenInstruction *RegInst) {
667 StringRef RegInstName = RegInst->getName();
668 if (RegInstName.ends_with(Suffix: "_REV") || RegInstName.ends_with(Suffix: "_alt"))
669 if (auto *RegAltRec = Records.getDef(Name: RegInstName.drop_back(N: 4)))
670 RegInst = &Target.getInstruction(InstRec: RegAltRec);
671 return RegInst;
672 };
673 // For each memory form instruction, try to find its register form
674 // instruction.
675 for (const CodeGenInstruction *MemInst : MemInsts) {
676 uint8_t Opc =
677 byteFromBitsInit(B: MemInst->TheDef->getValueAsBitsInit(FieldName: "Opcode"));
678
679 auto RegInstsIt = RegInsts.find(x: Opc);
680 if (RegInstsIt == RegInsts.end())
681 continue;
682
683 // Two forms (memory & register) of the same instruction must have the same
684 // opcode.
685 std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second;
686
687 // Memory fold tables
688 auto Match =
689 find_if(Range&: OpcRegInsts, P: IsMatch(MemInst, /*IsBroadcast=*/false, Variant));
690 if (Match != OpcRegInsts.end()) {
691 updateTables(RegInst: FixUp(*Match), MemInst);
692 OpcRegInsts.erase(position: Match);
693 }
694
695 // Broadcast tables
696 StringRef MemInstName = MemInst->getName();
697 if (!MemInstName.contains(Other: "mb") && !MemInstName.contains(Other: "mib"))
698 continue;
699 RegInstsIt = RegInstsForBroadcast.find(x: Opc);
700 assert(RegInstsIt != RegInstsForBroadcast.end() &&
701 "Unexpected control flow");
702 std::vector<const CodeGenInstruction *> &OpcRegInstsForBroadcast =
703 RegInstsIt->second;
704 Match = find_if(Range&: OpcRegInstsForBroadcast,
705 P: IsMatch(MemInst, /*IsBroadcast=*/true, Variant));
706 if (Match != OpcRegInstsForBroadcast.end()) {
707 updateTables(RegInst: FixUp(*Match), MemInst, S: 0, /*IsManual=*/false,
708 /*IsBroadcast=*/true);
709 OpcRegInstsForBroadcast.erase(position: Match);
710 }
711 }
712
713 // Add the manually mapped instructions listed above.
714 for (const ManualMapEntry &Entry : ManualMapSet) {
715 const Record *RegInstIter = Records.getDef(Name: Entry.RegInstStr);
716 const Record *MemInstIter = Records.getDef(Name: Entry.MemInstStr);
717
718 updateTables(RegInst: &(Target.getInstruction(InstRec: RegInstIter)),
719 MemInst: &(Target.getInstruction(InstRec: MemInstIter)), S: Entry.Strategy, IsManual: true);
720 }
721
722#ifndef NDEBUG
723 auto CheckMemFoldTable = [](const FoldTable &Table) -> void {
724 for (const auto &Record : Table) {
725 auto &FoldEntry = Record.second;
726 FoldEntry.checkCorrectness();
727 }
728 };
729 CheckMemFoldTable(Table2Addr);
730 CheckMemFoldTable(Table0);
731 CheckMemFoldTable(Table1);
732 CheckMemFoldTable(Table2);
733 CheckMemFoldTable(Table3);
734 CheckMemFoldTable(Table4);
735 CheckMemFoldTable(BroadcastTable1);
736 CheckMemFoldTable(BroadcastTable2);
737 CheckMemFoldTable(BroadcastTable3);
738 CheckMemFoldTable(BroadcastTable4);
739#endif
740#define PRINT_TABLE(TABLE) printTable(TABLE, #TABLE, OS);
741 // Print all tables.
742 PRINT_TABLE(Table2Addr)
743 PRINT_TABLE(Table0)
744 PRINT_TABLE(Table1)
745 PRINT_TABLE(Table2)
746 PRINT_TABLE(Table3)
747 PRINT_TABLE(Table4)
748 PRINT_TABLE(BroadcastTable1)
749 PRINT_TABLE(BroadcastTable2)
750 PRINT_TABLE(BroadcastTable3)
751 PRINT_TABLE(BroadcastTable4)
752}
753
754static TableGen::Emitter::OptClass<X86FoldTablesEmitter>
755 X("gen-x86-fold-tables", "Generate X86 fold tables");
756