1//===- Target/X86/X86LowerAMXType.cpp - -------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Pass to transform <256 x i32> load/store
10/// <256 x i32> is bitcasted to x86_amx on X86, and AMX instruction set only
11/// provides simple operation on x86_amx. The basic elementwise operation
12/// is not supported by AMX. Since x86_amx is bitcasted from vector <256 x i32>
13/// and only AMX intrinsics can operate on the type, we need transform
14/// load/store <256 x i32> instruction to AMX load/store. If the bitcast can
15/// not be combined with load/store, we transform the bitcast to amx load/store
16/// and <256 x i32> store/load.
17///
18/// If Front End not use O0 but the Mid/Back end use O0, (e.g. "Clang -O2 -S
19/// -emit-llvm t.c" + "llc t.ll") we should make sure the amx data is volatile,
20/// because that is necessary for AMX fast register allocation. (In Fast
21/// registera allocation, register will be allocated before spill/reload, so
22/// there is no additional register for amx to identify the step in spill.)
23/// The volatileTileData() will handle this case.
24/// e.g.
25/// ----------------------------------------------------------
26/// | def %td = ... |
27/// | ... |
28/// | "use %td" |
29/// ----------------------------------------------------------
30/// will transfer to -->
31/// ----------------------------------------------------------
32/// | def %td = ... |
33/// | call void @llvm.x86.tilestored64.internal(mem, %td) |
34/// | ... |
35/// | %td2 = call x86_amx @llvm.x86.tileloadd64.internal(mem)|
36/// | "use %td2" |
37/// ----------------------------------------------------------
38//
39//===----------------------------------------------------------------------===//
40//
41#include "X86.h"
42#include "llvm/ADT/PostOrderIterator.h"
43#include "llvm/ADT/SetVector.h"
44#include "llvm/Analysis/TargetLibraryInfo.h"
45#include "llvm/Analysis/TargetTransformInfo.h"
46#include "llvm/CodeGen/Passes.h"
47#include "llvm/CodeGen/TargetPassConfig.h"
48#include "llvm/CodeGen/ValueTypes.h"
49#include "llvm/IR/Analysis.h"
50#include "llvm/IR/DataLayout.h"
51#include "llvm/IR/Function.h"
52#include "llvm/IR/IRBuilder.h"
53#include "llvm/IR/Instructions.h"
54#include "llvm/IR/IntrinsicInst.h"
55#include "llvm/IR/IntrinsicsX86.h"
56#include "llvm/IR/PassManager.h"
57#include "llvm/IR/PatternMatch.h"
58#include "llvm/InitializePasses.h"
59#include "llvm/Pass.h"
60#include "llvm/Target/TargetMachine.h"
61#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
62#include "llvm/Transforms/Utils/Local.h"
63
64#include <map>
65
66using namespace llvm;
67using namespace PatternMatch;
68
69#define DEBUG_TYPE "x86-lower-amx-type"
70
71static bool isAMXCast(Instruction *II) {
72 return match(V: II,
73 P: m_Intrinsic<Intrinsic::x86_cast_vector_to_tile>(Ops: m_Value())) ||
74 match(V: II, P: m_Intrinsic<Intrinsic::x86_cast_tile_to_vector>(Ops: m_Value()));
75}
76
77static bool isAMXIntrinsic(Value *I) {
78 auto *II = dyn_cast<IntrinsicInst>(Val: I);
79 if (!II)
80 return false;
81 if (isAMXCast(II))
82 return false;
83 // Check if return type or parameter is x86_amx. If it is x86_amx
84 // the intrinsic must be x86 amx intrinsics.
85 if (II->getType()->isX86_AMXTy())
86 return true;
87 for (Value *V : II->args()) {
88 if (V->getType()->isX86_AMXTy())
89 return true;
90 }
91
92 return false;
93}
94
95static bool containsAMXCode(Function &F) {
96 for (BasicBlock &BB : F)
97 for (Instruction &I : BB)
98 if (I.getType()->isX86_AMXTy())
99 return true;
100 return false;
101}
102
103static AllocaInst *createAllocaInstAtEntry(IRBuilder<> &Builder, BasicBlock *BB,
104 Type *Ty) {
105 Function &F = *BB->getParent();
106 const DataLayout &DL = F.getDataLayout();
107
108 LLVMContext &Ctx = Builder.getContext();
109 auto AllocaAlignment = DL.getPrefTypeAlign(Ty: Type::getX86_AMXTy(C&: Ctx));
110 unsigned AllocaAS = DL.getAllocaAddrSpace();
111 AllocaInst *AllocaRes =
112 new AllocaInst(Ty, AllocaAS, "", F.getEntryBlock().begin());
113 AllocaRes->setAlignment(AllocaAlignment);
114 return AllocaRes;
115}
116
117static Instruction *getFirstNonAllocaInTheEntryBlock(Function &F) {
118 for (Instruction &I : F.getEntryBlock())
119 if (!isa<AllocaInst>(Val: &I))
120 return &I;
121 llvm_unreachable("No terminator in the entry block!");
122}
123
124static Value *getRowFromCol(Instruction *II, Value *V, unsigned Granularity) {
125 IRBuilder<> Builder(II);
126 Value *RealRow = nullptr;
127 if (isa<ConstantInt>(Val: V))
128 RealRow =
129 Builder.getInt16(C: (cast<ConstantInt>(Val: V)->getSExtValue()) / Granularity);
130 else if (isa<Instruction>(Val: V)) {
131 // When it is not a const value and it is not a function argument, we
132 // create Row after the definition of V instead of
133 // before II. For example, II is %118, we try to getshape for %117:
134 // %117 = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x
135 // i32> %115).
136 // %118 = call x86_amx @llvm.x86.tdpbf16ps.internal(i16
137 // %104, i16 %105, i16 %106, x86_amx %110, x86_amx %114, x86_amx
138 // %117).
139 // If we create %row = udiv i16 %106, 4 before %118(aka. II), then its
140 // definition is after its user(new tileload for %117).
141 // So, the best choice is to create %row right after the definition of
142 // %106.
143 Builder.SetInsertPoint(cast<Instruction>(Val: V));
144 RealRow = Builder.CreateUDiv(LHS: V, RHS: Builder.getInt16(C: 4));
145 cast<Instruction>(Val: RealRow)->moveAfter(MovePos: cast<Instruction>(Val: V));
146 } else {
147 // When it is not a const value and it is a function argument, we create
148 // Row at the entry bb.
149 IRBuilder<> NewBuilder(
150 getFirstNonAllocaInTheEntryBlock(F&: *II->getFunction()));
151 RealRow = NewBuilder.CreateUDiv(LHS: V, RHS: NewBuilder.getInt16(C: Granularity));
152 }
153 return RealRow;
154}
155
156// TODO: Refine the row and col-in-bytes of tile to row and col of matrix.
157std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
158 Value *Row = nullptr, *Col = nullptr;
159 switch (II->getIntrinsicID()) {
160 default:
161 llvm_unreachable("Expect amx intrinsics");
162 case Intrinsic::x86_tileloadd64_internal:
163 case Intrinsic::x86_tileloaddt164_internal:
164 case Intrinsic::x86_tilestored64_internal:
165 case Intrinsic::x86_tileloaddrs64_internal:
166 case Intrinsic::x86_tileloaddrst164_internal: {
167 Row = II->getArgOperand(i: 0);
168 Col = II->getArgOperand(i: 1);
169 break;
170 }
171 // a * b + c
172 // The shape depends on which operand.
173 case Intrinsic::x86_tcmmimfp16ps_internal:
174 case Intrinsic::x86_tcmmrlfp16ps_internal:
175 case Intrinsic::x86_tdpbssd_internal:
176 case Intrinsic::x86_tdpbsud_internal:
177 case Intrinsic::x86_tdpbusd_internal:
178 case Intrinsic::x86_tdpbuud_internal:
179 case Intrinsic::x86_tdpbf16ps_internal:
180 case Intrinsic::x86_tdpfp16ps_internal:
181 case Intrinsic::x86_tdpbf8ps_internal:
182 case Intrinsic::x86_tdpbhf8ps_internal:
183 case Intrinsic::x86_tdphbf8ps_internal:
184 case Intrinsic::x86_tdphf8ps_internal: {
185 switch (OpNo) {
186 case 3:
187 Row = II->getArgOperand(i: 0);
188 Col = II->getArgOperand(i: 1);
189 break;
190 case 4:
191 Row = II->getArgOperand(i: 0);
192 Col = II->getArgOperand(i: 2);
193 break;
194 case 5:
195 Row = getRowFromCol(II, V: II->getArgOperand(i: 2), Granularity: 4);
196 Col = II->getArgOperand(i: 1);
197 break;
198 }
199 break;
200 }
201 case Intrinsic::x86_tcvtrowd2ps_internal:
202 case Intrinsic::x86_tcvtrowps2bf16h_internal:
203 case Intrinsic::x86_tcvtrowps2bf16l_internal:
204 case Intrinsic::x86_tcvtrowps2phh_internal:
205 case Intrinsic::x86_tcvtrowps2phl_internal:
206 case Intrinsic::x86_tilemovrow_internal: {
207 assert(OpNo == 2 && "Illegal Operand Number.");
208 Row = II->getArgOperand(i: 0);
209 Col = II->getArgOperand(i: 1);
210 break;
211 }
212 }
213
214 return std::make_pair(x&: Row, y&: Col);
215}
216
217static std::pair<Value *, Value *> getShape(PHINode *Phi) {
218 Use &U = *(Phi->use_begin());
219 unsigned OpNo = U.getOperandNo();
220 User *V = U.getUser();
221 // TODO We don't traverse all users. To make the algorithm simple, here we
222 // just traverse the first user. If we can find shape, then return the shape,
223 // otherwise just return nullptr and the optimization for undef/zero will be
224 // abandoned.
225 while (V) {
226 if (isAMXCast(II: dyn_cast<Instruction>(Val: V))) {
227 if (V->use_empty())
228 break;
229 Use &U = *(V->use_begin());
230 OpNo = U.getOperandNo();
231 V = U.getUser();
232 } else if (isAMXIntrinsic(I: V)) {
233 return getShape(II: cast<IntrinsicInst>(Val: V), OpNo);
234 } else if (isa<PHINode>(Val: V)) {
235 if (V->use_empty())
236 break;
237 Use &U = *(V->use_begin());
238 V = U.getUser();
239 } else {
240 break;
241 }
242 }
243
244 return std::make_pair(x: nullptr, y: nullptr);
245}
246
247namespace {
248class X86LowerAMXType {
249 Function &Func;
250
251 // In AMX intrinsics we let Shape = {Row, Col}, but the
252 // RealCol = Col / ElementSize. We may use the RealCol
253 // as a new Row for other new created AMX intrinsics.
254 std::map<Value *, Value *> Col2Row;
255
256public:
257 X86LowerAMXType(Function &F) : Func(F) {}
258 bool visit();
259 void combineLoadBitcast(LoadInst *LD, BitCastInst *Bitcast);
260 void combineBitcastStore(BitCastInst *Bitcast, StoreInst *ST);
261 bool transformBitcast(BitCastInst *Bitcast);
262};
263
264// %src = load <256 x i32>, <256 x i32>* %addr, align 64
265// %2 = bitcast <256 x i32> %src to x86_amx
266// -->
267// %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
268// i8* %addr, i64 %stride64)
269void X86LowerAMXType::combineLoadBitcast(LoadInst *LD, BitCastInst *Bitcast) {
270 Value *Row = nullptr, *Col = nullptr;
271 Use &U = *(Bitcast->use_begin());
272 unsigned OpNo = U.getOperandNo();
273 auto *II = cast<IntrinsicInst>(Val: U.getUser());
274 std::tie(args&: Row, args&: Col) = getShape(II, OpNo);
275 IRBuilder<> Builder(Bitcast);
276 // Use the maximun column as stride.
277 Value *Stride = Builder.getInt64(C: 64);
278 Value *I8Ptr = LD->getOperand(i_nocapture: 0);
279 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
280
281 Value *NewInst =
282 Builder.CreateIntrinsic(ID: Intrinsic::x86_tileloadd64_internal, Args);
283 Bitcast->replaceAllUsesWith(V: NewInst);
284}
285
286// %src = call x86_amx @llvm.x86.tileloadd64.internal(%row, %col, %addr,
287// %stride);
288// %13 = bitcast x86_amx %src to <256 x i32>
289// store <256 x i32> %13, <256 x i32>* %addr, align 64
290// -->
291// call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
292// %stride64, %13)
293void X86LowerAMXType::combineBitcastStore(BitCastInst *Bitcast, StoreInst *ST) {
294
295 Value *Tile = Bitcast->getOperand(i_nocapture: 0);
296 auto *II = cast<IntrinsicInst>(Val: Tile);
297 // Tile is output from AMX intrinsic. The first operand of the
298 // intrinsic is row, the second operand of the intrinsic is column.
299 Value *Row = II->getOperand(i_nocapture: 0);
300 Value *Col = II->getOperand(i_nocapture: 1);
301 IRBuilder<> Builder(ST);
302 // Use the maximum column as stride. It must be the same with load
303 // stride.
304 Value *Stride = Builder.getInt64(C: 64);
305 Value *I8Ptr = ST->getOperand(i_nocapture: 1);
306 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Tile};
307 Builder.CreateIntrinsic(ID: Intrinsic::x86_tilestored64_internal, Args);
308 if (Bitcast->hasOneUse())
309 return;
310 // %13 = bitcast x86_amx %src to <256 x i32>
311 // store <256 x i32> %13, <256 x i32>* %addr, align 64
312 // %add = <256 x i32> %13, <256 x i32> %src2
313 // -->
314 // %13 = bitcast x86_amx %src to <256 x i32>
315 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
316 // %stride64, %13)
317 // %14 = load <256 x i32>, %addr
318 // %add = <256 x i32> %14, <256 x i32> %src2
319 Value *Vec = Builder.CreateLoad(Ty: Bitcast->getType(), Ptr: ST->getOperand(i_nocapture: 1));
320 Bitcast->replaceAllUsesWith(V: Vec);
321}
322
323// transform bitcast to <store, load> instructions.
324bool X86LowerAMXType::transformBitcast(BitCastInst *Bitcast) {
325 IRBuilder<> Builder(Bitcast);
326 AllocaInst *AllocaAddr;
327 Value *I8Ptr, *Stride;
328 auto *Src = Bitcast->getOperand(i_nocapture: 0);
329
330 auto Prepare = [&](Type *MemTy) {
331 AllocaAddr = createAllocaInstAtEntry(Builder, BB: Bitcast->getParent(), Ty: MemTy);
332 I8Ptr = AllocaAddr;
333 Stride = Builder.getInt64(C: 64);
334 };
335
336 if (Bitcast->getType()->isX86_AMXTy()) {
337 // %2 = bitcast <256 x i32> %src to x86_amx
338 // -->
339 // %addr = alloca <256 x i32>, align 64
340 // store <256 x i32> %src, <256 x i32>* %addr, align 64
341 // %addr2 = bitcast <256 x i32>* to i8*
342 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
343 // i8* %addr2,
344 // i64 64)
345 Use &U = *(Bitcast->use_begin());
346 unsigned OpNo = U.getOperandNo();
347 auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser());
348 if (!II)
349 return false; // May be bitcast from x86amx to <256 x i32>.
350 Prepare(Bitcast->getOperand(i_nocapture: 0)->getType());
351 Builder.CreateStore(Val: Src, Ptr: AllocaAddr);
352 // TODO we can pick an constant operand for the shape.
353 Value *Row = nullptr, *Col = nullptr;
354 std::tie(args&: Row, args&: Col) = getShape(II, OpNo);
355 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
356 Value *NewInst =
357 Builder.CreateIntrinsic(ID: Intrinsic::x86_tileloadd64_internal, Args);
358 Bitcast->replaceAllUsesWith(V: NewInst);
359 } else {
360 // %2 = bitcast x86_amx %src to <256 x i32>
361 // -->
362 // %addr = alloca <256 x i32>, align 64
363 // %addr2 = bitcast <256 x i32>* to i8*
364 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col,
365 // i8* %addr2, i64 %stride)
366 // %2 = load <256 x i32>, <256 x i32>* %addr, align 64
367 auto *II = dyn_cast<IntrinsicInst>(Val: Src);
368 if (!II)
369 return false; // May be bitcast from <256 x i32> to x86amx.
370 Prepare(Bitcast->getType());
371 Value *Row = II->getOperand(i_nocapture: 0);
372 Value *Col = II->getOperand(i_nocapture: 1);
373 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Src};
374 Builder.CreateIntrinsic(ID: Intrinsic::x86_tilestored64_internal, Args);
375 Value *NewInst = Builder.CreateLoad(Ty: Bitcast->getType(), Ptr: AllocaAddr);
376 Bitcast->replaceAllUsesWith(V: NewInst);
377 }
378
379 return true;
380}
381
382bool X86LowerAMXType::visit() {
383 SmallVector<Instruction *, 8> DeadInsts;
384 Col2Row.clear();
385
386 for (BasicBlock *BB : post_order(G: &Func)) {
387 for (Instruction &Inst : llvm::make_early_inc_range(Range: llvm::reverse(C&: *BB))) {
388 auto *Bitcast = dyn_cast<BitCastInst>(Val: &Inst);
389 if (!Bitcast)
390 continue;
391
392 Value *Src = Bitcast->getOperand(i_nocapture: 0);
393 if (Bitcast->getType()->isX86_AMXTy()) {
394 if (Bitcast->user_empty()) {
395 DeadInsts.push_back(Elt: Bitcast);
396 continue;
397 }
398 LoadInst *LD = dyn_cast<LoadInst>(Val: Src);
399 if (!LD) {
400 if (transformBitcast(Bitcast))
401 DeadInsts.push_back(Elt: Bitcast);
402 continue;
403 }
404 // If load has multi-user, duplicate a vector load.
405 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
406 // %2 = bitcast <256 x i32> %src to x86_amx
407 // %add = add <256 x i32> %src, <256 x i32> %src2
408 // -->
409 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
410 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
411 // i8* %addr, i64 %stride64)
412 // %add = add <256 x i32> %src, <256 x i32> %src2
413
414 // If load has one user, the load will be eliminated in DAG ISel.
415 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
416 // %2 = bitcast <256 x i32> %src to x86_amx
417 // -->
418 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
419 // i8* %addr, i64 %stride64)
420 combineLoadBitcast(LD, Bitcast);
421 DeadInsts.push_back(Elt: Bitcast);
422 if (LD->hasOneUse())
423 DeadInsts.push_back(Elt: LD);
424 } else if (Src->getType()->isX86_AMXTy()) {
425 if (Bitcast->user_empty()) {
426 DeadInsts.push_back(Elt: Bitcast);
427 continue;
428 }
429 StoreInst *ST = nullptr;
430 for (Use &U : Bitcast->uses()) {
431 ST = dyn_cast<StoreInst>(Val: U.getUser());
432 if (ST)
433 break;
434 }
435 if (!ST) {
436 if (transformBitcast(Bitcast))
437 DeadInsts.push_back(Elt: Bitcast);
438 continue;
439 }
440 // If bitcast (%13) has one use, combine bitcast and store to amx store.
441 // %src = call x86_amx @llvm.x86.tileloadd64.internal(%row, %col, %addr,
442 // %stride);
443 // %13 = bitcast x86_amx %src to <256 x i32>
444 // store <256 x i32> %13, <256 x i32>* %addr, align 64
445 // -->
446 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
447 // %stride64, %13)
448 //
449 // If bitcast (%13) has multi-use, transform as below.
450 // %13 = bitcast x86_amx %src to <256 x i32>
451 // store <256 x i32> %13, <256 x i32>* %addr, align 64
452 // %add = <256 x i32> %13, <256 x i32> %src2
453 // -->
454 // %13 = bitcast x86_amx %src to <256 x i32>
455 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
456 // %stride64, %13)
457 // %14 = load <256 x i32>, %addr
458 // %add = <256 x i32> %14, <256 x i32> %src2
459 //
460 combineBitcastStore(Bitcast, ST);
461 // Delete user first.
462 DeadInsts.push_back(Elt: ST);
463 DeadInsts.push_back(Elt: Bitcast);
464 }
465 }
466 }
467
468 bool C = !DeadInsts.empty();
469
470 for (auto *Inst : DeadInsts)
471 Inst->eraseFromParent();
472
473 return C;
474}
475} // anonymous namespace
476
477static Value *getAllocaPos(BasicBlock *BB) {
478 Function *F = BB->getParent();
479 IRBuilder<> Builder(&F->getEntryBlock().front());
480 const DataLayout &DL = F->getDataLayout();
481 unsigned AllocaAS = DL.getAllocaAddrSpace();
482 Type *V256I32Ty = VectorType::get(ElementType: Builder.getInt32Ty(), NumElements: 256, Scalable: false);
483 AllocaInst *AllocaRes =
484 new AllocaInst(V256I32Ty, AllocaAS, "", F->getEntryBlock().begin());
485 BasicBlock::iterator Iter = AllocaRes->getIterator();
486 ++Iter;
487 Builder.SetInsertPoint(&*Iter);
488 Value *I8Ptr = Builder.CreateBitCast(V: AllocaRes, DestTy: Builder.getPtrTy());
489 return I8Ptr;
490}
491
492static Instruction *createTileStore(Instruction *TileDef, Value *Ptr) {
493 assert(TileDef->getType()->isX86_AMXTy() && "Not define tile!");
494 auto *II = cast<IntrinsicInst>(Val: TileDef);
495
496 assert(II && "Not tile intrinsic!");
497 Value *Row = II->getOperand(i_nocapture: 0);
498 Value *Col = II->getOperand(i_nocapture: 1);
499
500 BasicBlock::iterator Iter = TileDef->getIterator();
501 IRBuilder<> Builder(++Iter);
502 Value *Stride = Builder.getInt64(C: 64);
503 std::array<Value *, 5> Args = {Row, Col, Ptr, Stride, TileDef};
504
505 Instruction *TileStore = Builder.CreateIntrinsicWithoutFolding(
506 ID: Intrinsic::x86_tilestored64_internal, Args);
507 return TileStore;
508}
509
510static void replaceWithTileLoad(Use &U, Value *Ptr, bool IsPHI = false) {
511 Value *V = U.get();
512 assert(V->getType()->isX86_AMXTy() && "Not define tile!");
513
514 // Get tile shape.
515 IntrinsicInst *II = nullptr;
516 if (IsPHI) {
517 Value *PhiOp = cast<PHINode>(Val: V)->getIncomingValue(i: 0);
518 II = cast<IntrinsicInst>(Val: PhiOp);
519 } else {
520 II = cast<IntrinsicInst>(Val: V);
521 }
522 Value *Row = II->getOperand(i_nocapture: 0);
523 Value *Col = II->getOperand(i_nocapture: 1);
524
525 Instruction *UserI = cast<Instruction>(Val: U.getUser());
526 IRBuilder<> Builder(UserI);
527 Value *Stride = Builder.getInt64(C: 64);
528 std::array<Value *, 4> Args = {Row, Col, Ptr, Stride};
529
530 Value *TileLoad =
531 Builder.CreateIntrinsic(ID: Intrinsic::x86_tileloadd64_internal, Args);
532 UserI->replaceUsesOfWith(From: V, To: TileLoad);
533}
534
535static bool isIncomingOfPHI(Instruction *I) {
536 for (Use &U : I->uses()) {
537 User *V = U.getUser();
538 if (isa<PHINode>(Val: V))
539 return true;
540 }
541 return false;
542}
543
544// Let all AMX tile data become volatile data, shorten the life range
545// of each tile register before fast register allocation.
546namespace {
547class X86VolatileTileData {
548 Function &F;
549
550public:
551 X86VolatileTileData(Function &Func) : F(Func) {}
552 Value *updatePhiIncomings(BasicBlock *BB,
553 SmallVector<Instruction *, 2> &Incomings);
554 void replacePhiDefWithLoad(Instruction *PHI, Value *StorePtr);
555 bool volatileTileData();
556 void volatileTilePHI(PHINode *PHI);
557 void volatileTileNonPHI(Instruction *I);
558};
559
560Value *X86VolatileTileData::updatePhiIncomings(
561 BasicBlock *BB, SmallVector<Instruction *, 2> &Incomings) {
562 Value *I8Ptr = getAllocaPos(BB);
563
564 for (auto *I : Incomings) {
565 User *Store = createTileStore(TileDef: I, Ptr: I8Ptr);
566
567 // All its uses (except phi) should load from stored mem.
568 for (Use &U : I->uses()) {
569 User *V = U.getUser();
570 if (isa<PHINode>(Val: V) || V == Store)
571 continue;
572 replaceWithTileLoad(U, Ptr: I8Ptr);
573 }
574 }
575 return I8Ptr;
576}
577
578void X86VolatileTileData::replacePhiDefWithLoad(Instruction *PHI,
579 Value *StorePtr) {
580 for (Use &U : PHI->uses())
581 replaceWithTileLoad(U, Ptr: StorePtr, IsPHI: true);
582 PHI->eraseFromParent();
583}
584
585// Smilar with volatileTileNonPHI, this function only handle PHI Nodes
586// and their related AMX intrinsics.
587// 1) PHI Def should change to tileload.
588// 2) PHI Incoming Values should tilestored in just after their def.
589// 3) The mem of these tileload and tilestores should be same.
590// e.g.
591// ------------------------------------------------------
592// bb_dom:
593// ...
594// br i1 %bool.cond, label %if.else, label %if.then
595//
596// if.then:
597// def %t0 = ...
598// ...
599// use %t0
600// ...
601// br label %if.end
602//
603// if.else:
604// def %t1 = ...
605// br label %if.end
606//
607// if.end:
608// %td = phi x86_amx [ %t1, %if.else ], [ %t0, %if.then ]
609// ...
610// use %td
611// ------------------------------------------------------
612// -->
613// ------------------------------------------------------
614// bb_entry:
615// %mem = alloca <256 x i32>, align 1024 *
616// ...
617// bb_dom:
618// ...
619// br i1 %bool.cond, label %if.else, label %if.then
620//
621// if.then:
622// def %t0 = ...
623// call void @llvm.x86.tilestored64.internal(mem, %t0) *
624// ...
625// %t0` = call x86_amx @llvm.x86.tileloadd64.internal(mem)*
626// use %t0` *
627// ...
628// br label %if.end
629//
630// if.else:
631// def %t1 = ...
632// call void @llvm.x86.tilestored64.internal(mem, %t1) *
633// br label %if.end
634//
635// if.end:
636// ...
637// %td = call x86_amx @llvm.x86.tileloadd64.internal(mem) *
638// use %td
639// ------------------------------------------------------
640void X86VolatileTileData::volatileTilePHI(PHINode *PHI) {
641 BasicBlock *BB = PHI->getParent();
642 SmallVector<Instruction *, 2> Incomings;
643
644 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) {
645 Value *Op = PHI->getIncomingValue(i: I);
646 Instruction *Inst = dyn_cast<Instruction>(Val: Op);
647 assert(Inst && "We shouldn't fold AMX instruction!");
648 Incomings.push_back(Elt: Inst);
649 }
650
651 Value *StorePtr = updatePhiIncomings(BB, Incomings);
652 replacePhiDefWithLoad(PHI, StorePtr);
653}
654
655// Store the defined tile and load it before use.
656// All its users are not PHI.
657// e.g.
658// ------------------------------------------------------
659// def %td = ...
660// ...
661// "use %td"
662// ------------------------------------------------------
663// -->
664// ------------------------------------------------------
665// def %td = ...
666// call void @llvm.x86.tilestored64.internal(mem, %td)
667// ...
668// %td2 = call x86_amx @llvm.x86.tileloadd64.internal(mem)
669// "use %td2"
670// ------------------------------------------------------
671void X86VolatileTileData::volatileTileNonPHI(Instruction *I) {
672 BasicBlock *BB = I->getParent();
673 Value *I8Ptr = getAllocaPos(BB);
674 User *Store = createTileStore(TileDef: I, Ptr: I8Ptr);
675
676 // All its uses should load from stored mem.
677 for (Use &U : I->uses()) {
678 User *V = U.getUser();
679 assert(!isa<PHINode>(V) && "PHI Nodes should be excluded!");
680 if (V != Store)
681 replaceWithTileLoad(U, Ptr: I8Ptr);
682 }
683}
684
685// Volatile Tile Model:
686// 1) All the uses of tile data comes from tileload in time.
687// 2) All the defs of tile data tilestore into mem immediately.
688// For example:
689// --------------------------------------------------------------------------
690// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(m, k, ...) key
691// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(k, n, ...)
692// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(m, n, ...) amx
693// %td = tail call x86_amx @llvm.x86.tdpbssd.internal(m, n, k, t1, t2, t3)
694// call void @llvm.x86.tilestored64.internal(... td) area
695// --------------------------------------------------------------------------
696// 3) No terminator, call or other amx instructions in the key amx area.
697bool X86VolatileTileData::volatileTileData() {
698 bool Changed = false;
699 for (BasicBlock &BB : F) {
700 SmallVector<Instruction *, 2> PHIInsts;
701 SmallVector<Instruction *, 8> AMXDefInsts;
702
703 for (Instruction &I : BB) {
704 if (!I.getType()->isX86_AMXTy())
705 continue;
706 if (isa<PHINode>(Val: &I))
707 PHIInsts.push_back(Elt: &I);
708 else
709 AMXDefInsts.push_back(Elt: &I);
710 }
711
712 // First we "volatile" the non-phi related amx intrinsics.
713 for (Instruction *I : AMXDefInsts) {
714 if (isIncomingOfPHI(I))
715 continue;
716 volatileTileNonPHI(I);
717 Changed = true;
718 }
719
720 for (Instruction *I : PHIInsts) {
721 volatileTilePHI(PHI: dyn_cast<PHINode>(Val: I));
722 Changed = true;
723 }
724 }
725 return Changed;
726}
727
728} // anonymous namespace
729
730namespace {
731
732class X86LowerAMXCast {
733 Function &Func;
734 std::unique_ptr<DominatorTree> DT;
735
736public:
737 X86LowerAMXCast(Function &F) : Func(F), DT(nullptr) {}
738 bool combineCastStore(IntrinsicInst *Cast, StoreInst *ST);
739 bool combineLoadCast(IntrinsicInst *Cast, LoadInst *LD);
740 bool combineTilezero(IntrinsicInst *Cast);
741 bool combineLdSt(SmallVectorImpl<Instruction *> &Casts);
742 bool combineAMXcast(TargetLibraryInfo *TLI);
743 bool transformAMXCast(IntrinsicInst *AMXCast);
744 bool transformAllAMXCast();
745 bool optimizeAMXCastFromPhi(IntrinsicInst *CI, PHINode *PN,
746 SmallSetVector<Instruction *, 16> &DeadInst);
747};
748
749static bool DCEInstruction(Instruction *I,
750 SmallSetVector<Instruction *, 16> &WorkList,
751 const TargetLibraryInfo *TLI) {
752 if (isInstructionTriviallyDead(I, TLI)) {
753 salvageDebugInfo(I&: *I);
754 salvageKnowledge(I);
755
756 // Null out all of the instruction's operands to see if any operand becomes
757 // dead as we go.
758 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
759 Value *OpV = I->getOperand(i);
760 I->setOperand(i, Val: nullptr);
761
762 if (!OpV->use_empty() || I == OpV)
763 continue;
764
765 // If the operand is an instruction that became dead as we nulled out the
766 // operand, and if it is 'trivially' dead, delete it in a future loop
767 // iteration.
768 if (Instruction *OpI = dyn_cast<Instruction>(Val: OpV)) {
769 if (isInstructionTriviallyDead(I: OpI, TLI)) {
770 WorkList.insert(X: OpI);
771 }
772 }
773 }
774 I->eraseFromParent();
775 return true;
776 }
777 return false;
778}
779
780/// This function handles following case
781///
782/// A -> B amxcast
783/// PHI
784/// B -> A amxcast
785///
786/// All the related PHI nodes can be replaced by new PHI nodes with type A.
787/// The uses of \p CI can be changed to the new PHI node corresponding to \p PN.
788bool X86LowerAMXCast::optimizeAMXCastFromPhi(
789 IntrinsicInst *CI, PHINode *PN,
790 SmallSetVector<Instruction *, 16> &DeadInst) {
791 IRBuilder<> Builder(CI);
792 Value *Src = CI->getOperand(i_nocapture: 0);
793 Type *SrcTy = Src->getType(); // Type B
794 Type *DestTy = CI->getType(); // Type A
795
796 SmallVector<PHINode *, 4> PhiWorklist;
797 SmallSetVector<PHINode *, 4> OldPhiNodes;
798
799 // Find all of the A->B casts and PHI nodes.
800 // We need to inspect all related PHI nodes, but PHIs can be cyclic, so
801 // OldPhiNodes is used to track all known PHI nodes, before adding a new
802 // PHI to PhiWorklist, it is checked against and added to OldPhiNodes first.
803 PhiWorklist.push_back(Elt: PN);
804 OldPhiNodes.insert(X: PN);
805 while (!PhiWorklist.empty()) {
806 auto *OldPN = PhiWorklist.pop_back_val();
807 for (unsigned I = 0; I < OldPN->getNumOperands(); ++I) {
808 Value *IncValue = OldPN->getIncomingValue(i: I);
809 // TODO: currently, We ignore cases where it is a const. In the future, we
810 // might support const.
811 if (isa<Constant>(Val: IncValue)) {
812 auto *IncConst = dyn_cast<Constant>(Val: IncValue);
813 if (!isa<UndefValue>(Val: IncValue) && !IncConst->isNullValue())
814 return false;
815 Value *Row = nullptr, *Col = nullptr;
816 std::tie(args&: Row, args&: Col) = getShape(Phi: OldPN);
817 // TODO: If it is not constant the Row and Col must domoniate tilezero
818 // that we are going to create.
819 if (!Row || !Col || !isa<Constant>(Val: Row) || !isa<Constant>(Val: Col))
820 return false;
821 // Create tilezero at the end of incoming block.
822 auto *Block = OldPN->getIncomingBlock(i: I);
823 BasicBlock::iterator Iter = Block->getTerminator()->getIterator();
824 Instruction *NewInst = Builder.CreateIntrinsicWithoutFolding(
825 ID: Intrinsic::x86_tilezero_internal, OverloadTypes: {}, Args: {Row, Col});
826 NewInst->moveBefore(InsertPos: Iter);
827 NewInst = Builder.CreateIntrinsicWithoutFolding(
828 ID: Intrinsic::x86_cast_tile_to_vector, OverloadTypes: {IncValue->getType()},
829 Args: {NewInst});
830 NewInst->moveBefore(InsertPos: Iter);
831 // Replace InValue with new Value.
832 OldPN->setIncomingValue(i: I, V: NewInst);
833 IncValue = NewInst;
834 }
835
836 if (auto *PNode = dyn_cast<PHINode>(Val: IncValue)) {
837 if (OldPhiNodes.insert(X: PNode))
838 PhiWorklist.push_back(Elt: PNode);
839 continue;
840 }
841 Instruction *ACI = dyn_cast<Instruction>(Val: IncValue);
842 if (ACI && isAMXCast(II: ACI)) {
843 // Verify it's a A->B cast.
844 Type *TyA = ACI->getOperand(i: 0)->getType();
845 Type *TyB = ACI->getType();
846 if (TyA != DestTy || TyB != SrcTy)
847 return false;
848 continue;
849 }
850 return false;
851 }
852 }
853
854 // Check that each user of each old PHI node is something that we can
855 // rewrite, so that all of the old PHI nodes can be cleaned up afterwards.
856 for (auto *OldPN : OldPhiNodes) {
857 for (User *V : OldPN->users()) {
858 Instruction *ACI = dyn_cast<Instruction>(Val: V);
859 if (ACI && isAMXCast(II: ACI)) {
860 // Verify it's a B->A cast.
861 Type *TyB = ACI->getOperand(i: 0)->getType();
862 Type *TyA = ACI->getType();
863 if (TyA != DestTy || TyB != SrcTy)
864 return false;
865 } else if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
866 // As long as the user is another old PHI node, then even if we don't
867 // rewrite it, the PHI web we're considering won't have any users
868 // outside itself, so it'll be dead.
869 // example:
870 // bb.0:
871 // %0 = amxcast ...
872 // bb.1:
873 // %1 = amxcast ...
874 // bb.2:
875 // %goodphi = phi %0, %1
876 // %3 = amxcast %goodphi
877 // bb.3:
878 // %goodphi2 = phi %0, %goodphi
879 // %4 = amxcast %goodphi2
880 // When optimizeAMXCastFromPhi process %3 and %goodphi, %goodphi2 is
881 // outside the phi-web, so the combination stop When
882 // optimizeAMXCastFromPhi process %4 and %goodphi2, the optimization
883 // will be done.
884 if (OldPhiNodes.count(key: PHI) == 0)
885 return false;
886 } else
887 return false;
888 }
889 }
890
891 // For each old PHI node, create a corresponding new PHI node with a type A.
892 SmallDenseMap<PHINode *, PHINode *> NewPNodes;
893 for (auto *OldPN : OldPhiNodes) {
894 Builder.SetInsertPoint(OldPN);
895 PHINode *NewPN = Builder.CreatePHI(Ty: DestTy, NumReservedValues: OldPN->getNumOperands());
896 NewPNodes[OldPN] = NewPN;
897 }
898
899 // Fill in the operands of new PHI nodes.
900 for (auto *OldPN : OldPhiNodes) {
901 PHINode *NewPN = NewPNodes[OldPN];
902 for (unsigned j = 0, e = OldPN->getNumOperands(); j != e; ++j) {
903 Value *V = OldPN->getOperand(i_nocapture: j);
904 Value *NewV = nullptr;
905 Instruction *ACI = dyn_cast<Instruction>(Val: V);
906 // There should not be a AMXcast from a const.
907 if (ACI && isAMXCast(II: ACI))
908 NewV = ACI->getOperand(i: 0);
909 else if (auto *PrevPN = dyn_cast<PHINode>(Val: V))
910 NewV = NewPNodes[PrevPN];
911 assert(NewV);
912 NewPN->addIncoming(V: NewV, BB: OldPN->getIncomingBlock(i: j));
913 }
914 }
915
916 // Traverse all accumulated PHI nodes and process its users,
917 // which are Stores and BitcCasts. Without this processing
918 // NewPHI nodes could be replicated and could lead to extra
919 // moves generated after DeSSA.
920 // If there is a store with type B, change it to type A.
921
922 // Replace users of BitCast B->A with NewPHI. These will help
923 // later to get rid of a closure formed by OldPHI nodes.
924 for (auto *OldPN : OldPhiNodes) {
925 PHINode *NewPN = NewPNodes[OldPN];
926 for (User *V : make_early_inc_range(Range: OldPN->users())) {
927 Instruction *ACI = dyn_cast<Instruction>(Val: V);
928 if (ACI && isAMXCast(II: ACI)) {
929 Type *TyB = ACI->getOperand(i: 0)->getType();
930 Type *TyA = ACI->getType();
931 assert(TyA == DestTy && TyB == SrcTy);
932 (void)TyA;
933 (void)TyB;
934 ACI->replaceAllUsesWith(V: NewPN);
935 DeadInst.insert(X: ACI);
936 } else if (auto *PHI = dyn_cast<PHINode>(Val: V)) {
937 // We don't need to push PHINode into DeadInst since they are operands
938 // of rootPN DCE can safely delete rootPN's operands if rootPN is dead.
939 assert(OldPhiNodes.contains(PHI));
940 (void)PHI;
941 } else
942 llvm_unreachable("all uses should be handled");
943 }
944 }
945 return true;
946}
947
948// %43 = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %42)
949// store <256 x i32> %43, <256 x i32>* %p, align 64
950// -->
951// call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, i8* %p,
952// i64 64, x86_amx %42)
953bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
954 Value *Tile = Cast->getOperand(i_nocapture: 0);
955
956 assert(Tile->getType()->isX86_AMXTy() && "Not Tile Operand!");
957
958 // TODO: Specially handle the multi-use case.
959 if (!Tile->hasOneUse())
960 return false;
961
962 auto *II = cast<IntrinsicInst>(Val: Tile);
963 // Tile is output from AMX intrinsic. The first operand of the
964 // intrinsic is row, the second operand of the intrinsic is column.
965 Value *Row = II->getOperand(i_nocapture: 0);
966 Value *Col = II->getOperand(i_nocapture: 1);
967
968 IRBuilder<> Builder(ST);
969
970 // Stride should be equal to col(measured by bytes)
971 Value *Stride = Builder.CreateSExt(V: Col, DestTy: Builder.getInt64Ty());
972 Value *I8Ptr = Builder.CreateBitCast(V: ST->getOperand(i_nocapture: 1), DestTy: Builder.getPtrTy());
973 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Tile};
974 Builder.CreateIntrinsic(ID: Intrinsic::x86_tilestored64_internal, Args);
975 return true;
976}
977
978// %65 = load <256 x i32>, <256 x i32>* %p, align 64
979// %66 = call x86_amx @llvm.x86.cast.vector.to.tile(<256 x i32> %65)
980// -->
981// %66 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
982// i8* %p, i64 64)
983bool X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
984 bool EraseLoad = true;
985 Value *Row = nullptr, *Col = nullptr;
986 Use &U = *(Cast->use_begin());
987 unsigned OpNo = U.getOperandNo();
988 auto *II = cast<IntrinsicInst>(Val: U.getUser());
989 // TODO: If it is cast intrinsic or phi node, we can propagate the
990 // shape information through def-use chain.
991 if (!isAMXIntrinsic(I: II))
992 return false;
993 std::tie(args&: Row, args&: Col) = getShape(II, OpNo);
994 IRBuilder<> Builder(LD);
995 Value *I8Ptr;
996
997 // To save compiling time, we create dominator tree when it is really needed.
998 if (!DT)
999 DT.reset(p: new DominatorTree(Func));
1000 if (!DT->dominates(Def: Row, User: LD) || !DT->dominates(Def: Col, User: LD)) {
1001 // store the value to stack and reload it from stack before cast.
1002 auto *AllocaAddr =
1003 createAllocaInstAtEntry(Builder, BB: Cast->getParent(), Ty: LD->getType());
1004 Builder.SetInsertPoint(&*std::next(x: LD->getIterator()));
1005 Builder.CreateStore(Val: LD, Ptr: AllocaAddr);
1006
1007 Builder.SetInsertPoint(Cast);
1008 I8Ptr = Builder.CreateBitCast(V: AllocaAddr, DestTy: Builder.getPtrTy());
1009 EraseLoad = false;
1010 } else {
1011 I8Ptr = Builder.CreateBitCast(V: LD->getOperand(i_nocapture: 0), DestTy: Builder.getPtrTy());
1012 }
1013 // Stride should be equal to col(measured by bytes)
1014 Value *Stride = Builder.CreateSExt(V: Col, DestTy: Builder.getInt64Ty());
1015 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
1016
1017 Value *NewInst =
1018 Builder.CreateIntrinsic(ID: Intrinsic::x86_tileloadd64_internal, Args);
1019 Cast->replaceAllUsesWith(V: NewInst);
1020
1021 return EraseLoad;
1022}
1023
1024// %19 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> zeroinitializer)
1025// -->
1026// %19 = tail call x86_amx @llvm.x86.tilezero.internal(i16 %row, i16 %col)
1027bool X86LowerAMXCast::combineTilezero(IntrinsicInst *Cast) {
1028 Value *Row = nullptr, *Col = nullptr;
1029 Use &U = *(Cast->use_begin());
1030 unsigned OpNo = U.getOperandNo();
1031 auto *II = cast<IntrinsicInst>(Val: U.getUser());
1032 if (!isAMXIntrinsic(I: II))
1033 return false;
1034
1035 std::tie(args&: Row, args&: Col) = getShape(II, OpNo);
1036
1037 IRBuilder<> Builder(Cast);
1038 Value *NewInst =
1039 Builder.CreateIntrinsic(ID: Intrinsic::x86_tilezero_internal, OverloadTypes: {}, Args: {Row, Col});
1040 Cast->replaceAllUsesWith(V: NewInst);
1041 return true;
1042}
1043
1044bool X86LowerAMXCast::combineLdSt(SmallVectorImpl<Instruction *> &Casts) {
1045 bool Change = false;
1046 for (auto *Cast : Casts) {
1047 auto *II = cast<IntrinsicInst>(Val: Cast);
1048 // %43 = call <256 x i32> @llvm.x86.cast.tile.to.vector(x86_amx %42)
1049 // store <256 x i32> %43, <256 x i32>* %p, align 64
1050 // -->
1051 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, i8* %p,
1052 // i64 64, x86_amx %42)
1053 if (II->getIntrinsicID() == Intrinsic::x86_cast_tile_to_vector) {
1054 SmallVector<Instruction *, 2> DeadStores;
1055 for (User *U : Cast->users()) {
1056 StoreInst *Store = dyn_cast<StoreInst>(Val: U);
1057 if (!Store)
1058 continue;
1059 if (combineCastStore(Cast: cast<IntrinsicInst>(Val: Cast), ST: Store)) {
1060 DeadStores.push_back(Elt: Store);
1061 Change = true;
1062 }
1063 }
1064 for (auto *Store : DeadStores)
1065 Store->eraseFromParent();
1066 } else { // x86_cast_vector_to_tile
1067 // %19 = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> zeroinitializer)
1068 // -->
1069 // %19 = tail call x86_amx @llvm.x86.tilezero.internal(i16 %row, i16 %col)
1070 if (isa<ConstantAggregateZero>(Val: Cast->getOperand(i: 0))) {
1071 Change |= combineTilezero(Cast: cast<IntrinsicInst>(Val: Cast));
1072 continue;
1073 }
1074
1075 auto *Load = dyn_cast<LoadInst>(Val: Cast->getOperand(i: 0));
1076 if (!Load || !Load->hasOneUse())
1077 continue;
1078 // %65 = load <256 x i32>, <256 x i32>* %p, align 64
1079 // %66 = call x86_amx @llvm.x86.cast.vector.to.tile(<256 x i32> %65)
1080 // -->
1081 // %66 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
1082 // i8* %p, i64 64)
1083 if (combineLoadCast(Cast: cast<IntrinsicInst>(Val: Cast), LD: Load)) {
1084 // Set the operand is null so that load instruction can be erased.
1085 Cast->setOperand(i: 0, Val: nullptr);
1086 Load->eraseFromParent();
1087 Change = true;
1088 }
1089 }
1090 }
1091 return Change;
1092}
1093
1094bool X86LowerAMXCast::combineAMXcast(TargetLibraryInfo *TLI) {
1095 bool Change = false;
1096 // Collect tile cast instruction.
1097 SmallVector<Instruction *, 8> Vec2TileInsts;
1098 SmallVector<Instruction *, 8> Tile2VecInsts;
1099 SmallVector<Instruction *, 8> PhiCastWorkList;
1100 SmallSetVector<Instruction *, 16> DeadInst;
1101 for (BasicBlock &BB : Func) {
1102 for (Instruction &I : BB) {
1103 Value *Vec;
1104 if (match(V: &I,
1105 P: m_Intrinsic<Intrinsic::x86_cast_vector_to_tile>(Ops: m_Value(V&: Vec))))
1106 Vec2TileInsts.push_back(Elt: &I);
1107 else if (match(V: &I, P: m_Intrinsic<Intrinsic::x86_cast_tile_to_vector>(
1108 Ops: m_Value(V&: Vec))))
1109 Tile2VecInsts.push_back(Elt: &I);
1110 }
1111 }
1112
1113 auto Convert = [&](SmallVectorImpl<Instruction *> &Insts, Intrinsic::ID IID) {
1114 for (auto *Inst : Insts) {
1115 for (User *U : Inst->users()) {
1116 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
1117 if (!II || II->getIntrinsicID() != IID)
1118 continue;
1119 // T1 = vec2tile V0
1120 // V2 = tile2vec T1
1121 // V3 = OP V2
1122 // -->
1123 // T1 = vec2tile V0
1124 // V2 = tile2vec T1
1125 // V3 = OP V0
1126 II->replaceAllUsesWith(V: Inst->getOperand(i: 0));
1127 Change = true;
1128 }
1129 }
1130 };
1131
1132 Convert(Vec2TileInsts, Intrinsic::x86_cast_tile_to_vector);
1133 Convert(Tile2VecInsts, Intrinsic::x86_cast_vector_to_tile);
1134
1135 SmallVector<Instruction *, 8> LiveCasts;
1136 auto EraseInst = [&](SmallVectorImpl<Instruction *> &Insts) {
1137 for (auto *Inst : Insts) {
1138 if (Inst->use_empty()) {
1139 Inst->eraseFromParent();
1140 Change = true;
1141 } else {
1142 LiveCasts.push_back(Elt: Inst);
1143 }
1144 }
1145 };
1146
1147 EraseInst(Vec2TileInsts);
1148 EraseInst(Tile2VecInsts);
1149 LLVM_DEBUG(dbgs() << "[LowerAMXTYpe][combineAMXcast] IR dump after combine "
1150 "Vec2Tile and Tile2Vec:\n";
1151 Func.dump());
1152 Change |= combineLdSt(Casts&: LiveCasts);
1153 EraseInst(LiveCasts);
1154 LLVM_DEBUG(dbgs() << "[LowerAMXTYpe][combineAMXcast] IR dump after combine "
1155 "AMXCast and load/store:\n";
1156 Func.dump());
1157
1158 // Handle the A->B->A cast, and there is an intervening PHI node.
1159 for (BasicBlock &BB : Func) {
1160 for (Instruction &I : BB) {
1161 if (isAMXCast(II: &I)) {
1162 if (isa<PHINode>(Val: I.getOperand(i: 0)))
1163 PhiCastWorkList.push_back(Elt: &I);
1164 }
1165 }
1166 }
1167 for (auto *I : PhiCastWorkList) {
1168 // We skip the dead Amxcast.
1169 if (DeadInst.contains(key: I))
1170 continue;
1171 PHINode *PN = cast<PHINode>(Val: I->getOperand(i: 0));
1172 if (optimizeAMXCastFromPhi(CI: cast<IntrinsicInst>(Val: I), PN, DeadInst)) {
1173 DeadInst.insert(X: PN);
1174 Change = true;
1175 }
1176 }
1177
1178 // Since we create new phi and merge AMXCast, some old phis and AMXCast might
1179 // have no uses. We do some DeadCodeElimination for them.
1180 while (!DeadInst.empty()) {
1181 Instruction *I = DeadInst.pop_back_val();
1182 Change |= DCEInstruction(I, WorkList&: DeadInst, TLI);
1183 }
1184 LLVM_DEBUG(dbgs() << "[LowerAMXTYpe][combineAMXcast] IR dump after "
1185 "optimizeAMXCastFromPhi:\n";
1186 Func.dump());
1187 return Change;
1188}
1189
1190// There might be remaining AMXcast after combineAMXcast and they should be
1191// handled elegantly.
1192bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
1193 IRBuilder<> Builder(AMXCast);
1194 AllocaInst *AllocaAddr;
1195 Value *I8Ptr, *Stride;
1196 auto *Src = AMXCast->getOperand(i_nocapture: 0);
1197
1198 auto Prepare = [&](Type *MemTy) {
1199 AllocaAddr = createAllocaInstAtEntry(Builder, BB: AMXCast->getParent(), Ty: MemTy);
1200 I8Ptr = Builder.CreateBitCast(V: AllocaAddr, DestTy: Builder.getPtrTy());
1201 Stride = Builder.getInt64(C: 64);
1202 };
1203
1204 if (AMXCast->getType()->isX86_AMXTy()) {
1205 // %2 = amxcast <225 x i32> %src to x86_amx
1206 // call void @llvm.x86.tilestored64.internal(i16 15, i16 60,
1207 // i8* %addr3, i64 60, x86_amx %2)
1208 // -->
1209 // %addr = alloca <225 x i32>, align 64
1210 // store <225 x i32> %src, <225 x i32>* %addr, align 64
1211 // %addr2 = bitcast <225 x i32>* %addr to i8*
1212 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 15, i16 60,
1213 // i8* %addr2,
1214 // i64 60)
1215 // call void @llvm.x86.tilestored64.internal(i16 15, i16 60,
1216 // i8* %addr3, i64 60, x86_amx %2)
1217 if (AMXCast->use_empty()) {
1218 AMXCast->eraseFromParent();
1219 return true;
1220 }
1221 Use &U = *(AMXCast->use_begin());
1222 unsigned OpNo = U.getOperandNo();
1223 auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser());
1224 if (!II)
1225 return false; // May be bitcast from x86amx to <256 x i32>.
1226 Prepare(AMXCast->getOperand(i_nocapture: 0)->getType());
1227 Builder.CreateStore(Val: Src, Ptr: AllocaAddr);
1228 // TODO we can pick an constant operand for the shape.
1229 Value *Row = nullptr, *Col = nullptr;
1230 std::tie(args&: Row, args&: Col) = getShape(II, OpNo);
1231 std::array<Value *, 4> Args = {
1232 Row, Col, I8Ptr, Builder.CreateSExt(V: Col, DestTy: Builder.getInt64Ty())};
1233 Value *NewInst =
1234 Builder.CreateIntrinsic(ID: Intrinsic::x86_tileloadd64_internal, Args);
1235 AMXCast->replaceAllUsesWith(V: NewInst);
1236 AMXCast->eraseFromParent();
1237 } else {
1238 // %2 = amxcast x86_amx %src to <225 x i32>
1239 // -->
1240 // %addr = alloca <225 x i32>, align 64
1241 // %addr2 = bitcast <225 x i32>* to i8*
1242 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col,
1243 // i8* %addr2, i64 %stride)
1244 // %2 = load <225 x i32>, <225 x i32>* %addr, align 64
1245 auto *II = dyn_cast<IntrinsicInst>(Val: Src);
1246 if (!II)
1247 return false; // May be bitcast from <256 x i32> to x86amx.
1248 Prepare(AMXCast->getType());
1249 Value *Row = II->getOperand(i_nocapture: 0);
1250 Value *Col = II->getOperand(i_nocapture: 1);
1251 std::array<Value *, 5> Args = {
1252 Row, Col, I8Ptr, Builder.CreateSExt(V: Col, DestTy: Builder.getInt64Ty()), Src};
1253 Builder.CreateIntrinsic(ID: Intrinsic::x86_tilestored64_internal, Args);
1254 Value *NewInst = Builder.CreateLoad(Ty: AMXCast->getType(), Ptr: AllocaAddr);
1255 AMXCast->replaceAllUsesWith(V: NewInst);
1256 AMXCast->eraseFromParent();
1257 }
1258
1259 return true;
1260}
1261
1262bool X86LowerAMXCast::transformAllAMXCast() {
1263 bool Change = false;
1264 // Collect tile cast instruction.
1265 SmallVector<Instruction *, 8> WorkLists;
1266 for (BasicBlock &BB : Func) {
1267 for (Instruction &I : BB) {
1268 if (isAMXCast(II: &I))
1269 WorkLists.push_back(Elt: &I);
1270 }
1271 }
1272
1273 for (auto *Inst : WorkLists) {
1274 Change |= transformAMXCast(AMXCast: cast<IntrinsicInst>(Val: Inst));
1275 }
1276
1277 return Change;
1278}
1279
1280bool lowerAmxType(Function &F, const TargetMachine *TM,
1281 TargetLibraryInfo *TLI) {
1282 // Performance optimization: most code doesn't use AMX, so return early if
1283 // there are no instructions that produce AMX values. This is sufficient, as
1284 // AMX arguments and constants are not allowed -- so any producer of an AMX
1285 // value must be an instruction.
1286 // TODO: find a cheaper way for this, without looking at all instructions.
1287 if (!containsAMXCode(F))
1288 return false;
1289
1290 bool C = false;
1291 X86LowerAMXCast LAC(F);
1292 C |= LAC.combineAMXcast(TLI);
1293 // There might be remaining AMXcast after combineAMXcast and they should be
1294 // handled elegantly.
1295 C |= LAC.transformAllAMXCast();
1296
1297 X86LowerAMXType LAT(F);
1298 C |= LAT.visit();
1299
1300 // Prepare for fast register allocation at O0.
1301 // Todo: May better check the volatile model of AMX code, not just
1302 // by checking Attribute::OptimizeNone and CodeGenOptLevel::None.
1303 if (TM->getOptLevel() == CodeGenOptLevel::None) {
1304 // If Front End not use O0 but the Mid/Back end use O0, (e.g.
1305 // "Clang -O2 -S -emit-llvm t.c" + "llc t.ll") we should make
1306 // sure the amx data is volatile, that is necessary for AMX fast
1307 // register allocation.
1308 if (!F.hasFnAttribute(Kind: Attribute::OptimizeNone)) {
1309 X86VolatileTileData VTD(F);
1310 C = VTD.volatileTileData() || C;
1311 }
1312 }
1313
1314 return C;
1315}
1316
1317} // anonymous namespace
1318
1319PreservedAnalyses X86LowerAMXTypePass::run(Function &F,
1320 FunctionAnalysisManager &FAM) {
1321 TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(IR&: F);
1322 bool Changed = lowerAmxType(F, TM, TLI: &TLI);
1323 if (!Changed)
1324 return PreservedAnalyses::all();
1325
1326 PreservedAnalyses PA = PreservedAnalyses::none();
1327 PA.preserveSet<CFGAnalyses>();
1328 return PA;
1329}
1330
1331namespace {
1332
1333class X86LowerAMXTypeLegacyPass : public FunctionPass {
1334public:
1335 static char ID;
1336
1337 X86LowerAMXTypeLegacyPass() : FunctionPass(ID) {}
1338
1339 bool runOnFunction(Function &F) override {
1340 TargetMachine *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
1341 TargetLibraryInfo *TLI =
1342 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
1343 return lowerAmxType(F, TM, TLI);
1344 }
1345
1346 void getAnalysisUsage(AnalysisUsage &AU) const override {
1347 AU.setPreservesCFG();
1348 AU.addRequired<TargetPassConfig>();
1349 AU.addRequired<TargetLibraryInfoWrapperPass>();
1350 }
1351};
1352
1353} // anonymous namespace
1354
1355static const char PassName[] = "Lower AMX type for load/store";
1356char X86LowerAMXTypeLegacyPass::ID = 0;
1357INITIALIZE_PASS_BEGIN(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,
1358 false)
1359INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
1360INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
1361INITIALIZE_PASS_END(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,
1362 false)
1363
1364FunctionPass *llvm::createX86LowerAMXTypeLegacyPass() {
1365 return new X86LowerAMXTypeLegacyPass();
1366}
1367