1//===-- HexagonVectorCombine.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// HexagonVectorCombine is a utility class implementing a variety of functions
9// that assist in vector-based optimizations.
10//
11// AlignVectors: replace unaligned vector loads and stores with aligned ones.
12// HvxIdioms: recognize various opportunities to generate HVX intrinsic code.
13//===----------------------------------------------------------------------===//
14
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/MapVector.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/AliasAnalysis.h"
22#include "llvm/Analysis/AssumeBundleQueries.h"
23#include "llvm/Analysis/AssumptionCache.h"
24#include "llvm/Analysis/InstSimplifyFolder.h"
25#include "llvm/Analysis/InstructionSimplify.h"
26#include "llvm/Analysis/OptimizationRemarkEmitter.h"
27#include "llvm/Analysis/ScalarEvolutionExpressions.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/CodeGen/TargetPassConfig.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/Dominators.h"
34#include "llvm/IR/IRBuilder.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsHexagon.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/PatternMatch.h"
40#include "llvm/InitializePasses.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/KnownBits.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Transforms/Utils/Local.h"
48
49#include "Hexagon.h"
50#include "HexagonSubtarget.h"
51#include "HexagonTargetMachine.h"
52
53#include <algorithm>
54#include <deque>
55#include <optional>
56#include <set>
57#include <utility>
58#include <vector>
59
60#define DEBUG_TYPE "hexagon-vc"
61
62// This is a const that represents default HVX VTCM page size.
63// It is boot time configurable, so we probably want an API to
64// read it, but for now assume 128KB
65#define DEFAULT_HVX_VTCM_PAGE_SIZE 131072
66
67using namespace llvm;
68
69namespace {
70cl::opt<bool> DumpModule("hvc-dump-module", cl::Hidden);
71cl::opt<bool> VAEnabled("hvc-va", cl::Hidden, cl::init(Val: true)); // Align
72cl::opt<bool> VIEnabled("hvc-vi", cl::Hidden, cl::init(Val: true)); // Idioms
73cl::opt<bool> VADoFullStores("hvc-va-full-stores", cl::Hidden);
74
75cl::opt<unsigned> VAGroupCountLimit("hvc-va-group-count-limit", cl::Hidden,
76 cl::init(Val: ~0));
77cl::opt<unsigned> VAGroupSizeLimit("hvc-va-group-size-limit", cl::Hidden,
78 cl::init(Val: ~0));
79cl::opt<unsigned>
80 MinLoadGroupSizeForAlignment("hvc-ld-min-group-size-for-alignment",
81 cl::Hidden, cl::init(Val: 4));
82
83class HexagonVectorCombine {
84public:
85 HexagonVectorCombine(Function &F_, AliasAnalysis &AA_, AssumptionCache &AC_,
86 DominatorTree &DT_, ScalarEvolution &SE_,
87 TargetLibraryInfo &TLI_, const TargetMachine &TM_,
88 OptimizationRemarkEmitter &ORE_)
89 : F(F_), DL(F.getDataLayout()), AA(AA_), AC(AC_), DT(DT_), SE(SE_),
90 TLI(TLI_),
91 HST(static_cast<const HexagonSubtarget &>(*TM_.getSubtargetImpl(F))),
92 ORE(ORE_) {}
93
94 bool run();
95
96 // Common integer type.
97 IntegerType *getIntTy(unsigned Width = 32) const;
98 // Byte type: either scalar (when Length = 0), or vector with given
99 // element count.
100 Type *getByteTy(int ElemCount = 0) const;
101 // Boolean type: either scalar (when Length = 0), or vector with given
102 // element count.
103 Type *getBoolTy(int ElemCount = 0) const;
104 // Create a ConstantInt of type returned by getIntTy with the value Val.
105 ConstantInt *getConstInt(int Val, unsigned Width = 32) const;
106 // Get the integer value of V, if it exists.
107 std::optional<APInt> getIntValue(const Value *Val) const;
108 // Is Val a constant 0, or a vector of 0s?
109 bool isZero(const Value *Val) const;
110 // Is Val an undef value?
111 bool isUndef(const Value *Val) const;
112 // Is Val a scalar (i1 true) or a vector of (i1 true)?
113 bool isTrue(const Value *Val) const;
114 // Is Val a scalar (i1 false) or a vector of (i1 false)?
115 bool isFalse(const Value *Val) const;
116
117 // Get HVX vector type with the given element type.
118 VectorType *getHvxTy(Type *ElemTy, bool Pair = false) const;
119
120 enum SizeKind {
121 Store, // Store size
122 Alloc, // Alloc size
123 };
124 int getSizeOf(const Value *Val, SizeKind Kind = Store) const;
125 int getSizeOf(const Type *Ty, SizeKind Kind = Store) const;
126 int getTypeAlignment(Type *Ty) const;
127 size_t length(Value *Val) const;
128 size_t length(Type *Ty) const;
129
130 Value *simplify(Value *Val) const;
131
132 Value *insertb(IRBuilderBase &Builder, Value *Dest, Value *Src, int Start,
133 int Length, int Where) const;
134 Value *vlalignb(IRBuilderBase &Builder, Value *Lo, Value *Hi,
135 Value *Amt) const;
136 Value *vralignb(IRBuilderBase &Builder, Value *Lo, Value *Hi,
137 Value *Amt) const;
138 Value *concat(IRBuilderBase &Builder, ArrayRef<Value *> Vecs) const;
139 Value *vresize(IRBuilderBase &Builder, Value *Val, int NewSize,
140 Value *Pad) const;
141 Value *rescale(IRBuilderBase &Builder, Value *Mask, Type *FromTy,
142 Type *ToTy) const;
143 Value *vlsb(IRBuilderBase &Builder, Value *Val) const;
144 Value *vbytes(IRBuilderBase &Builder, Value *Val) const;
145 Value *subvector(IRBuilderBase &Builder, Value *Val, unsigned Start,
146 unsigned Length) const;
147 Value *sublo(IRBuilderBase &Builder, Value *Val) const;
148 Value *subhi(IRBuilderBase &Builder, Value *Val) const;
149 Value *vdeal(IRBuilderBase &Builder, Value *Val0, Value *Val1) const;
150 Value *vshuff(IRBuilderBase &Builder, Value *Val0, Value *Val1) const;
151
152 Value *createHvxIntrinsic(IRBuilderBase &Builder, Intrinsic::ID IntID,
153 Type *RetTy, ArrayRef<Value *> Args,
154 ArrayRef<Type *> ArgTys = {},
155 ArrayRef<Value *> MDSources = {}) const;
156 SmallVector<Value *> splitVectorElements(IRBuilderBase &Builder, Value *Vec,
157 unsigned ToWidth) const;
158 Value *joinVectorElements(IRBuilderBase &Builder, ArrayRef<Value *> Values,
159 VectorType *ToType) const;
160
161 std::optional<int> calculatePointerDifference(Value *Ptr0, Value *Ptr1) const;
162
163 unsigned getNumSignificantBits(const Value *V,
164 const Instruction *CtxI = nullptr) const;
165 KnownBits getKnownBits(const Value *V,
166 const Instruction *CtxI = nullptr) const;
167
168 bool isSafeToClone(const Instruction &In) const;
169
170 template <typename T = std::vector<Instruction *>>
171 bool isSafeToMoveBeforeInBB(const Instruction &In,
172 BasicBlock::const_iterator To,
173 const T &IgnoreInsts = {}) const;
174
175 // This function is only used for assertions at the moment.
176 [[maybe_unused]] bool isByteVecTy(Type *Ty) const;
177
178 Function &F;
179 const DataLayout &DL;
180 AliasAnalysis &AA;
181 AssumptionCache &AC;
182 DominatorTree &DT;
183 ScalarEvolution &SE;
184 TargetLibraryInfo &TLI;
185 const HexagonSubtarget &HST;
186 OptimizationRemarkEmitter &ORE;
187
188private:
189 Value *getElementRange(IRBuilderBase &Builder, Value *Lo, Value *Hi,
190 int Start, int Length) const;
191};
192
193class AlignVectors {
194 // This code tries to replace unaligned vector loads/stores with aligned
195 // ones.
196 // Consider unaligned load:
197 // %v = original_load %some_addr, align <bad>
198 // %user = %v
199 // It will generate
200 // = load ..., align <good>
201 // = load ..., align <good>
202 // = valign
203 // etc.
204 // %synthesize = combine/shuffle the loaded data so that it looks
205 // exactly like what "original_load" has loaded.
206 // %user = %synthesize
207 // Similarly for stores.
208public:
209 AlignVectors(const HexagonVectorCombine &HVC_) : HVC(HVC_) {}
210
211 bool run();
212
213private:
214 using InstList = std::vector<Instruction *>;
215 using InstMap = DenseMap<Instruction *, Instruction *>;
216
217 struct AddrInfo {
218 AddrInfo(const AddrInfo &) = default;
219 AddrInfo &operator=(const AddrInfo &) = default;
220 AddrInfo(const HexagonVectorCombine &HVC, Instruction *I, Value *A, Type *T,
221 Align H)
222 : Inst(I), Addr(A), ValTy(T), HaveAlign(H),
223 NeedAlign(HVC.getTypeAlignment(Ty: ValTy)) {}
224
225 // XXX: add Size member?
226 Instruction *Inst;
227 Value *Addr;
228 Type *ValTy;
229 Align HaveAlign;
230 Align NeedAlign;
231 int Offset = 0; // Offset (in bytes) from the first member of the
232 // containing AddrList.
233 };
234 using AddrList = std::vector<AddrInfo>;
235
236 struct InstrLess {
237 bool operator()(const Instruction *A, const Instruction *B) const {
238 return A->comesBefore(Other: B);
239 }
240 };
241 using DepList = std::set<Instruction *, InstrLess>;
242
243 struct MoveGroup {
244 MoveGroup(const AddrInfo &AI, Instruction *B, bool Hvx, bool Load)
245 : Base(B), Main{AI.Inst}, Clones{}, IsHvx(Hvx), IsLoad(Load) {}
246 MoveGroup() = default;
247 Instruction *Base; // Base instruction of the parent address group.
248 InstList Main; // Main group of instructions.
249 InstList Deps; // List of dependencies.
250 InstMap Clones; // Map from original Deps to cloned ones.
251 bool IsHvx; // Is this group of HVX instructions?
252 bool IsLoad; // Is this a load group?
253 };
254 using MoveList = std::vector<MoveGroup>;
255
256 struct ByteSpan {
257 // A representation of "interesting" bytes within a given span of memory.
258 // These bytes are those that are loaded or stored, and they don't have
259 // to cover the entire span of memory.
260 //
261 // The representation works by picking a contiguous sequence of bytes
262 // from somewhere within a llvm::Value, and placing it at a given offset
263 // within the span.
264 //
265 // The sequence of bytes from llvm:Value is represented by Segment.
266 // Block is Segment, plus where it goes in the span.
267 //
268 // An important feature of ByteSpan is being able to make a "section",
269 // i.e. creating another ByteSpan corresponding to a range of offsets
270 // relative to the source span.
271
272 struct Segment {
273 // Segment of a Value: 'Len' bytes starting at byte 'Begin'.
274 Segment(Value *Val, int Begin, int Len)
275 : Val(Val), Start(Begin), Size(Len) {}
276 Segment(const Segment &Seg) = default;
277 Segment &operator=(const Segment &Seg) = default;
278 Value *Val; // Value representable as a sequence of bytes.
279 int Start; // First byte of the value that belongs to the segment.
280 int Size; // Number of bytes in the segment.
281 };
282
283 struct Block {
284 Block(Value *Val, int Len, int Pos) : Seg(Val, 0, Len), Pos(Pos) {}
285 Block(Value *Val, int Off, int Len, int Pos)
286 : Seg(Val, Off, Len), Pos(Pos) {}
287 Block(const Block &Blk) = default;
288 Block &operator=(const Block &Blk) = default;
289 Segment Seg; // Value segment.
290 int Pos; // Position (offset) of the block in the span.
291 };
292
293 int extent() const;
294 ByteSpan section(int Start, int Length) const;
295 ByteSpan &shift(int Offset);
296 SmallVector<Value *, 8> values() const;
297
298 int size() const { return Blocks.size(); }
299 Block &operator[](int i) { return Blocks[i]; }
300 const Block &operator[](int i) const { return Blocks[i]; }
301
302 std::vector<Block> Blocks;
303
304 using iterator = decltype(Blocks)::iterator;
305 iterator begin() { return Blocks.begin(); }
306 iterator end() { return Blocks.end(); }
307 using const_iterator = decltype(Blocks)::const_iterator;
308 const_iterator begin() const { return Blocks.begin(); }
309 const_iterator end() const { return Blocks.end(); }
310 };
311
312 std::optional<AddrInfo> getAddrInfo(Instruction &In) const;
313 bool isHvx(const AddrInfo &AI) const;
314 // This function is only used for assertions at the moment.
315 [[maybe_unused]] bool isSectorTy(Type *Ty) const;
316
317 Value *getPayload(Value *Val) const;
318 Value *getMask(Value *Val) const;
319 Value *getPassThrough(Value *Val) const;
320
321 Value *createAdjustedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy,
322 int Adjust,
323 const InstMap &CloneMap = InstMap()) const;
324 Value *createAlignedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy,
325 int Alignment,
326 const InstMap &CloneMap = InstMap()) const;
327
328 Value *createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
329 Value *Predicate, int Alignment, Value *Mask,
330 Value *PassThru, ArrayRef<Value *> MDSources = {}) const;
331 Value *createSimpleLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
332 int Alignment,
333 ArrayRef<Value *> MDSources = {}) const;
334
335 Value *createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
336 Value *Predicate, int Alignment, Value *Mask,
337 ArrayRef<Value *> MDSources = {}) const;
338 Value *createSimpleStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
339 int Alignment,
340 ArrayRef<Value *> MDSources = {}) const;
341
342 Value *createPredicatedLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
343 Value *Predicate, int Alignment,
344 ArrayRef<Value *> MDSources = {}) const;
345 Value *createPredicatedStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
346 Value *Predicate, int Alignment,
347 ArrayRef<Value *> MDSources = {}) const;
348
349 DepList getUpwardDeps(Instruction *In, Instruction *Base) const;
350 bool createAddressGroups();
351 MoveList createLoadGroups(const AddrList &Group) const;
352 MoveList createStoreGroups(const AddrList &Group) const;
353 bool moveTogether(MoveGroup &Move) const;
354 template <typename T>
355 InstMap cloneBefore(BasicBlock::iterator To, T &&Insts) const;
356
357 void realignLoadGroup(IRBuilderBase &Builder, const ByteSpan &VSpan,
358 int ScLen, Value *AlignVal, Value *AlignAddr) const;
359 void realignStoreGroup(IRBuilderBase &Builder, const ByteSpan &VSpan,
360 int ScLen, Value *AlignVal, Value *AlignAddr) const;
361 bool realignGroup(const MoveGroup &Move);
362 Value *makeTestIfUnaligned(IRBuilderBase &Builder, Value *AlignVal,
363 int Alignment) const;
364
365 using AddrGroupMap = MapVector<Instruction *, AddrList>;
366 AddrGroupMap AddrGroups;
367
368 friend raw_ostream &operator<<(raw_ostream &OS, const AddrList &L);
369 friend raw_ostream &operator<<(raw_ostream &OS, const AddrInfo &AI);
370 friend raw_ostream &operator<<(raw_ostream &OS, const MoveGroup &MG);
371 friend raw_ostream &operator<<(raw_ostream &OS, const MoveList &L);
372 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan::Block &B);
373 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan &BS);
374 friend raw_ostream &operator<<(raw_ostream &OS, const AddrGroupMap &AG);
375 friend raw_ostream &operator<<(raw_ostream &OS, const AddrList &L);
376 friend raw_ostream &operator<<(raw_ostream &OS, const AddrInfo &AI);
377 friend raw_ostream &operator<<(raw_ostream &OS, const MoveGroup &MG);
378 friend raw_ostream &operator<<(raw_ostream &OS, const MoveList &L);
379 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan::Block &B);
380 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan &BS);
381 friend raw_ostream &operator<<(raw_ostream &OS, const AddrGroupMap &AG);
382
383 const HexagonVectorCombine &HVC;
384};
385
386[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
387 const AlignVectors::AddrGroupMap &AG) {
388 OS << "Printing AddrGroups:"
389 << "\n";
390 for (auto &It : AG) {
391 OS << "\n\tInstruction: ";
392 It.first->dump();
393 OS << "\n\tAddrInfo: ";
394 for (auto &AI : It.second)
395 OS << AI << "\n";
396 }
397 return OS;
398}
399
400[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
401 const AlignVectors::AddrList &AL) {
402 OS << "\n *** Addr List: ***\n";
403 for (auto &AG : AL) {
404 OS << "\n *** Addr Group: ***\n";
405 OS << AG;
406 OS << "\n";
407 }
408 return OS;
409}
410
411[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
412 const AlignVectors::AddrInfo &AI) {
413 OS << "Inst: " << AI.Inst << " " << *AI.Inst << '\n';
414 OS << "Addr: " << *AI.Addr << '\n';
415 OS << "Type: " << *AI.ValTy << '\n';
416 OS << "HaveAlign: " << AI.HaveAlign.value() << '\n';
417 OS << "NeedAlign: " << AI.NeedAlign.value() << '\n';
418 OS << "Offset: " << AI.Offset;
419 return OS;
420}
421
422[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
423 const AlignVectors::MoveList &ML) {
424 OS << "\n *** Move List: ***\n";
425 for (auto &MG : ML) {
426 OS << "\n *** Move Group: ***\n";
427 OS << MG;
428 OS << "\n";
429 }
430 return OS;
431}
432
433[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
434 const AlignVectors::MoveGroup &MG) {
435 OS << "IsLoad:" << (MG.IsLoad ? "yes" : "no");
436 OS << ", IsHvx:" << (MG.IsHvx ? "yes" : "no") << '\n';
437 OS << "Main\n";
438 for (Instruction *I : MG.Main)
439 OS << " " << *I << '\n';
440 OS << "Deps\n";
441 for (Instruction *I : MG.Deps)
442 OS << " " << *I << '\n';
443 OS << "Clones\n";
444 for (auto [K, V] : MG.Clones) {
445 OS << " ";
446 K->printAsOperand(O&: OS, PrintType: false);
447 OS << "\t-> " << *V << '\n';
448 }
449 return OS;
450}
451
452[[maybe_unused]] raw_ostream &
453operator<<(raw_ostream &OS, const AlignVectors::ByteSpan::Block &B) {
454 OS << " @" << B.Pos << " [" << B.Seg.Start << ',' << B.Seg.Size << "] ";
455 if (B.Seg.Val == reinterpret_cast<const Value *>(&B)) {
456 OS << "(self:" << B.Seg.Val << ')';
457 } else if (B.Seg.Val != nullptr) {
458 OS << *B.Seg.Val;
459 } else {
460 OS << "(null)";
461 }
462 return OS;
463}
464
465[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
466 const AlignVectors::ByteSpan &BS) {
467 OS << "ByteSpan[size=" << BS.size() << ", extent=" << BS.extent() << '\n';
468 for (const AlignVectors::ByteSpan::Block &B : BS)
469 OS << B << '\n';
470 OS << ']';
471 return OS;
472}
473
474class HvxIdioms {
475public:
476 enum DstQualifier {
477 Undefined = 0,
478 Arithmetic,
479 LdSt,
480 LLVM_Gather,
481 LLVM_Scatter,
482 HEX_Gather_Scatter,
483 HEX_Gather,
484 HEX_Scatter,
485 Call
486 };
487
488 HvxIdioms(const HexagonVectorCombine &HVC_) : HVC(HVC_) {
489 auto *Int32Ty = HVC.getIntTy(Width: 32);
490 HvxI32Ty = HVC.getHvxTy(ElemTy: Int32Ty, /*Pair=*/false);
491 HvxP32Ty = HVC.getHvxTy(ElemTy: Int32Ty, /*Pair=*/true);
492 }
493
494 bool run();
495
496private:
497 enum Signedness { Positive, Signed, Unsigned };
498
499 // Value + sign
500 // This is to keep track of whether the value should be treated as signed
501 // or unsigned, or is known to be positive.
502 struct SValue {
503 Value *Val;
504 Signedness Sgn;
505 };
506
507 struct FxpOp {
508 unsigned Opcode;
509 unsigned Frac; // Number of fraction bits
510 SValue X, Y;
511 // If present, add 1 << RoundAt before shift:
512 std::optional<unsigned> RoundAt;
513 VectorType *ResTy;
514 };
515
516 auto getNumSignificantBits(Value *V, Instruction *In) const
517 -> std::pair<unsigned, Signedness>;
518 auto canonSgn(SValue X, SValue Y) const -> std::pair<SValue, SValue>;
519
520 auto matchFxpMul(Instruction &In) const -> std::optional<FxpOp>;
521 auto processFxpMul(Instruction &In, const FxpOp &Op) const -> Value *;
522
523 auto processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
524 const FxpOp &Op) const -> Value *;
525 auto createMulQ15(IRBuilderBase &Builder, SValue X, SValue Y,
526 bool Rounding) const -> Value *;
527 auto createMulQ31(IRBuilderBase &Builder, SValue X, SValue Y,
528 bool Rounding) const -> Value *;
529 // Return {Result, Carry}, where Carry is a vector predicate.
530 auto createAddCarry(IRBuilderBase &Builder, Value *X, Value *Y,
531 Value *CarryIn = nullptr) const
532 -> std::pair<Value *, Value *>;
533 auto createMul16(IRBuilderBase &Builder, SValue X, SValue Y) const -> Value *;
534 auto createMulH16(IRBuilderBase &Builder, SValue X, SValue Y) const
535 -> Value *;
536 auto createMul32(IRBuilderBase &Builder, SValue X, SValue Y) const
537 -> std::pair<Value *, Value *>;
538 auto createAddLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
539 ArrayRef<Value *> WordY) const -> SmallVector<Value *>;
540 auto createMulLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
541 Signedness SgnX, ArrayRef<Value *> WordY,
542 Signedness SgnY) const -> SmallVector<Value *>;
543
544 bool matchMLoad(Instruction &In) const;
545 bool matchMStore(Instruction &In) const;
546 Value *processMLoad(Instruction &In) const;
547 Value *processMStore(Instruction &In) const;
548 std::optional<uint64_t> getAlignment(Instruction &In, Value *ptr) const;
549 std::optional<uint64_t>
550 getAlignmentImpl(Instruction &In, Value *ptr,
551 SmallPtrSet<Value *, 16> &Visited) const;
552 std::optional<uint64_t> getPHIBaseMinAlignment(Instruction &In,
553 PHINode *PN) const;
554
555 // Vector manipulations for Ripple
556 bool matchScatter(Instruction &In) const;
557 bool matchGather(Instruction &In) const;
558 Value *processVScatter(Instruction &In) const;
559 Value *processVGather(Instruction &In) const;
560
561 VectorType *HvxI32Ty;
562 VectorType *HvxP32Ty;
563 const HexagonVectorCombine &HVC;
564
565 friend raw_ostream &operator<<(raw_ostream &, const FxpOp &);
566};
567
568[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
569 const HvxIdioms::FxpOp &Op) {
570 static const char *SgnNames[] = {"Positive", "Signed", "Unsigned"};
571 OS << Instruction::getOpcodeName(Opcode: Op.Opcode) << '.' << Op.Frac;
572 if (Op.RoundAt.has_value()) {
573 if (Op.Frac != 0 && *Op.RoundAt == Op.Frac - 1) {
574 OS << ":rnd";
575 } else {
576 OS << " + 1<<" << *Op.RoundAt;
577 }
578 }
579 OS << "\n X:(" << SgnNames[Op.X.Sgn] << ") " << *Op.X.Val << "\n"
580 << " Y:(" << SgnNames[Op.Y.Sgn] << ") " << *Op.Y.Val;
581 return OS;
582}
583
584} // namespace
585
586namespace {
587
588template <typename T> T *getIfUnordered(T *MaybeT) {
589 return MaybeT && MaybeT->isUnordered() ? MaybeT : nullptr;
590}
591template <typename T> T *isCandidate(Instruction *In) {
592 return dyn_cast<T>(In);
593}
594template <> LoadInst *isCandidate<LoadInst>(Instruction *In) {
595 return getIfUnordered(MaybeT: dyn_cast<LoadInst>(Val: In));
596}
597template <> StoreInst *isCandidate<StoreInst>(Instruction *In) {
598 return getIfUnordered(MaybeT: dyn_cast<StoreInst>(Val: In));
599}
600
601// Forward other erase_ifs to the LLVM implementations.
602template <typename Pred, typename T> void erase_if(T &&container, Pred p) {
603 llvm::erase_if(std::forward<T>(container), p);
604}
605
606} // namespace
607
608// --- Begin AlignVectors
609
610// For brevity, only consider loads. We identify a group of loads where we
611// know the relative differences between their addresses, so we know how they
612// are laid out in memory (relative to one another). These loads can overlap,
613// can be shorter or longer than the desired vector length.
614// Ultimately we want to generate a sequence of aligned loads that will load
615// every byte that the original loads loaded, and have the program use these
616// loaded values instead of the original loads.
617// We consider the contiguous memory area spanned by all these loads.
618//
619// Let's say that a single aligned vector load can load 16 bytes at a time.
620// If the program wanted to use a byte at offset 13 from the beginning of the
621// original span, it will be a byte at offset 13+x in the aligned data for
622// some x>=0. This may happen to be in the first aligned load, or in the load
623// following it. Since we generally don't know what the that alignment value
624// is at compile time, we proactively do valigns on the aligned loads, so that
625// byte that was at offset 13 is still at offset 13 after the valigns.
626//
627// This will be the starting point for making the rest of the program use the
628// data loaded by the new loads.
629// For each original load, and its users:
630// %v = load ...
631// ... = %v
632// ... = %v
633// we create
634// %new_v = extract/combine/shuffle data from loaded/valigned vectors so
635// it contains the same value as %v did before
636// then replace all users of %v with %new_v.
637// ... = %new_v
638// ... = %new_v
639
640auto AlignVectors::ByteSpan::extent() const -> int {
641 if (size() == 0)
642 return 0;
643 int Min = Blocks[0].Pos;
644 int Max = Blocks[0].Pos + Blocks[0].Seg.Size;
645 for (int i = 1, e = size(); i != e; ++i) {
646 Min = std::min(a: Min, b: Blocks[i].Pos);
647 Max = std::max(a: Max, b: Blocks[i].Pos + Blocks[i].Seg.Size);
648 }
649 return Max - Min;
650}
651
652auto AlignVectors::ByteSpan::section(int Start, int Length) const -> ByteSpan {
653 ByteSpan Section;
654 for (const ByteSpan::Block &B : Blocks) {
655 int L = std::max(a: B.Pos, b: Start); // Left end.
656 int R = std::min(a: B.Pos + B.Seg.Size, b: Start + Length); // Right end+1.
657 if (L < R) {
658 // How much to chop off the beginning of the segment:
659 int Off = L > B.Pos ? L - B.Pos : 0;
660 Section.Blocks.emplace_back(args: B.Seg.Val, args: B.Seg.Start + Off, args: R - L, args&: L);
661 }
662 }
663 return Section;
664}
665
666auto AlignVectors::ByteSpan::shift(int Offset) -> ByteSpan & {
667 for (Block &B : Blocks)
668 B.Pos += Offset;
669 return *this;
670}
671
672auto AlignVectors::ByteSpan::values() const -> SmallVector<Value *, 8> {
673 SmallVector<Value *, 8> Values(Blocks.size());
674 for (int i = 0, e = Blocks.size(); i != e; ++i)
675 Values[i] = Blocks[i].Seg.Val;
676 return Values;
677}
678
679// Turn a requested integer alignment into the effective Align to use.
680// If Requested == 0 -> use ABI alignment of the value type (old semantics).
681// 0 means "ABI alignment" in old IR.
682static Align effectiveAlignForValueTy(const DataLayout &DL, Type *ValTy,
683 int Requested) {
684 if (Requested > 0)
685 return Align(static_cast<uint64_t>(Requested));
686 return Align(DL.getABITypeAlign(Ty: ValTy).value());
687}
688
689auto AlignVectors::getAddrInfo(Instruction &In) const
690 -> std::optional<AddrInfo> {
691 if (auto *L = isCandidate<LoadInst>(In: &In))
692 return AddrInfo(HVC, L, L->getPointerOperand(), L->getType(),
693 L->getAlign());
694 if (auto *S = isCandidate<StoreInst>(In: &In))
695 return AddrInfo(HVC, S, S->getPointerOperand(),
696 S->getValueOperand()->getType(), S->getAlign());
697 if (auto *II = isCandidate<IntrinsicInst>(In: &In)) {
698 Intrinsic::ID ID = II->getIntrinsicID();
699 switch (ID) {
700 case Intrinsic::masked_load:
701 return AddrInfo(HVC, II, II->getArgOperand(i: 0), II->getType(),
702 II->getParamAlign(ArgNo: 0).valueOrOne());
703 case Intrinsic::masked_store:
704 return AddrInfo(HVC, II, II->getArgOperand(i: 1),
705 II->getArgOperand(i: 0)->getType(),
706 II->getParamAlign(ArgNo: 1).valueOrOne());
707 }
708 }
709 return std::nullopt;
710}
711
712auto AlignVectors::isHvx(const AddrInfo &AI) const -> bool {
713 return HVC.HST.isTypeForHVX(VecTy: AI.ValTy);
714}
715
716auto AlignVectors::getPayload(Value *Val) const -> Value * {
717 if (auto *In = dyn_cast<Instruction>(Val)) {
718 Intrinsic::ID ID = 0;
719 if (auto *II = dyn_cast<IntrinsicInst>(Val: In))
720 ID = II->getIntrinsicID();
721 if (isa<StoreInst>(Val: In) || ID == Intrinsic::masked_store)
722 return In->getOperand(i: 0);
723 }
724 return Val;
725}
726
727auto AlignVectors::getMask(Value *Val) const -> Value * {
728 if (auto *II = dyn_cast<IntrinsicInst>(Val)) {
729 switch (II->getIntrinsicID()) {
730 case Intrinsic::masked_load:
731 return II->getArgOperand(i: 1);
732 case Intrinsic::masked_store:
733 return II->getArgOperand(i: 2);
734 }
735 }
736
737 Type *ValTy = getPayload(Val)->getType();
738 if (auto *VecTy = dyn_cast<VectorType>(Val: ValTy))
739 return Constant::getAllOnesValue(Ty: HVC.getBoolTy(ElemCount: HVC.length(Ty: VecTy)));
740 return Constant::getAllOnesValue(Ty: HVC.getBoolTy());
741}
742
743auto AlignVectors::getPassThrough(Value *Val) const -> Value * {
744 if (auto *II = dyn_cast<IntrinsicInst>(Val)) {
745 if (II->getIntrinsicID() == Intrinsic::masked_load)
746 return II->getArgOperand(i: 2);
747 }
748 return UndefValue::get(T: getPayload(Val)->getType());
749}
750
751auto AlignVectors::createAdjustedPointer(IRBuilderBase &Builder, Value *Ptr,
752 Type *ValTy, int Adjust,
753 const InstMap &CloneMap) const
754 -> Value * {
755 if (auto *I = dyn_cast<Instruction>(Val: Ptr))
756 if (Instruction *New = CloneMap.lookup(Val: I))
757 Ptr = New;
758 return Builder.CreatePtrAdd(Ptr, Offset: HVC.getConstInt(Val: Adjust), Name: "gep");
759}
760
761auto AlignVectors::createAlignedPointer(IRBuilderBase &Builder, Value *Ptr,
762 Type *ValTy, int Alignment,
763 const InstMap &CloneMap) const
764 -> Value * {
765 auto remap = [&](Value *V) -> Value * {
766 if (auto *I = dyn_cast<Instruction>(Val: V)) {
767 for (auto [Old, New] : CloneMap)
768 I->replaceUsesOfWith(From: Old, To: New);
769 return I;
770 }
771 return V;
772 };
773 Value *AsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: HVC.getIntTy(), Name: "pti");
774 Value *Mask = HVC.getConstInt(Val: -Alignment);
775 Value *And = Builder.CreateAnd(LHS: remap(AsInt), RHS: Mask, Name: "and");
776 return Builder.CreateIntToPtr(
777 V: And, DestTy: PointerType::getUnqual(C&: ValTy->getContext()), Name: "itp");
778}
779
780auto AlignVectors::createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
781 Value *Predicate, int Alignment, Value *Mask,
782 Value *PassThru,
783 ArrayRef<Value *> MDSources) const -> Value * {
784 // Predicate is nullptr if not creating predicated load
785 if (Predicate) {
786 assert(!Predicate->getType()->isVectorTy() &&
787 "Expectning scalar predicate");
788 if (HVC.isFalse(Val: Predicate))
789 return UndefValue::get(T: ValTy);
790 if (!HVC.isTrue(Val: Predicate)) {
791 Value *Load = createPredicatedLoad(Builder, ValTy, Ptr, Predicate,
792 Alignment, MDSources);
793 return Builder.CreateSelect(C: Mask, True: Load, False: PassThru);
794 }
795 // Predicate == true here.
796 }
797 assert(!HVC.isUndef(Mask)); // Should this be allowed?
798 if (HVC.isZero(Val: Mask))
799 return PassThru;
800
801 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy, Requested: Alignment);
802 if (HVC.isTrue(Val: Mask))
803 return createSimpleLoad(Builder, ValTy, Ptr, Alignment: EffA.value(), MDSources);
804
805 Instruction *Load =
806 Builder.CreateMaskedLoad(Ty: ValTy, Ptr, Alignment: EffA, Mask, PassThru, Name: "mld");
807 LLVM_DEBUG(dbgs() << "\t[Creating masked Load:] "; Load->dump());
808 propagateMetadata(I: Load, VL: MDSources);
809 return Load;
810}
811
812auto AlignVectors::createSimpleLoad(IRBuilderBase &Builder, Type *ValTy,
813 Value *Ptr, int Alignment,
814 ArrayRef<Value *> MDSources) const
815 -> Value * {
816 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy, Requested: Alignment);
817 Instruction *Load = Builder.CreateAlignedLoad(Ty: ValTy, Ptr, Align: EffA, Name: "ald");
818 propagateMetadata(I: Load, VL: MDSources);
819 LLVM_DEBUG(dbgs() << "\t[Creating Load:] "; Load->dump());
820 return Load;
821}
822
823auto AlignVectors::createPredicatedLoad(IRBuilderBase &Builder, Type *ValTy,
824 Value *Ptr, Value *Predicate,
825 int Alignment,
826 ArrayRef<Value *> MDSources) const
827 -> Value * {
828 assert(HVC.HST.isTypeForHVX(ValTy) &&
829 "Predicates 'scalar' vector loads not yet supported");
830 assert(Predicate);
831 assert(!Predicate->getType()->isVectorTy() && "Expectning scalar predicate");
832 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy, Requested: Alignment);
833 assert(HVC.getSizeOf(ValTy, HVC.Alloc) % EffA.value() == 0);
834
835 if (HVC.isFalse(Val: Predicate))
836 return UndefValue::get(T: ValTy);
837 if (HVC.isTrue(Val: Predicate))
838 return createSimpleLoad(Builder, ValTy, Ptr, Alignment: EffA.value(), MDSources);
839
840 auto V6_vL32b_pred_ai = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vL32b_pred_ai);
841 // FIXME: This may not put the offset from Ptr into the vmem offset.
842 return HVC.createHvxIntrinsic(Builder, IntID: V6_vL32b_pred_ai, RetTy: ValTy,
843 Args: {Predicate, Ptr, HVC.getConstInt(Val: 0)}, ArgTys: {},
844 MDSources);
845}
846
847auto AlignVectors::createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
848 Value *Predicate, int Alignment, Value *Mask,
849 ArrayRef<Value *> MDSources) const -> Value * {
850 if (HVC.isZero(Val: Mask) || HVC.isUndef(Val) || HVC.isUndef(Val: Mask))
851 return UndefValue::get(T: Val->getType());
852 assert(!Predicate || (!Predicate->getType()->isVectorTy() &&
853 "Expectning scalar predicate"));
854 if (Predicate) {
855 if (HVC.isFalse(Val: Predicate))
856 return UndefValue::get(T: Val->getType());
857 if (HVC.isTrue(Val: Predicate))
858 Predicate = nullptr;
859 }
860 // Here both Predicate and Mask are true or unknown.
861
862 if (HVC.isTrue(Val: Mask)) {
863 if (Predicate) { // Predicate unknown
864 return createPredicatedStore(Builder, Val, Ptr, Predicate, Alignment,
865 MDSources);
866 }
867 // Predicate is true:
868 return createSimpleStore(Builder, Val, Ptr, Alignment, MDSources);
869 }
870
871 // Mask is unknown
872 if (!Predicate) {
873 Instruction *Store =
874 Builder.CreateMaskedStore(Val, Ptr, Alignment: Align(Alignment), Mask);
875 propagateMetadata(I: Store, VL: MDSources);
876 return Store;
877 }
878
879 // Both Predicate and Mask are unknown.
880 // Emulate masked store with predicated-load + mux + predicated-store.
881 Value *PredLoad = createPredicatedLoad(Builder, ValTy: Val->getType(), Ptr,
882 Predicate, Alignment, MDSources);
883 Value *Mux = Builder.CreateSelect(C: Mask, True: Val, False: PredLoad);
884 return createPredicatedStore(Builder, Val: Mux, Ptr, Predicate, Alignment,
885 MDSources);
886}
887
888auto AlignVectors::createSimpleStore(IRBuilderBase &Builder, Value *Val,
889 Value *Ptr, int Alignment,
890 ArrayRef<Value *> MDSources) const
891 -> Value * {
892 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy: Val->getType(), Requested: Alignment);
893 Instruction *Store = Builder.CreateAlignedStore(Val, Ptr, Align: EffA);
894 LLVM_DEBUG(dbgs() << "\t[Creating store:] "; Store->dump());
895 propagateMetadata(I: Store, VL: MDSources);
896 return Store;
897}
898
899auto AlignVectors::createPredicatedStore(IRBuilderBase &Builder, Value *Val,
900 Value *Ptr, Value *Predicate,
901 int Alignment,
902 ArrayRef<Value *> MDSources) const
903 -> Value * {
904 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy: Val->getType(), Requested: Alignment);
905 assert(HVC.HST.isTypeForHVX(Val->getType()) &&
906 "Predicates 'scalar' vector stores not yet supported");
907 assert(Predicate);
908 if (HVC.isFalse(Val: Predicate))
909 return UndefValue::get(T: Val->getType());
910 if (HVC.isTrue(Val: Predicate))
911 return createSimpleStore(Builder, Val, Ptr, Alignment: EffA.value(), MDSources);
912
913 assert(HVC.getSizeOf(Val, HVC.Alloc) % EffA.value() == 0);
914 auto V6_vS32b_pred_ai = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vS32b_pred_ai);
915 // FIXME: This may not put the offset from Ptr into the vmem offset.
916 return HVC.createHvxIntrinsic(Builder, IntID: V6_vS32b_pred_ai, RetTy: nullptr,
917 Args: {Predicate, Ptr, HVC.getConstInt(Val: 0), Val}, ArgTys: {},
918 MDSources);
919}
920
921auto AlignVectors::getUpwardDeps(Instruction *In, Instruction *Base) const
922 -> DepList {
923 BasicBlock *Parent = Base->getParent();
924 assert(In->getParent() == Parent &&
925 "Base and In should be in the same block");
926 assert(Base->comesBefore(In) && "Base should come before In");
927
928 DepList Deps;
929 std::deque<Instruction *> WorkQ = {In};
930 while (!WorkQ.empty()) {
931 Instruction *D = WorkQ.front();
932 WorkQ.pop_front();
933 if (D != In)
934 Deps.insert(x: D);
935 for (Value *Op : D->operands()) {
936 if (auto *I = dyn_cast<Instruction>(Val: Op)) {
937 if (I->getParent() == Parent && Base->comesBefore(Other: I))
938 WorkQ.push_back(x: I);
939 }
940 }
941 }
942 return Deps;
943}
944
945auto AlignVectors::createAddressGroups() -> bool {
946 // An address group created here may contain instructions spanning
947 // multiple basic blocks.
948 AddrList WorkStack;
949
950 auto findBaseAndOffset = [&](AddrInfo &AI) -> std::pair<Instruction *, int> {
951 for (AddrInfo &W : WorkStack) {
952 if (auto D = HVC.calculatePointerDifference(Ptr0: AI.Addr, Ptr1: W.Addr))
953 return std::make_pair(x&: W.Inst, y&: *D);
954 }
955 return std::make_pair(x: nullptr, y: 0);
956 };
957
958 auto traverseBlock = [&](DomTreeNode *DomN, auto Visit) -> void {
959 BasicBlock &Block = *DomN->getBlock();
960 for (Instruction &I : Block) {
961 auto AI = this->getAddrInfo(In&: I); // Use this-> for gcc6.
962 if (!AI)
963 continue;
964 auto F = findBaseAndOffset(*AI);
965 Instruction *GroupInst;
966 if (Instruction *BI = F.first) {
967 AI->Offset = F.second;
968 GroupInst = BI;
969 } else {
970 WorkStack.push_back(x: *AI);
971 GroupInst = AI->Inst;
972 }
973 AddrGroups[GroupInst].push_back(x: *AI);
974 }
975
976 for (DomTreeNode *C : DomN->children())
977 Visit(C, Visit);
978
979 while (!WorkStack.empty() && WorkStack.back().Inst->getParent() == &Block)
980 WorkStack.pop_back();
981 };
982
983 traverseBlock(HVC.DT.getRootNode(), traverseBlock);
984 assert(WorkStack.empty());
985
986 // AddrGroups are formed.
987 // Remove groups of size 1.
988 AddrGroups.remove_if(Pred: [](auto &G) { return G.second.size() == 1; });
989 // Remove groups that don't use HVX types.
990 AddrGroups.remove_if(Pred: [&](auto &G) {
991 return llvm::none_of(
992 G.second, [&](auto &I) { return HVC.HST.isTypeForHVX(VecTy: I.ValTy); });
993 });
994
995 LLVM_DEBUG(dbgs() << AddrGroups);
996 return !AddrGroups.empty();
997}
998
999auto AlignVectors::createLoadGroups(const AddrList &Group) const -> MoveList {
1000 // Form load groups.
1001 // To avoid complications with moving code across basic blocks, only form
1002 // groups that are contained within a single basic block.
1003 unsigned SizeLimit = VAGroupSizeLimit;
1004 if (SizeLimit == 0)
1005 return {};
1006
1007 auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) {
1008 assert(!Move.Main.empty() && "Move group should have non-empty Main");
1009 if (Move.Main.size() >= SizeLimit) {
1010 HVC.ORE.emit(RemarkBuilder: [&]() {
1011 return OptimizationRemarkMissed(DEBUG_TYPE, "GroupSizeLimitExceeded",
1012 Info.Inst->getDebugLoc(),
1013 Info.Inst->getParent())
1014 << "alignment group exceeds size limit";
1015 });
1016 return false;
1017 }
1018 // Don't mix HVX and non-HVX instructions.
1019 if (Move.IsHvx != isHvx(AI: Info))
1020 return false;
1021 // Leading instruction in the load group.
1022 Instruction *Base = Move.Main.front();
1023 if (Base->getParent() != Info.Inst->getParent())
1024 return false;
1025 // Check if it's safe to move the load.
1026 if (!HVC.isSafeToMoveBeforeInBB(In: *Info.Inst, To: Base->getIterator())) {
1027 HVC.ORE.emit(RemarkBuilder: [&]() {
1028 return OptimizationRemarkMissed(DEBUG_TYPE, "UnsafeToRelocate",
1029 Info.Inst->getDebugLoc(),
1030 Info.Inst->getParent())
1031 << "unsafe to relocate memory access for alignment";
1032 });
1033 return false;
1034 }
1035 // And if it's safe to clone the dependencies.
1036 auto isSafeToCopyAtBase = [&](const Instruction *I) {
1037 return HVC.isSafeToMoveBeforeInBB(In: *I, To: Base->getIterator()) &&
1038 HVC.isSafeToClone(In: *I);
1039 };
1040 DepList Deps = getUpwardDeps(In: Info.Inst, Base);
1041 if (!llvm::all_of(Range&: Deps, P: isSafeToCopyAtBase))
1042 return false;
1043
1044 Move.Main.push_back(x: Info.Inst);
1045 llvm::append_range(C&: Move.Deps, R&: Deps);
1046 return true;
1047 };
1048
1049 MoveList LoadGroups;
1050
1051 for (const AddrInfo &Info : Group) {
1052 if (!Info.Inst->mayReadFromMemory())
1053 continue;
1054 if (LoadGroups.empty() || !tryAddTo(Info, LoadGroups.back()))
1055 LoadGroups.emplace_back(args: Info, args: Group.front().Inst, args: isHvx(AI: Info), args: true);
1056 }
1057
1058 // Erase groups smaller than the minimum load group size.
1059 unsigned LoadGroupSizeLimit = MinLoadGroupSizeForAlignment;
1060 erase_if(container&: LoadGroups, p: [LoadGroupSizeLimit](const MoveGroup &G) {
1061 return G.Main.size() < LoadGroupSizeLimit;
1062 });
1063
1064 // Erase HVX groups on targets < HvxV62 (due to lack of predicated loads).
1065 if (!HVC.HST.useHVXV62Ops()) {
1066 bool HadHvx =
1067 llvm::any_of(Range&: LoadGroups, P: [](const MoveGroup &G) { return G.IsHvx; });
1068 erase_if(container&: LoadGroups, p: [](const MoveGroup &G) { return G.IsHvx; });
1069 if (HadHvx) {
1070 HVC.ORE.emit(RemarkBuilder: [&]() {
1071 return OptimizationRemarkMissed(DEBUG_TYPE, "HvxVersionTooLow",
1072 HVC.F.getSubprogram(), &HVC.F.front())
1073 << "HVX version too low for predicated load operations";
1074 });
1075 }
1076 }
1077
1078 LLVM_DEBUG(dbgs() << "LoadGroups list: " << LoadGroups);
1079 return LoadGroups;
1080}
1081
1082auto AlignVectors::createStoreGroups(const AddrList &Group) const -> MoveList {
1083 // Form store groups.
1084 // To avoid complications with moving code across basic blocks, only form
1085 // groups that are contained within a single basic block.
1086 unsigned SizeLimit = VAGroupSizeLimit;
1087 if (SizeLimit == 0)
1088 return {};
1089
1090 auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) {
1091 assert(!Move.Main.empty() && "Move group should have non-empty Main");
1092 if (Move.Main.size() >= SizeLimit) {
1093 HVC.ORE.emit(RemarkBuilder: [&]() {
1094 return OptimizationRemarkMissed(DEBUG_TYPE, "GroupSizeLimitExceeded",
1095 Info.Inst->getDebugLoc(),
1096 Info.Inst->getParent())
1097 << "alignment group exceeds size limit";
1098 });
1099 return false;
1100 }
1101 // For stores with return values we'd have to collect downward dependencies.
1102 // There are no such stores that we handle at the moment, so omit that.
1103 assert(Info.Inst->getType()->isVoidTy() &&
1104 "Not handling stores with return values");
1105 // Don't mix HVX and non-HVX instructions.
1106 if (Move.IsHvx != isHvx(AI: Info))
1107 return false;
1108 // For stores we need to be careful whether it's safe to move them.
1109 // Stores that are otherwise safe to move together may not appear safe
1110 // to move over one another (i.e. isSafeToMoveBefore may return false).
1111 Instruction *Base = Move.Main.front();
1112 if (Base->getParent() != Info.Inst->getParent())
1113 return false;
1114 if (!HVC.isSafeToMoveBeforeInBB(In: *Info.Inst, To: Base->getIterator(),
1115 IgnoreInsts: Move.Main)) {
1116 HVC.ORE.emit(RemarkBuilder: [&]() {
1117 return OptimizationRemarkMissed(DEBUG_TYPE, "UnsafeToRelocate",
1118 Info.Inst->getDebugLoc(),
1119 Info.Inst->getParent())
1120 << "unsafe to relocate memory access for alignment";
1121 });
1122 return false;
1123 }
1124 Move.Main.push_back(x: Info.Inst);
1125 return true;
1126 };
1127
1128 MoveList StoreGroups;
1129
1130 for (auto I = Group.rbegin(), E = Group.rend(); I != E; ++I) {
1131 const AddrInfo &Info = *I;
1132 if (!Info.Inst->mayWriteToMemory())
1133 continue;
1134 if (StoreGroups.empty() || !tryAddTo(Info, StoreGroups.back()))
1135 StoreGroups.emplace_back(args: Info, args: Group.front().Inst, args: isHvx(AI: Info), args: false);
1136 }
1137
1138 // Erase singleton groups.
1139 erase_if(container&: StoreGroups, p: [](const MoveGroup &G) { return G.Main.size() <= 1; });
1140
1141 // Erase HVX groups on targets < HvxV62 (due to lack of predicated loads).
1142 if (!HVC.HST.useHVXV62Ops()) {
1143 bool HadHvx =
1144 llvm::any_of(Range&: StoreGroups, P: [](const MoveGroup &G) { return G.IsHvx; });
1145 erase_if(container&: StoreGroups, p: [](const MoveGroup &G) { return G.IsHvx; });
1146 if (HadHvx) {
1147 HVC.ORE.emit(RemarkBuilder: [&]() {
1148 return OptimizationRemarkMissed(DEBUG_TYPE, "HvxVersionTooLow",
1149 HVC.F.getSubprogram(), &HVC.F.front())
1150 << "HVX version too low for predicated store operations";
1151 });
1152 }
1153 }
1154
1155 // Erase groups where every store is a full HVX vector. The reason is that
1156 // aligning predicated stores generates complex code that may be less
1157 // efficient than a sequence of unaligned vector stores.
1158 if (!VADoFullStores) {
1159 erase_if(container&: StoreGroups, p: [this](const MoveGroup &G) {
1160 return G.IsHvx && llvm::all_of(Range: G.Main, P: [this](Instruction *S) {
1161 auto MaybeInfo = this->getAddrInfo(In&: *S);
1162 assert(MaybeInfo.has_value());
1163 return HVC.HST.isHVXVectorType(
1164 VecTy: EVT::getEVT(Ty: MaybeInfo->ValTy, HandleUnknown: false));
1165 });
1166 });
1167 }
1168
1169 return StoreGroups;
1170}
1171
1172auto AlignVectors::moveTogether(MoveGroup &Move) const -> bool {
1173 // Move all instructions to be adjacent.
1174 assert(!Move.Main.empty() && "Move group should have non-empty Main");
1175 Instruction *Where = Move.Main.front();
1176
1177 if (Move.IsLoad) {
1178 // Move all the loads (and dependencies) to where the first load is.
1179 // Clone all deps to before Where, keeping order.
1180 Move.Clones = cloneBefore(To: Where->getIterator(), Insts&: Move.Deps);
1181 // Move all main instructions to after Where, keeping order.
1182 ArrayRef<Instruction *> Main(Move.Main);
1183 for (Instruction *M : Main) {
1184 if (M != Where)
1185 M->moveAfter(MovePos: Where);
1186 for (auto [Old, New] : Move.Clones)
1187 M->replaceUsesOfWith(From: Old, To: New);
1188 Where = M;
1189 }
1190 // Replace Deps with the clones.
1191 for (int i = 0, e = Move.Deps.size(); i != e; ++i)
1192 Move.Deps[i] = Move.Clones[Move.Deps[i]];
1193 } else {
1194 // Move all the stores to where the last store is.
1195 // NOTE: Deps are empty for "store" groups. If they need to be
1196 // non-empty, decide on the order.
1197 assert(Move.Deps.empty());
1198 // Move all main instructions to before Where, inverting order.
1199 ArrayRef<Instruction *> Main(Move.Main);
1200 for (Instruction *M : Main.drop_front(N: 1)) {
1201 M->moveBefore(InsertPos: Where->getIterator());
1202 Where = M;
1203 }
1204 }
1205
1206 return Move.Main.size() + Move.Deps.size() > 1;
1207}
1208
1209template <typename T>
1210auto AlignVectors::cloneBefore(BasicBlock::iterator To, T &&Insts) const
1211 -> InstMap {
1212 InstMap Map;
1213
1214 for (Instruction *I : Insts) {
1215 assert(HVC.isSafeToClone(*I));
1216 Instruction *C = I->clone();
1217 C->setName(Twine("c.") + I->getName() + ".");
1218 C->insertBefore(InsertPos: To);
1219
1220 for (auto [Old, New] : Map)
1221 C->replaceUsesOfWith(From: Old, To: New);
1222 Map.insert(KV: std::make_pair(x&: I, y&: C));
1223 }
1224 return Map;
1225}
1226
1227auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder,
1228 const ByteSpan &VSpan, int ScLen,
1229 Value *AlignVal, Value *AlignAddr) const
1230 -> void {
1231 LLVM_DEBUG(dbgs() << __func__ << "\n");
1232
1233 Type *SecTy = HVC.getByteTy(ElemCount: ScLen);
1234 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1235 bool DoAlign = !HVC.isZero(Val: AlignVal);
1236 BasicBlock::iterator BasePos = Builder.GetInsertPoint();
1237 BasicBlock *BaseBlock = Builder.GetInsertBlock();
1238
1239 ByteSpan ASpan;
1240 auto *True = Constant::getAllOnesValue(Ty: HVC.getBoolTy(ElemCount: ScLen));
1241 auto *Undef = UndefValue::get(T: SecTy);
1242
1243 // Created load does not have to be "Instruction" (e.g. "undef").
1244 SmallVector<Value *> Loads(NumSectors + DoAlign, nullptr);
1245
1246 // We could create all of the aligned loads, and generate the valigns
1247 // at the location of the first load, but for large load groups, this
1248 // could create highly suboptimal code (there have been groups of 140+
1249 // loads in real code).
1250 // Instead, place the loads/valigns as close to the users as possible.
1251 // In any case we need to have a mapping from the blocks of VSpan (the
1252 // span covered by the pre-existing loads) to ASpan (the span covered
1253 // by the aligned loads). There is a small problem, though: ASpan needs
1254 // to have pointers to the loads/valigns, but we don't have these loads
1255 // because we don't know where to put them yet. We find out by creating
1256 // a section of ASpan that corresponds to values (blocks) from VSpan,
1257 // and checking where the new load should be placed. We need to attach
1258 // this location information to each block in ASpan somehow, so we put
1259 // distincts values for Seg.Val in each ASpan.Blocks[i], and use a map
1260 // to store the location for each Seg.Val.
1261 // The distinct values happen to be Blocks[i].Seg.Val = &Blocks[i],
1262 // which helps with printing ByteSpans without crashing when printing
1263 // Segments with these temporary identifiers in place of Val.
1264
1265 // Populate the blocks first, to avoid reallocations of the vector
1266 // interfering with generating the placeholder addresses.
1267 for (int Index = 0; Index != NumSectors; ++Index)
1268 ASpan.Blocks.emplace_back(args: nullptr, args&: ScLen, args: Index * ScLen);
1269 for (int Index = 0; Index != NumSectors; ++Index) {
1270 ASpan.Blocks[Index].Seg.Val =
1271 reinterpret_cast<Value *>(&ASpan.Blocks[Index]);
1272 }
1273
1274 // Multiple values from VSpan can map to the same value in ASpan. Since we
1275 // try to create loads lazily, we need to find the earliest use for each
1276 // value from ASpan.
1277 DenseMap<void *, Instruction *> EarliestUser;
1278 auto isEarlier = [](Instruction *A, Instruction *B) {
1279 if (B == nullptr)
1280 return true;
1281 if (A == nullptr)
1282 return false;
1283 assert(A->getParent() == B->getParent());
1284 return A->comesBefore(Other: B);
1285 };
1286 auto earliestUser = [&](const auto &Uses) {
1287 Instruction *User = nullptr;
1288 for (const Use &U : Uses) {
1289 auto *I = dyn_cast<Instruction>(Val: U.getUser());
1290 assert(I != nullptr && "Load used in a non-instruction?");
1291 // Make sure we only consider users in this block, but we need
1292 // to remember if there were users outside the block too. This is
1293 // because if no users are found, aligned loads will not be created.
1294 if (I->getParent() == BaseBlock) {
1295 if (!isa<PHINode>(Val: I))
1296 User = std::min(a: User, b: I, comp: isEarlier);
1297 } else {
1298 User = std::min(a: User, b: BaseBlock->getTerminator(), comp: isEarlier);
1299 }
1300 }
1301 return User;
1302 };
1303
1304 for (const ByteSpan::Block &B : VSpan) {
1305 ByteSpan ASection = ASpan.section(Start: B.Pos, Length: B.Seg.Size);
1306 for (const ByteSpan::Block &S : ASection) {
1307 auto &EU = EarliestUser[S.Seg.Val];
1308 EU = std::min(a: EU, b: earliestUser(B.Seg.Val->uses()), comp: isEarlier);
1309 }
1310 }
1311
1312 LLVM_DEBUG({
1313 dbgs() << "ASpan:\n" << ASpan << '\n';
1314 dbgs() << "Earliest users of ASpan:\n";
1315 for (auto &[Val, User] : EarliestUser) {
1316 dbgs() << Val << "\n ->" << *User << '\n';
1317 }
1318 });
1319
1320 auto createLoad = [&](IRBuilderBase &Builder, const ByteSpan &VSpan,
1321 int Index, bool MakePred) {
1322 Value *Ptr =
1323 createAdjustedPointer(Builder, Ptr: AlignAddr, ValTy: SecTy, Adjust: Index * ScLen);
1324 Value *Predicate =
1325 MakePred ? makeTestIfUnaligned(Builder, AlignVal, Alignment: ScLen) : nullptr;
1326
1327 // If vector shifting is potentially needed, accumulate metadata
1328 // from source sections of twice the load width.
1329 int Start = (Index - DoAlign) * ScLen;
1330 int Width = (1 + DoAlign) * ScLen;
1331 return this->createLoad(Builder, ValTy: SecTy, Ptr, Predicate, Alignment: ScLen, Mask: True, PassThru: Undef,
1332 MDSources: VSpan.section(Start, Length: Width).values());
1333 };
1334
1335 auto moveBefore = [this](BasicBlock::iterator In, BasicBlock::iterator To) {
1336 // Move In and its upward dependencies to before To.
1337 assert(In->getParent() == To->getParent());
1338 DepList Deps = getUpwardDeps(In: &*In, Base: &*To);
1339 In->moveBefore(InsertPos: To);
1340 // DepList is sorted with respect to positions in the basic block.
1341 InstMap Map = cloneBefore(To: In, Insts&: Deps);
1342 for (auto [Old, New] : Map)
1343 In->replaceUsesOfWith(From: Old, To: New);
1344 };
1345
1346 // Generate necessary loads at appropriate locations.
1347 LLVM_DEBUG(dbgs() << "Creating loads for ASpan sectors\n");
1348 for (int Index = 0; Index != NumSectors + 1; ++Index) {
1349 // In ASpan, each block will be either a single aligned load, or a
1350 // valign of a pair of loads. In the latter case, an aligned load j
1351 // will belong to the current valign, and the one in the previous
1352 // block (for j > 0).
1353 // Place the load at a location which will dominate the valign, assuming
1354 // the valign will be placed right before the earliest user.
1355 Instruction *PrevAt =
1356 DoAlign && Index > 0 ? EarliestUser[&ASpan[Index - 1]] : nullptr;
1357 Instruction *ThisAt =
1358 Index < NumSectors ? EarliestUser[&ASpan[Index]] : nullptr;
1359 if (auto *Where = std::min(a: PrevAt, b: ThisAt, comp: isEarlier)) {
1360 Builder.SetInsertPoint(Where);
1361 Loads[Index] =
1362 createLoad(Builder, VSpan, Index, DoAlign && Index == NumSectors);
1363 // We know it's safe to put the load at BasePos, but we'd prefer to put
1364 // it at "Where". To see if the load is safe to be placed at Where, put
1365 // it there first and then check if it's safe to move it to BasePos.
1366 // If not, then the load needs to be placed at BasePos.
1367 // We can't do this check proactively because we need the load to exist
1368 // in order to check legality.
1369 if (auto *Load = dyn_cast<Instruction>(Val: Loads[Index])) {
1370 if (!HVC.isSafeToMoveBeforeInBB(In: *Load, To: BasePos))
1371 moveBefore(Load->getIterator(), BasePos);
1372 }
1373 LLVM_DEBUG(dbgs() << "Loads[" << Index << "]:" << *Loads[Index] << '\n');
1374 }
1375 }
1376
1377 // Generate valigns if needed, and fill in proper values in ASpan
1378 LLVM_DEBUG(dbgs() << "Creating values for ASpan sectors\n");
1379 for (int Index = 0; Index != NumSectors; ++Index) {
1380 ASpan[Index].Seg.Val = nullptr;
1381 if (auto *Where = EarliestUser[&ASpan[Index]]) {
1382 Builder.SetInsertPoint(Where);
1383 Value *Val = Loads[Index];
1384 assert(Val != nullptr);
1385 if (DoAlign) {
1386 Value *NextLoad = Loads[Index + 1];
1387 assert(NextLoad != nullptr);
1388 Val = HVC.vralignb(Builder, Lo: Val, Hi: NextLoad, Amt: AlignVal);
1389 }
1390 ASpan[Index].Seg.Val = Val;
1391 LLVM_DEBUG(dbgs() << "ASpan[" << Index << "]:" << *Val << '\n');
1392 }
1393 }
1394
1395 for (const ByteSpan::Block &B : VSpan) {
1396 ByteSpan ASection = ASpan.section(Start: B.Pos, Length: B.Seg.Size).shift(Offset: -B.Pos);
1397 Value *Accum = UndefValue::get(T: HVC.getByteTy(ElemCount: B.Seg.Size));
1398 Builder.SetInsertPoint(cast<Instruction>(Val: B.Seg.Val));
1399
1400 // We're generating a reduction, where each instruction depends on
1401 // the previous one, so we need to order them according to the position
1402 // of their inputs in the code.
1403 std::vector<ByteSpan::Block *> ABlocks;
1404 for (ByteSpan::Block &S : ASection) {
1405 if (S.Seg.Val != nullptr)
1406 ABlocks.push_back(x: &S);
1407 }
1408 llvm::sort(C&: ABlocks,
1409 Comp: [&](const ByteSpan::Block *A, const ByteSpan::Block *B) {
1410 return isEarlier(cast<Instruction>(Val: A->Seg.Val),
1411 cast<Instruction>(Val: B->Seg.Val));
1412 });
1413 for (ByteSpan::Block *S : ABlocks) {
1414 // The processing of the data loaded by the aligned loads
1415 // needs to be inserted after the data is available.
1416 Instruction *SegI = cast<Instruction>(Val: S->Seg.Val);
1417 Builder.SetInsertPoint(&*std::next(x: SegI->getIterator()));
1418 Value *Pay = HVC.vbytes(Builder, Val: getPayload(Val: S->Seg.Val));
1419 Accum =
1420 HVC.insertb(Builder, Dest: Accum, Src: Pay, Start: S->Seg.Start, Length: S->Seg.Size, Where: S->Pos);
1421 }
1422 // Instead of casting everything to bytes for the vselect, cast to the
1423 // original value type. This will avoid complications with casting masks.
1424 // For example, in cases when the original mask applied to i32, it could
1425 // be converted to a mask applicable to i8 via pred_typecast intrinsic,
1426 // but if the mask is not exactly of HVX length, extra handling would be
1427 // needed to make it work.
1428 Type *ValTy = getPayload(Val: B.Seg.Val)->getType();
1429 Value *Cast = Builder.CreateBitCast(V: Accum, DestTy: ValTy, Name: "cst");
1430 Value *Sel = Builder.CreateSelect(C: getMask(Val: B.Seg.Val), True: Cast,
1431 False: getPassThrough(Val: B.Seg.Val), Name: "sel");
1432 B.Seg.Val->replaceAllUsesWith(V: Sel);
1433 }
1434}
1435
1436auto AlignVectors::realignStoreGroup(IRBuilderBase &Builder,
1437 const ByteSpan &VSpan, int ScLen,
1438 Value *AlignVal, Value *AlignAddr) const
1439 -> void {
1440 LLVM_DEBUG(dbgs() << __func__ << "\n");
1441
1442 Type *SecTy = HVC.getByteTy(ElemCount: ScLen);
1443 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1444 bool DoAlign = !HVC.isZero(Val: AlignVal);
1445
1446 // Stores.
1447 ByteSpan ASpanV, ASpanM;
1448
1449 // Return a vector value corresponding to the input value Val:
1450 // either <1 x Val> for scalar Val, or Val itself for vector Val.
1451 auto MakeVec = [](IRBuilderBase &Builder, Value *Val) -> Value * {
1452 Type *Ty = Val->getType();
1453 if (Ty->isVectorTy())
1454 return Val;
1455 auto *VecTy = VectorType::get(ElementType: Ty, NumElements: 1, /*Scalable=*/false);
1456 return Builder.CreateBitCast(V: Val, DestTy: VecTy, Name: "cst");
1457 };
1458
1459 // Create an extra "undef" sector at the beginning and at the end.
1460 // They will be used as the left/right filler in the vlalign step.
1461 for (int Index = (DoAlign ? -1 : 0); Index != NumSectors + DoAlign; ++Index) {
1462 // For stores, the size of each section is an aligned vector length.
1463 // Adjust the store offsets relative to the section start offset.
1464 ByteSpan VSection =
1465 VSpan.section(Start: Index * ScLen, Length: ScLen).shift(Offset: -Index * ScLen);
1466 Value *Undef = UndefValue::get(T: SecTy);
1467 Value *Zero = Constant::getNullValue(Ty: SecTy);
1468 Value *AccumV = Undef;
1469 Value *AccumM = Zero;
1470 for (ByteSpan::Block &S : VSection) {
1471 Value *Pay = getPayload(Val: S.Seg.Val);
1472 Value *Mask = HVC.rescale(Builder, Mask: MakeVec(Builder, getMask(Val: S.Seg.Val)),
1473 FromTy: Pay->getType(), ToTy: HVC.getByteTy());
1474 Value *PartM = HVC.insertb(Builder, Dest: Zero, Src: HVC.vbytes(Builder, Val: Mask),
1475 Start: S.Seg.Start, Length: S.Seg.Size, Where: S.Pos);
1476 AccumM = Builder.CreateOr(LHS: AccumM, RHS: PartM);
1477
1478 Value *PartV = HVC.insertb(Builder, Dest: Undef, Src: HVC.vbytes(Builder, Val: Pay),
1479 Start: S.Seg.Start, Length: S.Seg.Size, Where: S.Pos);
1480
1481 AccumV = Builder.CreateSelect(
1482 C: Builder.CreateICmp(P: CmpInst::ICMP_NE, LHS: PartM, RHS: Zero), True: PartV, False: AccumV);
1483 }
1484 ASpanV.Blocks.emplace_back(args&: AccumV, args&: ScLen, args: Index * ScLen);
1485 ASpanM.Blocks.emplace_back(args&: AccumM, args&: ScLen, args: Index * ScLen);
1486 }
1487
1488 LLVM_DEBUG({
1489 dbgs() << "ASpanV before vlalign:\n" << ASpanV << '\n';
1490 dbgs() << "ASpanM before vlalign:\n" << ASpanM << '\n';
1491 });
1492
1493 // vlalign
1494 if (DoAlign) {
1495 for (int Index = 1; Index != NumSectors + 2; ++Index) {
1496 Value *PrevV = ASpanV[Index - 1].Seg.Val, *ThisV = ASpanV[Index].Seg.Val;
1497 Value *PrevM = ASpanM[Index - 1].Seg.Val, *ThisM = ASpanM[Index].Seg.Val;
1498 assert(isSectorTy(PrevV->getType()) && isSectorTy(PrevM->getType()));
1499 ASpanV[Index - 1].Seg.Val = HVC.vlalignb(Builder, Lo: PrevV, Hi: ThisV, Amt: AlignVal);
1500 ASpanM[Index - 1].Seg.Val = HVC.vlalignb(Builder, Lo: PrevM, Hi: ThisM, Amt: AlignVal);
1501 }
1502 }
1503
1504 LLVM_DEBUG({
1505 dbgs() << "ASpanV after vlalign:\n" << ASpanV << '\n';
1506 dbgs() << "ASpanM after vlalign:\n" << ASpanM << '\n';
1507 });
1508
1509 auto createStore = [&](IRBuilderBase &Builder, const ByteSpan &ASpanV,
1510 const ByteSpan &ASpanM, int Index, bool MakePred) {
1511 Value *Val = ASpanV[Index].Seg.Val;
1512 Value *Mask = ASpanM[Index].Seg.Val; // bytes
1513 if (HVC.isUndef(Val) || HVC.isZero(Val: Mask))
1514 return;
1515 Value *Ptr =
1516 createAdjustedPointer(Builder, Ptr: AlignAddr, ValTy: SecTy, Adjust: Index * ScLen);
1517 Value *Predicate =
1518 MakePred ? makeTestIfUnaligned(Builder, AlignVal, Alignment: ScLen) : nullptr;
1519
1520 // If vector shifting is potentially needed, accumulate metadata
1521 // from source sections of twice the store width.
1522 int Start = (Index - DoAlign) * ScLen;
1523 int Width = (1 + DoAlign) * ScLen;
1524 this->createStore(Builder, Val, Ptr, Predicate, Alignment: ScLen,
1525 Mask: HVC.vlsb(Builder, Val: Mask),
1526 MDSources: VSpan.section(Start, Length: Width).values());
1527 };
1528
1529 for (int Index = 0; Index != NumSectors + DoAlign; ++Index) {
1530 createStore(Builder, ASpanV, ASpanM, Index, DoAlign && Index == NumSectors);
1531 }
1532}
1533
1534auto AlignVectors::realignGroup(const MoveGroup &Move) -> bool {
1535 LLVM_DEBUG(dbgs() << "Realigning group:\n" << Move << '\n');
1536
1537 // TODO: Needs support for masked loads/stores of "scalar" vectors.
1538 if (!Move.IsHvx)
1539 return false;
1540
1541 // Return the element with the maximum alignment from Range,
1542 // where GetValue obtains the value to compare from an element.
1543 auto getMaxOf = [](auto Range, auto GetValue) {
1544 return *llvm::max_element(Range, [&GetValue](auto &A, auto &B) {
1545 return GetValue(A) < GetValue(B);
1546 });
1547 };
1548
1549 AddrList &BaseInfos = AddrGroups[Move.Base];
1550
1551 // Conceptually, there is a vector of N bytes covering the addresses
1552 // starting from the minimum offset (i.e. Base.Addr+Start). This vector
1553 // represents a contiguous memory region that spans all accessed memory
1554 // locations.
1555 // The correspondence between loaded or stored values will be expressed
1556 // in terms of this vector. For example, the 0th element of the vector
1557 // from the Base address info will start at byte Start from the beginning
1558 // of this conceptual vector.
1559 //
1560 // This vector will be loaded/stored starting at the nearest down-aligned
1561 // address and the amount of the down-alignment will be AlignVal:
1562 // valign(load_vector(align_down(Base+Start)), AlignVal)
1563
1564 std::set<Instruction *> TestSet(Move.Main.begin(), Move.Main.end());
1565 AddrList MoveInfos;
1566
1567 llvm::copy_if(
1568 Range&: BaseInfos, Out: std::back_inserter(x&: MoveInfos),
1569 P: [&TestSet](const AddrInfo &AI) { return TestSet.count(x: AI.Inst); });
1570
1571 // Maximum alignment present in the whole address group.
1572 const AddrInfo &WithMaxAlign =
1573 getMaxOf(MoveInfos, [](const AddrInfo &AI) { return AI.HaveAlign; });
1574 Align MaxGiven = WithMaxAlign.HaveAlign;
1575
1576 // Minimum alignment present in the move address group.
1577 const AddrInfo &WithMinOffset =
1578 getMaxOf(MoveInfos, [](const AddrInfo &AI) { return -AI.Offset; });
1579
1580 const AddrInfo &WithMaxNeeded =
1581 getMaxOf(MoveInfos, [](const AddrInfo &AI) { return AI.NeedAlign; });
1582 Align MinNeeded = WithMaxNeeded.NeedAlign;
1583
1584 // Set the builder's insertion point right before the load group, or
1585 // immediately after the store group. (Instructions in a store group are
1586 // listed in reverse order.)
1587 Instruction *InsertAt = Move.Main.front();
1588 if (!Move.IsLoad) {
1589 // There should be a terminator (which store isn't, but check anyways).
1590 assert(InsertAt->getIterator() != InsertAt->getParent()->end());
1591 InsertAt = &*std::next(x: InsertAt->getIterator());
1592 }
1593
1594 IRBuilder Builder(InsertAt->getIterator(), InstSimplifyFolder(HVC.DL));
1595 Value *AlignAddr = nullptr; // Actual aligned address.
1596 Value *AlignVal = nullptr; // Right-shift amount (for valign).
1597
1598 if (MinNeeded <= MaxGiven) {
1599 int Start = WithMinOffset.Offset;
1600 int OffAtMax = WithMaxAlign.Offset;
1601 // Shift the offset of the maximally aligned instruction (OffAtMax)
1602 // back by just enough multiples of the required alignment to cover the
1603 // distance from Start to OffAtMax.
1604 // Calculate the address adjustment amount based on the address with the
1605 // maximum alignment. This is to allow a simple gep instruction instead
1606 // of potential bitcasts to i8*.
1607 int Adjust = -alignTo(Value: OffAtMax - Start, Align: MinNeeded.value());
1608 AlignAddr = createAdjustedPointer(Builder, Ptr: WithMaxAlign.Addr,
1609 ValTy: WithMaxAlign.ValTy, Adjust, CloneMap: Move.Clones);
1610 int Diff = Start - (OffAtMax + Adjust);
1611 AlignVal = HVC.getConstInt(Val: Diff);
1612 assert(Diff >= 0);
1613 assert(static_cast<decltype(MinNeeded.value())>(Diff) < MinNeeded.value());
1614 } else {
1615 // WithMinOffset is the lowest address in the group,
1616 // WithMinOffset.Addr = Base+Start.
1617 // Align instructions for both HVX (V6_valign) and scalar (S2_valignrb)
1618 // mask off unnecessary bits, so it's ok to just the original pointer as
1619 // the alignment amount.
1620 // Do an explicit down-alignment of the address to avoid creating an
1621 // aligned instruction with an address that is not really aligned.
1622 AlignAddr =
1623 createAlignedPointer(Builder, Ptr: WithMinOffset.Addr, ValTy: WithMinOffset.ValTy,
1624 Alignment: MinNeeded.value(), CloneMap: Move.Clones);
1625 AlignVal =
1626 Builder.CreatePtrToInt(V: WithMinOffset.Addr, DestTy: HVC.getIntTy(), Name: "pti");
1627 if (auto *I = dyn_cast<Instruction>(Val: AlignVal)) {
1628 for (auto [Old, New] : Move.Clones)
1629 I->replaceUsesOfWith(From: Old, To: New);
1630 }
1631 }
1632
1633 ByteSpan VSpan;
1634 for (const AddrInfo &AI : MoveInfos) {
1635 VSpan.Blocks.emplace_back(args: AI.Inst, args: HVC.getSizeOf(Ty: AI.ValTy),
1636 args: AI.Offset - WithMinOffset.Offset);
1637 }
1638
1639 // The aligned loads/stores will use blocks that are either scalars,
1640 // or HVX vectors. Let "sector" be the unified term for such a block.
1641 // blend(scalar, vector) -> sector...
1642 int ScLen = Move.IsHvx ? HVC.HST.getVectorLength()
1643 : std::max<int>(a: MinNeeded.value(), b: 4);
1644 assert(!Move.IsHvx || ScLen == 64 || ScLen == 128);
1645 assert(Move.IsHvx || ScLen == 4 || ScLen == 8);
1646
1647 LLVM_DEBUG({
1648 dbgs() << "ScLen: " << ScLen << "\n";
1649 dbgs() << "AlignVal:" << *AlignVal << "\n";
1650 dbgs() << "AlignAddr:" << *AlignAddr << "\n";
1651 dbgs() << "VSpan:\n" << VSpan << '\n';
1652 });
1653
1654 if (Move.IsLoad)
1655 realignLoadGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1656 else
1657 realignStoreGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1658
1659 Instruction *Front = Move.Main.front();
1660 HVC.ORE.emit(RemarkBuilder: [&]() {
1661 return OptimizationRemark(DEBUG_TYPE, "VectorsAligned",
1662 Front->getDebugLoc(), Front->getParent())
1663 << "aligned vector memory operations";
1664 });
1665
1666 for (auto *Inst : Move.Main)
1667 Inst->eraseFromParent();
1668
1669 return true;
1670}
1671
1672auto AlignVectors::makeTestIfUnaligned(IRBuilderBase &Builder, Value *AlignVal,
1673 int Alignment) const -> Value * {
1674 auto *AlignTy = AlignVal->getType();
1675 Value *And = Builder.CreateAnd(
1676 LHS: AlignVal, RHS: ConstantInt::get(Ty: AlignTy, V: Alignment - 1), Name: "and");
1677 Value *Zero = ConstantInt::get(Ty: AlignTy, V: 0);
1678 return Builder.CreateICmpNE(LHS: And, RHS: Zero, Name: "isz");
1679}
1680
1681auto AlignVectors::isSectorTy(Type *Ty) const -> bool {
1682 if (!HVC.isByteVecTy(Ty))
1683 return false;
1684 int Size = HVC.getSizeOf(Ty);
1685 if (HVC.HST.isTypeForHVX(VecTy: Ty))
1686 return Size == static_cast<int>(HVC.HST.getVectorLength());
1687 return Size == 4 || Size == 8;
1688}
1689
1690auto AlignVectors::run() -> bool {
1691 LLVM_DEBUG(dbgs() << "\nRunning HVC::AlignVectors on " << HVC.F.getName()
1692 << '\n');
1693 if (!createAddressGroups())
1694 return false;
1695
1696 LLVM_DEBUG({
1697 dbgs() << "Address groups(" << AddrGroups.size() << "):\n";
1698 for (auto &[In, AL] : AddrGroups) {
1699 for (const AddrInfo &AI : AL)
1700 dbgs() << "---\n" << AI << '\n';
1701 }
1702 });
1703
1704 bool Changed = false;
1705 MoveList LoadGroups, StoreGroups;
1706
1707 for (auto &G : AddrGroups) {
1708 llvm::append_range(C&: LoadGroups, R: createLoadGroups(Group: G.second));
1709 llvm::append_range(C&: StoreGroups, R: createStoreGroups(Group: G.second));
1710 }
1711
1712 LLVM_DEBUG({
1713 dbgs() << "\nLoad groups(" << LoadGroups.size() << "):\n";
1714 for (const MoveGroup &G : LoadGroups)
1715 dbgs() << G << "\n";
1716 dbgs() << "Store groups(" << StoreGroups.size() << "):\n";
1717 for (const MoveGroup &G : StoreGroups)
1718 dbgs() << G << "\n";
1719 });
1720
1721 // Cumulative limit on the number of groups.
1722 unsigned CountLimit = VAGroupCountLimit;
1723 if (CountLimit == 0)
1724 return false;
1725
1726 if (LoadGroups.size() > CountLimit) {
1727 LoadGroups.resize(new_size: CountLimit);
1728 StoreGroups.clear();
1729 } else {
1730 unsigned StoreLimit = CountLimit - LoadGroups.size();
1731 if (StoreGroups.size() > StoreLimit)
1732 StoreGroups.resize(new_size: StoreLimit);
1733 }
1734
1735 for (auto &M : LoadGroups)
1736 Changed |= moveTogether(Move&: M);
1737 for (auto &M : StoreGroups)
1738 Changed |= moveTogether(Move&: M);
1739
1740 LLVM_DEBUG(dbgs() << "After moveTogether:\n" << HVC.F);
1741
1742 for (auto &M : LoadGroups)
1743 Changed |= realignGroup(Move: M);
1744 for (auto &M : StoreGroups)
1745 Changed |= realignGroup(Move: M);
1746
1747 return Changed;
1748}
1749
1750// --- End AlignVectors
1751
1752// --- Begin HvxIdioms
1753
1754auto HvxIdioms::getNumSignificantBits(Value *V, Instruction *In) const
1755 -> std::pair<unsigned, Signedness> {
1756 unsigned Bits = HVC.getNumSignificantBits(V, CtxI: In);
1757 // The significant bits are calculated including the sign bit. This may
1758 // add an extra bit for zero-extended values, e.g. (zext i32 to i64) may
1759 // result in 33 significant bits. To avoid extra words, skip the extra
1760 // sign bit, but keep information that the value is to be treated as
1761 // unsigned.
1762 KnownBits Known = HVC.getKnownBits(V, CtxI: In);
1763 Signedness Sign = Signed;
1764 unsigned NumToTest = 0; // Number of bits used in test for unsignedness.
1765 if (isPowerOf2_32(Value: Bits))
1766 NumToTest = Bits;
1767 else if (Bits > 1 && isPowerOf2_32(Value: Bits - 1))
1768 NumToTest = Bits - 1;
1769
1770 if (NumToTest != 0 && Known.Zero.ashr(ShiftAmt: NumToTest).isAllOnes()) {
1771 Sign = Unsigned;
1772 Bits = NumToTest;
1773 }
1774
1775 // If the top bit of the nearest power-of-2 is zero, this value is
1776 // positive. It could be treated as either signed or unsigned.
1777 if (unsigned Pow2 = PowerOf2Ceil(A: Bits); Pow2 != Bits) {
1778 if (Known.Zero.ashr(ShiftAmt: Pow2 - 1).isAllOnes())
1779 Sign = Positive;
1780 }
1781 return {Bits, Sign};
1782}
1783
1784auto HvxIdioms::canonSgn(SValue X, SValue Y) const
1785 -> std::pair<SValue, SValue> {
1786 // Canonicalize the signedness of X and Y, so that the result is one of:
1787 // S, S
1788 // U/P, S
1789 // U/P, U/P
1790 if (X.Sgn == Signed && Y.Sgn != Signed)
1791 std::swap(a&: X, b&: Y);
1792 return {X, Y};
1793}
1794
1795// Match
1796// (X * Y) [>> N], or
1797// ((X * Y) + (1 << M)) >> N
1798auto HvxIdioms::matchFxpMul(Instruction &In) const -> std::optional<FxpOp> {
1799 using namespace PatternMatch;
1800 auto *Ty = In.getType();
1801
1802 if (!Ty->isVectorTy() || !Ty->getScalarType()->isIntegerTy())
1803 return std::nullopt;
1804
1805 unsigned Width = cast<IntegerType>(Val: Ty->getScalarType())->getBitWidth();
1806
1807 FxpOp Op;
1808 Value *Exp = &In;
1809
1810 // Fixed-point multiplication is always shifted right (except when the
1811 // fraction is 0 bits).
1812 auto m_Shr = [](auto &&V, auto &&S) {
1813 return m_CombineOr(m_LShr(V, S), m_AShr(V, S));
1814 };
1815
1816 uint64_t Qn = 0;
1817 if (Value *T; match(V: Exp, P: m_Shr(m_Value(V&: T), m_ConstantInt(V&: Qn)))) {
1818 Op.Frac = Qn;
1819 Exp = T;
1820 } else {
1821 Op.Frac = 0;
1822 }
1823
1824 if (Op.Frac > Width)
1825 return std::nullopt;
1826
1827 // Check if there is rounding added.
1828 uint64_t CV;
1829 if (Value *T;
1830 Op.Frac > 0 && match(V: Exp, P: m_Add(L: m_Value(V&: T), R: m_ConstantInt(V&: CV)))) {
1831 if (CV != 0 && !isPowerOf2_64(Value: CV))
1832 return std::nullopt;
1833 if (CV != 0)
1834 Op.RoundAt = Log2_64(Value: CV);
1835 Exp = T;
1836 }
1837
1838 // Check if the rest is a multiplication.
1839 if (match(V: Exp, P: m_Mul(L: m_Value(V&: Op.X.Val), R: m_Value(V&: Op.Y.Val)))) {
1840 Op.Opcode = Instruction::Mul;
1841 // FIXME: The information below is recomputed.
1842 Op.X.Sgn = getNumSignificantBits(V: Op.X.Val, In: &In).second;
1843 Op.Y.Sgn = getNumSignificantBits(V: Op.Y.Val, In: &In).second;
1844 Op.ResTy = cast<VectorType>(Val: Ty);
1845 return Op;
1846 }
1847
1848 return std::nullopt;
1849}
1850
1851auto HvxIdioms::processFxpMul(Instruction &In, const FxpOp &Op) const
1852 -> Value * {
1853 assert(Op.X.Val->getType() == Op.Y.Val->getType());
1854
1855 auto *VecTy = dyn_cast<VectorType>(Val: Op.X.Val->getType());
1856 if (VecTy == nullptr)
1857 return nullptr;
1858 auto *ElemTy = cast<IntegerType>(Val: VecTy->getElementType());
1859 unsigned ElemWidth = ElemTy->getBitWidth();
1860
1861 // TODO: This can be relaxed after legalization is done pre-isel.
1862 if ((HVC.length(Ty: VecTy) * ElemWidth) % (8 * HVC.HST.getVectorLength()) != 0)
1863 return nullptr;
1864
1865 // There are no special intrinsics that should be used for multiplying
1866 // signed 8-bit values, so just skip them. Normal codegen should handle
1867 // this just fine.
1868 if (ElemWidth <= 8)
1869 return nullptr;
1870 // Similarly, if this is just a multiplication that can be handled without
1871 // intervention, then leave it alone.
1872 if (ElemWidth <= 32 && Op.Frac == 0)
1873 return nullptr;
1874
1875 auto [BitsX, SignX] = getNumSignificantBits(V: Op.X.Val, In: &In);
1876 auto [BitsY, SignY] = getNumSignificantBits(V: Op.Y.Val, In: &In);
1877
1878 // TODO: Add multiplication of vectors by scalar registers (up to 4 bytes).
1879
1880 Value *X = Op.X.Val, *Y = Op.Y.Val;
1881 IRBuilder Builder(In.getIterator(), InstSimplifyFolder(HVC.DL));
1882
1883 auto roundUpWidth = [](unsigned Width) -> unsigned {
1884 if (Width <= 32 && !isPowerOf2_32(Value: Width)) {
1885 // If the element width is not a power of 2, round it up
1886 // to the next one. Do this for widths not exceeding 32.
1887 return PowerOf2Ceil(A: Width);
1888 }
1889 if (Width > 32 && Width % 32 != 0) {
1890 // For wider elements, round it up to the multiple of 32.
1891 return alignTo(Value: Width, Align: 32u);
1892 }
1893 return Width;
1894 };
1895
1896 BitsX = roundUpWidth(BitsX);
1897 BitsY = roundUpWidth(BitsY);
1898
1899 // For elementwise multiplication vectors must have the same lengths, so
1900 // resize the elements of both inputs to the same width, the max of the
1901 // calculated significant bits.
1902 unsigned Width = std::max(a: BitsX, b: BitsY);
1903
1904 auto *ResizeTy = VectorType::get(ElementType: HVC.getIntTy(Width), Other: VecTy);
1905 if (Width < ElemWidth) {
1906 X = Builder.CreateTrunc(V: X, DestTy: ResizeTy, Name: "trn");
1907 Y = Builder.CreateTrunc(V: Y, DestTy: ResizeTy, Name: "trn");
1908 } else if (Width > ElemWidth) {
1909 X = SignX == Signed ? Builder.CreateSExt(V: X, DestTy: ResizeTy, Name: "sxt")
1910 : Builder.CreateZExt(V: X, DestTy: ResizeTy, Name: "zxt");
1911 Y = SignY == Signed ? Builder.CreateSExt(V: Y, DestTy: ResizeTy, Name: "sxt")
1912 : Builder.CreateZExt(V: Y, DestTy: ResizeTy, Name: "zxt");
1913 };
1914
1915 assert(X->getType() == Y->getType() && X->getType() == ResizeTy);
1916
1917 unsigned VecLen = HVC.length(Ty: ResizeTy);
1918 unsigned ChopLen = (8 * HVC.HST.getVectorLength()) / std::min(a: Width, b: 32u);
1919
1920 SmallVector<Value *> Results;
1921 FxpOp ChopOp = Op;
1922 ChopOp.ResTy = VectorType::get(ElementType: Op.ResTy->getElementType(), NumElements: ChopLen, Scalable: false);
1923
1924 for (unsigned V = 0; V != VecLen / ChopLen; ++V) {
1925 ChopOp.X.Val = HVC.subvector(Builder, Val: X, Start: V * ChopLen, Length: ChopLen);
1926 ChopOp.Y.Val = HVC.subvector(Builder, Val: Y, Start: V * ChopLen, Length: ChopLen);
1927 Results.push_back(Elt: processFxpMulChopped(Builder, In, Op: ChopOp));
1928 if (Results.back() == nullptr)
1929 break;
1930 }
1931
1932 if (Results.empty() || Results.back() == nullptr)
1933 return nullptr;
1934
1935 Value *Cat = HVC.concat(Builder, Vecs: Results);
1936 Value *Ext = SignX == Signed || SignY == Signed
1937 ? Builder.CreateSExt(V: Cat, DestTy: VecTy, Name: "sxt")
1938 : Builder.CreateZExt(V: Cat, DestTy: VecTy, Name: "zxt");
1939 return Ext;
1940}
1941
1942inline bool HvxIdioms::matchScatter(Instruction &In) const {
1943 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1944 if (!II)
1945 return false;
1946 return (II->getIntrinsicID() == Intrinsic::masked_scatter);
1947}
1948
1949inline bool HvxIdioms::matchGather(Instruction &In) const {
1950 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1951 if (!II)
1952 return false;
1953 return (II->getIntrinsicID() == Intrinsic::masked_gather);
1954}
1955
1956inline bool HvxIdioms::matchMLoad(Instruction &In) const {
1957 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1958 if (!II)
1959 return false;
1960 return (II->getIntrinsicID() == Intrinsic::masked_load);
1961}
1962
1963inline bool HvxIdioms::matchMStore(Instruction &In) const {
1964 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1965 if (!II)
1966 return false;
1967 return (II->getIntrinsicID() == Intrinsic::masked_store);
1968}
1969
1970Instruction *locateDestination(Instruction *In, HvxIdioms::DstQualifier &Qual);
1971
1972// Binary instructions we want to handle as users of gather/scatter.
1973inline bool isArithmetic(unsigned Opc) {
1974 switch (Opc) {
1975 case Instruction::Add:
1976 case Instruction::Sub:
1977 case Instruction::Mul:
1978 case Instruction::And:
1979 case Instruction::Or:
1980 case Instruction::Xor:
1981 case Instruction::AShr:
1982 case Instruction::LShr:
1983 case Instruction::Shl:
1984 case Instruction::UDiv:
1985 return true;
1986 }
1987 return false;
1988}
1989
1990// TODO: Maybe use MemoryLocation for this. See getLocOrNone above.
1991inline Value *getPointer(Value *Ptr) {
1992 assert(Ptr && "Unable to extract pointer");
1993 if (isa<AllocaInst>(Val: Ptr) || isa<Argument>(Val: Ptr) || isa<GlobalValue>(Val: Ptr))
1994 return Ptr;
1995 if (isa<LoadInst>(Val: Ptr) || isa<StoreInst>(Val: Ptr))
1996 return getLoadStorePointerOperand(V: Ptr);
1997 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
1998 if (II->getIntrinsicID() == Intrinsic::masked_store)
1999 return II->getOperand(i_nocapture: 1);
2000 }
2001 return nullptr;
2002}
2003
2004static Instruction *selectDestination(Instruction *In,
2005 HvxIdioms::DstQualifier &Qual) {
2006 Instruction *Destination = nullptr;
2007 if (!In)
2008 return Destination;
2009 if (isa<StoreInst>(Val: In)) {
2010 Destination = In;
2011 Qual = HvxIdioms::LdSt;
2012 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: In)) {
2013 if (II->getIntrinsicID() == Intrinsic::masked_gather) {
2014 Destination = In;
2015 Qual = HvxIdioms::LLVM_Gather;
2016 } else if (II->getIntrinsicID() == Intrinsic::masked_scatter) {
2017 Destination = In;
2018 Qual = HvxIdioms::LLVM_Scatter;
2019 } else if (II->getIntrinsicID() == Intrinsic::masked_store) {
2020 Destination = In;
2021 Qual = HvxIdioms::LdSt;
2022 } else if (II->getIntrinsicID() ==
2023 Intrinsic::hexagon_V6_vgather_vscattermh) {
2024 Destination = In;
2025 Qual = HvxIdioms::HEX_Gather_Scatter;
2026 } else if (II->getIntrinsicID() == Intrinsic::hexagon_V6_vscattermh_128B) {
2027 Destination = In;
2028 Qual = HvxIdioms::HEX_Scatter;
2029 } else if (II->getIntrinsicID() == Intrinsic::hexagon_V6_vgathermh_128B) {
2030 Destination = In;
2031 Qual = HvxIdioms::HEX_Gather;
2032 }
2033 } else if (isa<ZExtInst>(Val: In)) {
2034 return locateDestination(In, Qual);
2035 } else if (isa<CastInst>(Val: In)) {
2036 return locateDestination(In, Qual);
2037 } else if (isa<CallInst>(Val: In)) {
2038 Destination = In;
2039 Qual = HvxIdioms::Call;
2040 } else if (isa<GetElementPtrInst>(Val: In)) {
2041 return locateDestination(In, Qual);
2042 } else if (isArithmetic(Opc: In->getOpcode())) {
2043 Destination = In;
2044 Qual = HvxIdioms::Arithmetic;
2045 } else {
2046 LLVM_DEBUG(dbgs() << "Unhandled destination : " << *In << "\n");
2047 }
2048 return Destination;
2049}
2050
2051// This method attempts to find destination (user) for a given intrinsic.
2052// Given that these are produced only by Ripple, the number of options is
2053// limited. Simplest case is explicit store which in fact is redundant (since
2054// HVX gater creates its own store during packetization). Nevertheless we need
2055// to figure address where we storing. Other cases are more complicated, but
2056// still few.
2057Instruction *locateDestination(Instruction *In, HvxIdioms::DstQualifier &Qual) {
2058 Instruction *Destination = nullptr;
2059 if (!In)
2060 return Destination;
2061 // Get all possible destinations
2062 SmallVector<Instruction *> Users;
2063 // Iterate over the uses of the instruction
2064 for (auto &U : In->uses()) {
2065 if (auto *UI = dyn_cast<Instruction>(Val: U.getUser())) {
2066 Destination = selectDestination(In: UI, Qual);
2067 if (Destination)
2068 Users.push_back(Elt: Destination);
2069 }
2070 }
2071 // Now see which of the users (if any) is a memory destination.
2072 for (auto *I : Users)
2073 if (getPointer(Ptr: I))
2074 return I;
2075 return Destination;
2076}
2077
2078// The two intrinsics we handle here have GEP in a different position.
2079inline GetElementPtrInst *locateGepFromIntrinsic(Instruction *In) {
2080 assert(In && "Bad instruction");
2081 IntrinsicInst *IIn = dyn_cast<IntrinsicInst>(Val: In);
2082 assert((IIn && (IIn->getIntrinsicID() == Intrinsic::masked_gather ||
2083 IIn->getIntrinsicID() == Intrinsic::masked_scatter)) &&
2084 "Not a gather Intrinsic");
2085 GetElementPtrInst *GEPIndex = nullptr;
2086 if (IIn->getIntrinsicID() == Intrinsic::masked_gather)
2087 GEPIndex = dyn_cast<GetElementPtrInst>(Val: IIn->getOperand(i_nocapture: 0));
2088 else
2089 GEPIndex = dyn_cast<GetElementPtrInst>(Val: IIn->getOperand(i_nocapture: 1));
2090 return GEPIndex;
2091}
2092
2093// Given the intrinsic find its GEP argument and extract base address it uses.
2094// The method relies on the way how Ripple typically forms the GEP for
2095// scatter/gather.
2096static Value *locateAddressFromIntrinsic(Instruction *In) {
2097 GetElementPtrInst *GEPIndex = locateGepFromIntrinsic(In);
2098 if (!GEPIndex) {
2099 LLVM_DEBUG(dbgs() << " No GEP in intrinsic\n");
2100 return nullptr;
2101 }
2102 Value *BaseAddress = GEPIndex->getPointerOperand();
2103 auto *IndexLoad = dyn_cast<LoadInst>(Val: BaseAddress);
2104 if (IndexLoad)
2105 return IndexLoad;
2106
2107 auto *IndexZEx = dyn_cast<ZExtInst>(Val: BaseAddress);
2108 if (IndexZEx) {
2109 IndexLoad = dyn_cast<LoadInst>(Val: IndexZEx->getOperand(i_nocapture: 0));
2110 if (IndexLoad)
2111 return IndexLoad;
2112 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: IndexZEx->getOperand(i_nocapture: 0));
2113 if (II && II->getIntrinsicID() == Intrinsic::masked_gather)
2114 return locateAddressFromIntrinsic(In: II);
2115 }
2116 auto *BaseShuffle = dyn_cast<ShuffleVectorInst>(Val: BaseAddress);
2117 if (BaseShuffle) {
2118 IndexLoad = dyn_cast<LoadInst>(Val: BaseShuffle->getOperand(i_nocapture: 0));
2119 if (IndexLoad)
2120 return IndexLoad;
2121 auto *IE = dyn_cast<InsertElementInst>(Val: BaseShuffle->getOperand(i_nocapture: 0));
2122 if (IE) {
2123 auto *Src = IE->getOperand(i_nocapture: 1);
2124 IndexLoad = dyn_cast<LoadInst>(Val: Src);
2125 if (IndexLoad)
2126 return IndexLoad;
2127 auto *Alloca = dyn_cast<AllocaInst>(Val: Src);
2128 if (Alloca)
2129 return Alloca;
2130 if (isa<Argument>(Val: Src)) {
2131 return Src;
2132 }
2133 if (isa<GlobalValue>(Val: Src)) {
2134 return Src;
2135 }
2136 }
2137 }
2138 LLVM_DEBUG(dbgs() << " Unable to locate Address from intrinsic\n");
2139 return nullptr;
2140}
2141
2142static Type *getIndexType(Value *In) {
2143 if (!In)
2144 return nullptr;
2145
2146 if (isa<LoadInst>(Val: In) || isa<StoreInst>(Val: In))
2147 return getLoadStoreType(I: In);
2148
2149 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: In)) {
2150 if (II->getIntrinsicID() == Intrinsic::masked_load)
2151 return II->getType();
2152 if (II->getIntrinsicID() == Intrinsic::masked_store)
2153 return II->getOperand(i_nocapture: 0)->getType();
2154 }
2155 return In->getType();
2156}
2157
2158static Value *locateIndexesFromGEP(Value *In) {
2159 if (!In)
2160 return nullptr;
2161 if (isa<LoadInst>(Val: In))
2162 return In;
2163 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: In)) {
2164 if (II->getIntrinsicID() == Intrinsic::masked_load)
2165 return In;
2166 if (II->getIntrinsicID() == Intrinsic::masked_gather)
2167 return In;
2168 }
2169 if (auto *IndexZEx = dyn_cast<ZExtInst>(Val: In))
2170 return locateIndexesFromGEP(In: IndexZEx->getOperand(i_nocapture: 0));
2171 if (auto *IndexSEx = dyn_cast<SExtInst>(Val: In))
2172 return locateIndexesFromGEP(In: IndexSEx->getOperand(i_nocapture: 0));
2173 if (auto *BaseShuffle = dyn_cast<ShuffleVectorInst>(Val: In))
2174 return locateIndexesFromGEP(In: BaseShuffle->getOperand(i_nocapture: 0));
2175 if (auto *IE = dyn_cast<InsertElementInst>(Val: In))
2176 return locateIndexesFromGEP(In: IE->getOperand(i_nocapture: 1));
2177 if (auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: In))
2178 return cstDataVector;
2179 if (auto *GEPIndex = dyn_cast<GetElementPtrInst>(Val: In))
2180 return GEPIndex->getOperand(i_nocapture: 0);
2181 return nullptr;
2182}
2183
2184// Given the intrinsic find its GEP argument and extract offsetts from the base
2185// address it uses.
2186static Value *locateIndexesFromIntrinsic(Instruction *In) {
2187 GetElementPtrInst *GEPIndex = locateGepFromIntrinsic(In);
2188 if (!GEPIndex) {
2189 LLVM_DEBUG(dbgs() << " No GEP in intrinsic\n");
2190 return nullptr;
2191 }
2192 Value *Indexes = GEPIndex->getOperand(i_nocapture: 1);
2193 if (auto *IndexLoad = locateIndexesFromGEP(In: Indexes))
2194 return IndexLoad;
2195
2196 LLVM_DEBUG(dbgs() << " Unable to locate Index from intrinsic\n");
2197 return nullptr;
2198}
2199
2200// Because of aukward definition of many Hex intrinsics we often have to
2201// reinterprete HVX native <64 x i16> as <32 x i32> which in practice is a NOP
2202// for all use cases, so this only exist to make IR builder happy.
2203inline Value *getReinterpretiveCast_i16_to_i32(const HexagonVectorCombine &HVC,
2204 IRBuilderBase &Builder,
2205 LLVMContext &Ctx, Value *I) {
2206 assert(I && "Unable to reinterprete cast");
2207 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2208 std::vector<unsigned> shuffleMask;
2209 for (unsigned i = 0; i < 64; ++i)
2210 shuffleMask.push_back(x: i);
2211 Constant *Mask = llvm::ConstantDataVector::get(Context&: Ctx, Elts: shuffleMask);
2212 Value *CastShuffle =
2213 Builder.CreateShuffleVector(V1: I, V2: I, Mask, Name: "identity_shuffle");
2214 return Builder.CreateBitCast(V: CastShuffle, DestTy: NT, Name: "cst64_i16_to_32_i32");
2215}
2216
2217// Recast <128 x i8> as <32 x i32>
2218inline Value *getReinterpretiveCast_i8_to_i32(const HexagonVectorCombine &HVC,
2219 IRBuilderBase &Builder,
2220 LLVMContext &Ctx, Value *I) {
2221 assert(I && "Unable to reinterprete cast");
2222 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2223 std::vector<unsigned> shuffleMask;
2224 for (unsigned i = 0; i < 128; ++i)
2225 shuffleMask.push_back(x: i);
2226 Constant *Mask = llvm::ConstantDataVector::get(Context&: Ctx, Elts: shuffleMask);
2227 Value *CastShuffle =
2228 Builder.CreateShuffleVector(V1: I, V2: I, Mask, Name: "identity_shuffle");
2229 return Builder.CreateBitCast(V: CastShuffle, DestTy: NT, Name: "cst128_i8_to_32_i32");
2230}
2231
2232// Create <32 x i32> mask reinterpreted as <128 x i1> with a given pattern
2233inline Value *get_i32_Mask(const HexagonVectorCombine &HVC,
2234 IRBuilderBase &Builder, LLVMContext &Ctx,
2235 unsigned int pattern) {
2236 std::vector<unsigned int> byteMask;
2237 for (unsigned i = 0; i < 32; ++i)
2238 byteMask.push_back(x: pattern);
2239
2240 return Builder.CreateIntrinsic(
2241 RetTy: HVC.getBoolTy(ElemCount: 128), ID: HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vandvrt),
2242 Args: {llvm::ConstantDataVector::get(Context&: Ctx, Elts: byteMask), HVC.getConstInt(Val: ~0)},
2243 FMFSource: nullptr);
2244}
2245
2246Value *HvxIdioms::processVScatter(Instruction &In) const {
2247 auto *InpTy = dyn_cast<VectorType>(Val: In.getOperand(i: 0)->getType());
2248 assert(InpTy && "Cannot handle no vector type for llvm.scatter/gather");
2249 unsigned InpSize = HVC.getSizeOf(Ty: InpTy);
2250 auto *F = In.getFunction();
2251 LLVMContext &Ctx = F->getContext();
2252 auto *ElemTy = dyn_cast<IntegerType>(Val: InpTy->getElementType());
2253 assert(ElemTy && "llvm.scatter needs integer type argument");
2254 unsigned ElemWidth = HVC.DL.getTypeAllocSize(Ty: ElemTy);
2255 LLVM_DEBUG({
2256 unsigned Elements = HVC.length(InpTy);
2257 dbgs() << "\n[Process scatter](" << In << ")\n" << *In.getParent() << "\n";
2258 dbgs() << " Input type(" << *InpTy << ") elements(" << Elements
2259 << ") VecLen(" << InpSize << ") type(" << *ElemTy << ") ElemWidth("
2260 << ElemWidth << ")\n";
2261 });
2262
2263 IRBuilder Builder(In.getIterator(), InstSimplifyFolder(HVC.DL));
2264
2265 auto *ValueToScatter = In.getOperand(i: 0);
2266 LLVM_DEBUG(dbgs() << " ValueToScatter : " << *ValueToScatter << "\n");
2267
2268 if (HVC.HST.getVectorLength() != InpSize) {
2269 LLVM_DEBUG(dbgs() << "Unhandled vector size(" << InpSize
2270 << ") for vscatter\n");
2271 return nullptr;
2272 }
2273
2274 // Base address of indexes.
2275 auto *IndexLoad = locateAddressFromIntrinsic(In: &In);
2276 if (!IndexLoad)
2277 return nullptr;
2278 LLVM_DEBUG(dbgs() << " IndexLoad : " << *IndexLoad << "\n");
2279
2280 // Address of destination. Must be in VTCM.
2281 auto *Ptr = getPointer(Ptr: IndexLoad);
2282 if (!Ptr)
2283 return nullptr;
2284 LLVM_DEBUG(dbgs() << " Ptr : " << *Ptr << "\n");
2285 // Indexes/offsets
2286 auto *Indexes = locateIndexesFromIntrinsic(In: &In);
2287 if (!Indexes)
2288 return nullptr;
2289 LLVM_DEBUG(dbgs() << " Indexes : " << *Indexes << "\n");
2290 Value *CastedDst = Builder.CreateBitOrPointerCast(V: Ptr, DestTy: Type::getInt32Ty(C&: Ctx),
2291 Name: "cst_ptr_to_i32");
2292 LLVM_DEBUG(dbgs() << " CastedDst : " << *CastedDst << "\n");
2293 // Adjust Indexes
2294 auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: Indexes);
2295 Value *CastIndex = nullptr;
2296 if (cstDataVector) {
2297 // Our indexes are represented as a constant. We need it in a reg.
2298 Type *IndexVectorType = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2299 AllocaInst *IndexesAlloca = Builder.CreateAlloca(Ty: IndexVectorType);
2300 [[maybe_unused]] auto *StoreIndexes =
2301 Builder.CreateStore(Val: cstDataVector, Ptr: IndexesAlloca);
2302 LLVM_DEBUG(dbgs() << " StoreIndexes : " << *StoreIndexes << "\n");
2303 CastIndex =
2304 Builder.CreateLoad(Ty: IndexVectorType, Ptr: IndexesAlloca, Name: "reload_index");
2305 } else {
2306 if (ElemWidth == 2)
2307 CastIndex = getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: Indexes);
2308 else
2309 CastIndex = Indexes;
2310 }
2311 LLVM_DEBUG(dbgs() << " Cast index : " << *CastIndex << ")\n");
2312
2313 if (ElemWidth == 1) {
2314 // v128i8 There is no native instruction for this.
2315 // Do this as two Hi/Lo gathers with masking.
2316 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2317 // Extend indexes. We assume that indexes are in 128i8 format - need to
2318 // expand them to Hi/Lo 64i16
2319 Value *CastIndexes = Builder.CreateBitCast(V: CastIndex, DestTy: NT, Name: "cast_to_32i32");
2320 auto V6_vunpack = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vunpackub);
2321 auto *UnpackedIndexes = Builder.CreateIntrinsic(
2322 RetTy: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: true), ID: V6_vunpack, Args: CastIndexes, FMFSource: nullptr);
2323 LLVM_DEBUG(dbgs() << " UnpackedIndexes : " << *UnpackedIndexes << ")\n");
2324
2325 auto V6_hi = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_hi);
2326 auto V6_lo = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_lo);
2327 [[maybe_unused]] Value *IndexHi =
2328 HVC.createHvxIntrinsic(Builder, IntID: V6_hi, RetTy: NT, Args: UnpackedIndexes);
2329 [[maybe_unused]] Value *IndexLo =
2330 HVC.createHvxIntrinsic(Builder, IntID: V6_lo, RetTy: NT, Args: UnpackedIndexes);
2331 LLVM_DEBUG(dbgs() << " UnpackedIndHi : " << *IndexHi << ")\n");
2332 LLVM_DEBUG(dbgs() << " UnpackedIndLo : " << *IndexLo << ")\n");
2333 // Now unpack values to scatter
2334 Value *CastSrc =
2335 getReinterpretiveCast_i8_to_i32(HVC, Builder, Ctx, I: ValueToScatter);
2336 LLVM_DEBUG(dbgs() << " CastSrc : " << *CastSrc << ")\n");
2337 auto *UnpackedValueToScatter = Builder.CreateIntrinsic(
2338 RetTy: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: true), ID: V6_vunpack, Args: CastSrc, FMFSource: nullptr);
2339 LLVM_DEBUG(dbgs() << " UnpackedValToScat: " << *UnpackedValueToScatter
2340 << ")\n");
2341
2342 [[maybe_unused]] Value *UVSHi =
2343 HVC.createHvxIntrinsic(Builder, IntID: V6_hi, RetTy: NT, Args: UnpackedValueToScatter);
2344 [[maybe_unused]] Value *UVSLo =
2345 HVC.createHvxIntrinsic(Builder, IntID: V6_lo, RetTy: NT, Args: UnpackedValueToScatter);
2346 LLVM_DEBUG(dbgs() << " UVSHi : " << *UVSHi << ")\n");
2347 LLVM_DEBUG(dbgs() << " UVSLo : " << *UVSLo << ")\n");
2348
2349 // Create the mask for individual bytes
2350 auto *QByteMask = get_i32_Mask(HVC, Builder, Ctx, pattern: 0x00ff00ff);
2351 LLVM_DEBUG(dbgs() << " QByteMask : " << *QByteMask << "\n");
2352 [[maybe_unused]] auto *ResHi = Builder.CreateIntrinsic(
2353 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermhq_128B,
2354 Args: {QByteMask, CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2355 IndexHi, UVSHi},
2356 FMFSource: nullptr);
2357 LLVM_DEBUG(dbgs() << " ResHi : " << *ResHi << ")\n");
2358 return Builder.CreateIntrinsic(
2359 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermhq_128B,
2360 Args: {QByteMask, CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2361 IndexLo, UVSLo},
2362 FMFSource: nullptr);
2363 } else if (ElemWidth == 2) {
2364 Value *CastSrc =
2365 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: ValueToScatter);
2366 LLVM_DEBUG(dbgs() << " CastSrc : " << *CastSrc << ")\n");
2367 return Builder.CreateIntrinsic(
2368 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermh_128B,
2369 Args: {CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), CastIndex,
2370 CastSrc},
2371 FMFSource: nullptr);
2372 } else if (ElemWidth == 4) {
2373 return Builder.CreateIntrinsic(
2374 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermw_128B,
2375 Args: {CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), CastIndex,
2376 ValueToScatter},
2377 FMFSource: nullptr);
2378 } else {
2379 LLVM_DEBUG(dbgs() << "Unhandled element type for vscatter\n");
2380 return nullptr;
2381 }
2382}
2383
2384Value *HvxIdioms::processVGather(Instruction &In) const {
2385 [[maybe_unused]] auto *InpTy =
2386 dyn_cast<VectorType>(Val: In.getOperand(i: 0)->getType());
2387 assert(InpTy && "Cannot handle no vector type for llvm.gather");
2388 [[maybe_unused]] auto *ElemTy =
2389 dyn_cast<PointerType>(Val: InpTy->getElementType());
2390 assert(ElemTy && "llvm.gather needs vector of ptr argument");
2391 auto *F = In.getFunction();
2392 LLVMContext &Ctx = F->getContext();
2393 LLVM_DEBUG(dbgs() << "\n[Process gather](" << In << ")\n"
2394 << *In.getParent() << "\n");
2395 LLVM_DEBUG(dbgs() << " Input type(" << *InpTy << ") elements("
2396 << HVC.length(InpTy) << ") VecLen(" << HVC.getSizeOf(InpTy)
2397 << ") type(" << *ElemTy << ") Access alignment("
2398 << *In.getOperand(1) << ") AddressSpace("
2399 << ElemTy->getAddressSpace() << ")\n");
2400
2401 // TODO: Handle masking of elements.
2402 assert(dyn_cast<VectorType>(In.getOperand(2)->getType()) &&
2403 "llvm.gather needs vector for mask");
2404 IRBuilder Builder(In.getIterator(), InstSimplifyFolder(HVC.DL));
2405
2406 // See who is using the result. The difference between LLVM and HVX vgather
2407 // Intrinsic makes it impossible to handle all cases with temp storage. Alloca
2408 // in VTCM is not yet supported, so for now we just bail out for those cases.
2409 HvxIdioms::DstQualifier Qual = HvxIdioms::Undefined;
2410 Instruction *Dst = locateDestination(In: &In, Qual);
2411 if (!Dst) {
2412 LLVM_DEBUG(dbgs() << " Unable to locate vgather destination\n");
2413 return nullptr;
2414 }
2415 LLVM_DEBUG(dbgs() << " Destination : " << *Dst << " Qual(" << Qual
2416 << ")\n");
2417
2418 // Address of destination. Must be in VTCM.
2419 auto *Ptr = getPointer(Ptr: Dst);
2420 if (!Ptr) {
2421 LLVM_DEBUG(dbgs() << "Could not locate vgather destination ptr\n");
2422 return nullptr;
2423 }
2424
2425 // Result type. Assume it is a vector type.
2426 auto *DstType = cast<VectorType>(Val: getIndexType(In: Dst));
2427 assert(DstType && "Cannot handle non vector dst type for llvm.gather");
2428
2429 // Base address for sources to be loaded
2430 auto *IndexLoad = locateAddressFromIntrinsic(In: &In);
2431 if (!IndexLoad)
2432 return nullptr;
2433 LLVM_DEBUG(dbgs() << " IndexLoad : " << *IndexLoad << "\n");
2434
2435 // Gather indexes/offsets
2436 auto *Indexes = locateIndexesFromIntrinsic(In: &In);
2437 if (!Indexes)
2438 return nullptr;
2439 LLVM_DEBUG(dbgs() << " Indexes : " << *Indexes << "\n");
2440
2441 Value *Gather = nullptr;
2442 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2443 if (Qual == HvxIdioms::LdSt || Qual == HvxIdioms::Arithmetic) {
2444 // We fully assume the address space is in VTCM. We also assume that all
2445 // pointers in Operand(0) have the same base(!).
2446 // This is the most basic case of all the above.
2447 unsigned OutputSize = HVC.getSizeOf(Ty: DstType);
2448 auto *DstElemTy = cast<IntegerType>(Val: DstType->getElementType());
2449 unsigned ElemWidth = HVC.DL.getTypeAllocSize(Ty: DstElemTy);
2450 LLVM_DEBUG(dbgs() << " Buffer type : " << *Ptr->getType()
2451 << " Address space ("
2452 << Ptr->getType()->getPointerAddressSpace() << ")\n"
2453 << " Result type : " << *DstType
2454 << "\n Size in bytes : " << OutputSize
2455 << " element type(" << *DstElemTy
2456 << ")\n ElemWidth : " << ElemWidth << " bytes\n");
2457
2458 auto *IndexType = cast<VectorType>(Val: getIndexType(In: Indexes));
2459 assert(IndexType && "Cannot handle non vector index type for llvm.gather");
2460 unsigned IndexWidth = HVC.DL.getTypeAllocSize(Ty: IndexType->getElementType());
2461 LLVM_DEBUG(dbgs() << " IndexWidth(" << IndexWidth << ")\n");
2462
2463 // Intrinsic takes i32 instead of pointer so cast.
2464 Value *CastedPtr = Builder.CreateBitOrPointerCast(
2465 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2466 // [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, ...]
2467 // int_hexagon_V6_vgathermh [... , llvm_v16i32_ty]
2468 // int_hexagon_V6_vgathermh_128B [... , llvm_v32i32_ty]
2469 // int_hexagon_V6_vgathermhw [... , llvm_v32i32_ty]
2470 // int_hexagon_V6_vgathermhw_128B [... , llvm_v64i32_ty]
2471 // int_hexagon_V6_vgathermw [... , llvm_v16i32_ty]
2472 // int_hexagon_V6_vgathermw_128B [... , llvm_v32i32_ty]
2473 if (HVC.HST.getVectorLength() == OutputSize) {
2474 if (ElemWidth == 1) {
2475 // v128i8 There is no native instruction for this.
2476 // Do this as two Hi/Lo gathers with masking.
2477 // Unpack indexes. We assume that indexes are in 128i8 format - need to
2478 // expand them to Hi/Lo 64i16
2479 Value *CastIndexes =
2480 Builder.CreateBitCast(V: Indexes, DestTy: NT, Name: "cast_to_32i32");
2481 auto V6_vunpack = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vunpackub);
2482 auto *UnpackedIndexes =
2483 Builder.CreateIntrinsic(RetTy: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: true),
2484 ID: V6_vunpack, Args: CastIndexes, FMFSource: nullptr);
2485 LLVM_DEBUG(dbgs() << " UnpackedIndexes : " << *UnpackedIndexes
2486 << ")\n");
2487
2488 auto V6_hi = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_hi);
2489 auto V6_lo = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_lo);
2490 [[maybe_unused]] Value *IndexHi =
2491 HVC.createHvxIntrinsic(Builder, IntID: V6_hi, RetTy: NT, Args: UnpackedIndexes);
2492 [[maybe_unused]] Value *IndexLo =
2493 HVC.createHvxIntrinsic(Builder, IntID: V6_lo, RetTy: NT, Args: UnpackedIndexes);
2494 LLVM_DEBUG(dbgs() << " UnpackedIndHi : " << *IndexHi << ")\n");
2495 LLVM_DEBUG(dbgs() << " UnpackedIndLo : " << *IndexLo << ")\n");
2496 // Create the mask for individual bytes
2497 auto *QByteMask = get_i32_Mask(HVC, Builder, Ctx, pattern: 0x00ff00ff);
2498 LLVM_DEBUG(dbgs() << " QByteMask : " << *QByteMask << "\n");
2499 // We use our destination allocation as a temp storage
2500 // This is unlikely to work properly for masked gather.
2501 auto V6_vgather = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vgathermhq);
2502 [[maybe_unused]] auto GatherHi = Builder.CreateIntrinsic(
2503 RetTy: Type::getVoidTy(C&: Ctx), ID: V6_vgather,
2504 Args: {Ptr, QByteMask, CastedPtr,
2505 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), IndexHi},
2506 FMFSource: nullptr);
2507 LLVM_DEBUG(dbgs() << " GatherHi : " << *GatherHi << ")\n");
2508 // Rematerialize the result
2509 [[maybe_unused]] Value *LoadedResultHi = Builder.CreateLoad(
2510 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false), Ptr, Name: "temp_result_hi");
2511 LLVM_DEBUG(dbgs() << " LoadedResultHi : " << *LoadedResultHi << "\n");
2512 // Same for the low part. Here we use Gather to return non-NULL result
2513 // from this function and continue to iterate. We also are deleting Dst
2514 // store below.
2515 Gather = Builder.CreateIntrinsic(
2516 RetTy: Type::getVoidTy(C&: Ctx), ID: V6_vgather,
2517 Args: {Ptr, QByteMask, CastedPtr,
2518 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), IndexLo},
2519 FMFSource: nullptr);
2520 LLVM_DEBUG(dbgs() << " GatherLo : " << *Gather << ")\n");
2521 Value *LoadedResultLo = Builder.CreateLoad(
2522 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false), Ptr, Name: "temp_result_lo");
2523 LLVM_DEBUG(dbgs() << " LoadedResultLo : " << *LoadedResultLo << "\n");
2524 // Now we have properly sized bytes in every other position
2525 // B b A a c a A b B c f F g G h H is presented as
2526 // B . b . A . a . c . a . A . b . B . c . f . F . g . G . h . H
2527 // Use vpack to gather them
2528 auto V6_vpackeb = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vpackeb);
2529 [[maybe_unused]] auto Res = Builder.CreateIntrinsic(
2530 RetTy: NT, ID: V6_vpackeb, Args: {LoadedResultHi, LoadedResultLo}, FMFSource: nullptr);
2531 LLVM_DEBUG(dbgs() << " ScaledRes : " << *Res << "\n");
2532 [[maybe_unused]] auto *StoreRes = Builder.CreateStore(Val: Res, Ptr);
2533 LLVM_DEBUG(dbgs() << " StoreRes : " << *StoreRes << "\n");
2534 } else if (ElemWidth == 2) {
2535 // v32i16
2536 if (IndexWidth == 2) {
2537 // Reinterprete 64i16 as 32i32. Only needed for syntactic IR match.
2538 Value *CastIndex =
2539 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: Indexes);
2540 LLVM_DEBUG(dbgs() << " Cast index: " << *CastIndex << ")\n");
2541 // shift all i16 left by 1 to match short addressing mode instead of
2542 // byte.
2543 auto V6_vaslh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaslh);
2544 Value *AdjustedIndex = HVC.createHvxIntrinsic(
2545 Builder, IntID: V6_vaslh, RetTy: NT, Args: {CastIndex, HVC.getConstInt(Val: 1)});
2546 LLVM_DEBUG(dbgs()
2547 << " Shifted half index: " << *AdjustedIndex << ")\n");
2548
2549 auto V6_vgather = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vgathermh);
2550 // The 3rd argument is the size of the region to gather from. Probably
2551 // want to set it to max VTCM size.
2552 Gather = Builder.CreateIntrinsic(
2553 RetTy: Type::getVoidTy(C&: Ctx), ID: V6_vgather,
2554 Args: {Ptr, CastedPtr, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2555 AdjustedIndex},
2556 FMFSource: nullptr);
2557 for (auto &U : Dst->uses()) {
2558 if (auto *UI = dyn_cast<Instruction>(Val: U.getUser()))
2559 dbgs() << " dst used by: " << *UI << "\n";
2560 }
2561 for (auto &U : In.uses()) {
2562 if (auto *UI = dyn_cast<Instruction>(Val: U.getUser()))
2563 dbgs() << " In used by : " << *UI << "\n";
2564 }
2565 // Create temp load from result in case the result is used by any
2566 // other instruction.
2567 Value *LoadedResult = Builder.CreateLoad(
2568 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr, Name: "temp_result");
2569 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2570 In.replaceAllUsesWith(V: LoadedResult);
2571 } else {
2572 dbgs() << " Unhandled index type for vgather\n";
2573 return nullptr;
2574 }
2575 } else if (ElemWidth == 4) {
2576 if (IndexWidth == 4) {
2577 // v32i32
2578 auto V6_vaslh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaslh);
2579 Value *AdjustedIndex = HVC.createHvxIntrinsic(
2580 Builder, IntID: V6_vaslh, RetTy: NT, Args: {Indexes, HVC.getConstInt(Val: 2)});
2581 LLVM_DEBUG(dbgs()
2582 << " Shifted word index: " << *AdjustedIndex << ")\n");
2583 Gather = Builder.CreateIntrinsic(
2584 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermw_128B,
2585 Args: {Ptr, CastedPtr, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2586 AdjustedIndex},
2587 FMFSource: nullptr);
2588 } else {
2589 LLVM_DEBUG(dbgs() << " Unhandled index type for vgather\n");
2590 return nullptr;
2591 }
2592 } else {
2593 LLVM_DEBUG(dbgs() << " Unhandled element type for vgather\n");
2594 return nullptr;
2595 }
2596 } else if (HVC.HST.getVectorLength() == OutputSize * 2) {
2597 // This is half of the reg width, duplicate low in high
2598 LLVM_DEBUG(dbgs() << " Unhandled half of register size\n");
2599 return nullptr;
2600 } else if (HVC.HST.getVectorLength() * 2 == OutputSize) {
2601 LLVM_DEBUG(dbgs() << " Unhandle twice the register size\n");
2602 return nullptr;
2603 }
2604 // Erase the original intrinsic and store that consumes it.
2605 // HVX will create a pseudo for gather that is expanded to gather + store
2606 // during packetization.
2607 Dst->eraseFromParent();
2608 } else if (Qual == HvxIdioms::LLVM_Scatter) {
2609 // Gather feeds directly into scatter.
2610 LLVM_DEBUG({
2611 auto *DstInpTy = cast<VectorType>(Dst->getOperand(1)->getType());
2612 assert(DstInpTy && "Cannot handle no vector type for llvm.scatter");
2613 unsigned DstInpSize = HVC.getSizeOf(DstInpTy);
2614 unsigned DstElements = HVC.length(DstInpTy);
2615 auto *DstElemTy = cast<PointerType>(DstInpTy->getElementType());
2616 assert(DstElemTy && "llvm.scatter needs vector of ptr argument");
2617 dbgs() << " Gather feeds into scatter\n Values to scatter : "
2618 << *Dst->getOperand(0) << "\n";
2619 dbgs() << " Dst type(" << *DstInpTy << ") elements(" << DstElements
2620 << ") VecLen(" << DstInpSize << ") type(" << *DstElemTy
2621 << ") Access alignment(" << *Dst->getOperand(2) << ")\n";
2622 });
2623 // Address of source
2624 auto *Src = getPointer(Ptr: IndexLoad);
2625 if (!Src)
2626 return nullptr;
2627 LLVM_DEBUG(dbgs() << " Src : " << *Src << "\n");
2628
2629 if (!isa<PointerType>(Val: Src->getType())) {
2630 LLVM_DEBUG(dbgs() << " Source is not a pointer type...\n");
2631 return nullptr;
2632 }
2633
2634 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2635 V: Src, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2636 LLVM_DEBUG(dbgs() << " CastedSrc: " << *CastedSrc << "\n");
2637
2638 auto *DstLoad = locateAddressFromIntrinsic(In: Dst);
2639 if (!DstLoad) {
2640 LLVM_DEBUG(dbgs() << " Unable to locate DstLoad\n");
2641 return nullptr;
2642 }
2643 LLVM_DEBUG(dbgs() << " DstLoad : " << *DstLoad << "\n");
2644
2645 Value *Ptr = getPointer(Ptr: DstLoad);
2646 if (!Ptr)
2647 return nullptr;
2648 LLVM_DEBUG(dbgs() << " Ptr : " << *Ptr << "\n");
2649 Value *CastIndex =
2650 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: IndexLoad);
2651 LLVM_DEBUG(dbgs() << " Cast index: " << *CastIndex << ")\n");
2652 // Shift all i16 left by 1 to match short addressing mode instead of
2653 // byte.
2654 auto V6_vaslh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaslh);
2655 Value *AdjustedIndex = HVC.createHvxIntrinsic(
2656 Builder, IntID: V6_vaslh, RetTy: NT, Args: {CastIndex, HVC.getConstInt(Val: 1)});
2657 LLVM_DEBUG(dbgs() << " Shifted half index: " << *AdjustedIndex << ")\n");
2658
2659 return Builder.CreateIntrinsic(
2660 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2661 Args: {Ptr, CastedSrc, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2662 AdjustedIndex},
2663 FMFSource: nullptr);
2664 } else if (Qual == HvxIdioms::HEX_Gather_Scatter) {
2665 // Gather feeds into previously inserted pseudo intrinsic.
2666 // These could not be in the same packet, so we need to generate another
2667 // pseudo that is expanded to .tmp + store V6_vgathermh_pseudo
2668 // V6_vgathermh_pseudo (ins IntRegs:$_dst_, s4_0Imm:$Ii, IntRegs:$Rt,
2669 // ModRegs:$Mu, HvxVR:$Vv)
2670 if (isa<AllocaInst>(Val: IndexLoad)) {
2671 auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: Indexes);
2672 if (cstDataVector) {
2673 // Our indexes are represented as a constant. We need THEM in a reg.
2674 // This most likely will not work properly since alloca gives us DDR
2675 // stack location. This will be fixed once we teach compiler about VTCM.
2676 AllocaInst *IndexesAlloca = Builder.CreateAlloca(Ty: NT);
2677 [[maybe_unused]] auto *StoreIndexes =
2678 Builder.CreateStore(Val: cstDataVector, Ptr: IndexesAlloca);
2679 LLVM_DEBUG(dbgs() << " StoreIndexes : " << *StoreIndexes << "\n");
2680 Value *LoadedIndex =
2681 Builder.CreateLoad(Ty: NT, Ptr: IndexesAlloca, Name: "reload_index");
2682 AllocaInst *ResultAlloca = Builder.CreateAlloca(Ty: NT);
2683 LLVM_DEBUG(dbgs() << " ResultAlloca : " << *ResultAlloca << "\n");
2684
2685 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2686 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2687 LLVM_DEBUG(dbgs() << " CastedSrc : " << *CastedSrc << "\n");
2688
2689 Gather = Builder.CreateIntrinsic(
2690 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2691 Args: {ResultAlloca, CastedSrc,
2692 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), LoadedIndex},
2693 FMFSource: nullptr);
2694 Value *LoadedResult = Builder.CreateLoad(
2695 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr: ResultAlloca, Name: "temp_result");
2696 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2697 LLVM_DEBUG(dbgs() << " Gather : " << *Gather << "\n");
2698 In.replaceAllUsesWith(V: LoadedResult);
2699 }
2700 } else {
2701 // Address of source
2702 auto *Src = getPointer(Ptr: IndexLoad);
2703 if (!Src)
2704 return nullptr;
2705 LLVM_DEBUG(dbgs() << " Src : " << *Src << "\n");
2706
2707 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2708 V: Src, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2709 LLVM_DEBUG(dbgs() << " CastedSrc: " << *CastedSrc << "\n");
2710
2711 auto *DstLoad = locateAddressFromIntrinsic(In: Dst);
2712 if (!DstLoad)
2713 return nullptr;
2714 LLVM_DEBUG(dbgs() << " DstLoad : " << *DstLoad << "\n");
2715 auto *Ptr = getPointer(Ptr: DstLoad);
2716 if (!Ptr)
2717 return nullptr;
2718 LLVM_DEBUG(dbgs() << " Ptr : " << *Ptr << "\n");
2719
2720 Gather = Builder.CreateIntrinsic(
2721 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgather_vscattermh,
2722 Args: {Ptr, CastedSrc, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2723 Indexes},
2724 FMFSource: nullptr);
2725 }
2726 return Gather;
2727 } else if (Qual == HvxIdioms::HEX_Scatter) {
2728 // This is the case when result of a gather is used as an argument to
2729 // Intrinsic::hexagon_V6_vscattermh_128B. Most likely we just inserted it
2730 // ourselves. We have to create alloca, store to it, and replace all uses
2731 // with that.
2732 AllocaInst *ResultAlloca = Builder.CreateAlloca(Ty: NT);
2733 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2734 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2735 LLVM_DEBUG(dbgs() << " CastedSrc : " << *CastedSrc << "\n");
2736 Value *CastIndex =
2737 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: Indexes);
2738 LLVM_DEBUG(dbgs() << " Cast index : " << *CastIndex << ")\n");
2739
2740 Gather = Builder.CreateIntrinsic(
2741 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2742 Args: {ResultAlloca, CastedSrc, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2743 CastIndex},
2744 FMFSource: nullptr);
2745 Value *LoadedResult = Builder.CreateLoad(
2746 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr: ResultAlloca, Name: "temp_result");
2747 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2748 In.replaceAllUsesWith(V: LoadedResult);
2749 } else if (Qual == HvxIdioms::HEX_Gather) {
2750 // Gather feeds to another gather but already replaced with
2751 // hexagon_V6_vgathermh_128B
2752 if (isa<AllocaInst>(Val: IndexLoad)) {
2753 auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: Indexes);
2754 if (cstDataVector) {
2755 // Our indexes are represented as a constant. We need it in a reg.
2756 AllocaInst *IndexesAlloca = Builder.CreateAlloca(Ty: NT);
2757
2758 [[maybe_unused]] auto *StoreIndexes =
2759 Builder.CreateStore(Val: cstDataVector, Ptr: IndexesAlloca);
2760 LLVM_DEBUG(dbgs() << " StoreIndexes : " << *StoreIndexes << "\n");
2761 Value *LoadedIndex =
2762 Builder.CreateLoad(Ty: NT, Ptr: IndexesAlloca, Name: "reload_index");
2763 AllocaInst *ResultAlloca = Builder.CreateAlloca(Ty: NT);
2764 LLVM_DEBUG(dbgs() << " ResultAlloca : " << *ResultAlloca
2765 << "\n AddressSpace: "
2766 << ResultAlloca->getAddressSpace() << "\n";);
2767
2768 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2769 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2770 LLVM_DEBUG(dbgs() << " CastedSrc : " << *CastedSrc << "\n");
2771
2772 Gather = Builder.CreateIntrinsic(
2773 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2774 Args: {ResultAlloca, CastedSrc,
2775 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), LoadedIndex},
2776 FMFSource: nullptr);
2777 Value *LoadedResult = Builder.CreateLoad(
2778 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr: ResultAlloca, Name: "temp_result");
2779 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2780 LLVM_DEBUG(dbgs() << " Gather : " << *Gather << "\n");
2781 In.replaceAllUsesWith(V: LoadedResult);
2782 }
2783 }
2784 } else if (Qual == HvxIdioms::LLVM_Gather) {
2785 // Gather feeds into another gather
2786 errs() << " Underimplemented vgather to vgather sequence\n";
2787 return nullptr;
2788 } else
2789 llvm_unreachable("Unhandled Qual enum");
2790
2791 return Gather;
2792}
2793
2794// Go through all PHI incomming values and find minimal alignment for non GEP
2795// members.
2796std::optional<uint64_t> HvxIdioms::getPHIBaseMinAlignment(Instruction &In,
2797 PHINode *PN) const {
2798 if (!PN)
2799 return std::nullopt;
2800
2801 SmallVector<Value *, 16> Worklist;
2802 SmallPtrSet<Value *, 16> Visited;
2803 uint64_t minPHIAlignment = Value::MaximumAlignment;
2804 Worklist.push_back(Elt: PN);
2805
2806 while (!Worklist.empty()) {
2807 Value *V = Worklist.back();
2808 Worklist.pop_back();
2809 if (!Visited.insert(Ptr: V).second)
2810 continue;
2811
2812 if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
2813 for (unsigned i = 0; i < PN->getNumIncomingValues(); ++i) {
2814 Worklist.push_back(Elt: PN->getIncomingValue(i));
2815 }
2816 } else if (isa<GetElementPtrInst>(Val: V)) {
2817 // Ignore geps for now.
2818 continue;
2819 } else {
2820 Align KnownAlign = getKnownAlignment(V, DL: HVC.DL, CtxI: &In, AC: &HVC.AC, DT: &HVC.DT);
2821 if (KnownAlign.value() < minPHIAlignment)
2822 minPHIAlignment = KnownAlign.value();
2823 }
2824 }
2825 if (minPHIAlignment != Value::MaximumAlignment)
2826 return minPHIAlignment;
2827 return std::nullopt;
2828}
2829
2830// Helper function to discover alignment for a ptr.
2831std::optional<uint64_t> HvxIdioms::getAlignment(Instruction &In,
2832 Value *ptr) const {
2833 SmallPtrSet<Value *, 16> Visited;
2834 return getAlignmentImpl(In, ptr, Visited);
2835}
2836
2837std::optional<uint64_t>
2838HvxIdioms::getAlignmentImpl(Instruction &In, Value *ptr,
2839 SmallPtrSet<Value *, 16> &Visited) const {
2840 LLVM_DEBUG(dbgs() << "[getAlignment] for : " << *ptr << "\n");
2841 // Prevent infinite recursion
2842 if (!Visited.insert(Ptr: ptr).second)
2843 return std::nullopt;
2844 // Try AssumptionCache.
2845 Align KnownAlign = getKnownAlignment(V: ptr, DL: HVC.DL, CtxI: &In, AC: &HVC.AC, DT: &HVC.DT);
2846 // This is the most formal and reliable source of information.
2847 if (KnownAlign.value() > 1) {
2848 LLVM_DEBUG(dbgs() << " VC align(" << KnownAlign.value() << ")\n");
2849 return KnownAlign.value();
2850 }
2851
2852 // If it is a PHI try to iterate through inputs
2853 if (PHINode *PN = dyn_cast<PHINode>(Val: ptr)) {
2854 // See if we have a common base to which we know alignment.
2855 auto baseAlignmentOpt = getPHIBaseMinAlignment(In, PN);
2856 if (!baseAlignmentOpt)
2857 return std::nullopt;
2858
2859 uint64_t minBaseAlignment = *baseAlignmentOpt;
2860 // If it is 1, there is no point to keep on looking.
2861 if (minBaseAlignment == 1)
2862 return 1;
2863 // No see if all other incomming phi nodes are just loop carried constants.
2864 uint64_t minPHIAlignment = minBaseAlignment;
2865 LLVM_DEBUG(dbgs() << " It is a PHI with(" << PN->getNumIncomingValues()
2866 << ")nodes and min base aligned to (" << minBaseAlignment
2867 << ")\n");
2868 for (unsigned i = 0; i < PN->getNumIncomingValues(); ++i) {
2869 Value *IV = PN->getIncomingValue(i);
2870 // We have already looked at all other values.
2871 if (!isa<GetElementPtrInst>(Val: IV))
2872 continue;
2873 uint64_t MemberAlignment = Value::MaximumAlignment;
2874 if (auto res = getAlignment(In&: *PN, ptr: IV))
2875 MemberAlignment = *res;
2876 else
2877 return std::nullopt;
2878 // Adjust total PHI alignment.
2879 if (minPHIAlignment > MemberAlignment)
2880 minPHIAlignment = MemberAlignment;
2881 }
2882 LLVM_DEBUG(dbgs() << " total PHI alignment(" << minPHIAlignment << ")\n");
2883 return minPHIAlignment;
2884 }
2885
2886 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: ptr)) {
2887 auto *GEPPtr = GEP->getPointerOperand();
2888 // Only if this is the induction variable with const offset
2889 // Implicit assumption is that induction variable itself is a PHI
2890 if (&In == GEPPtr) {
2891 APInt Offset(HVC.DL.getPointerSizeInBits(
2892 AS: GEPPtr->getType()->getPointerAddressSpace()),
2893 0);
2894 if (GEP->accumulateConstantOffset(DL: HVC.DL, Offset)) {
2895 LLVM_DEBUG(dbgs() << " Induction GEP with const step of ("
2896 << Offset.getZExtValue() << ")\n");
2897 return Offset.getZExtValue();
2898 }
2899 }
2900 }
2901
2902 return std::nullopt;
2903}
2904
2905Value *HvxIdioms::processMStore(Instruction &In) const {
2906 [[maybe_unused]] auto *InpTy =
2907 dyn_cast<VectorType>(Val: In.getOperand(i: 0)->getType());
2908 assert(InpTy && "Cannot handle no vector type for llvm.masked.store");
2909
2910 LLVM_DEBUG(dbgs() << "\n[Process mstore](" << In << ")\n"
2911 << *In.getParent() << "\n");
2912 LLVM_DEBUG(dbgs() << " Input type(" << *InpTy << ") elements("
2913 << HVC.length(InpTy) << ") VecLen(" << HVC.getSizeOf(InpTy)
2914 << ") type(" << *InpTy->getElementType() << ") of size("
2915 << InpTy->getScalarSizeInBits() << ")bits\n");
2916 auto *CI = dyn_cast<CallBase>(Val: &In);
2917 assert(CI && "Expected llvm.masked.store to be a call");
2918 Align HaveAlign = CI->getParamAlign(ArgNo: 1).valueOrOne();
2919
2920 uint64_t KA = 1;
2921 if (auto res = getAlignment(In, ptr: In.getOperand(i: 1))) // ptr operand
2922 KA = *res;
2923 LLVM_DEBUG(dbgs() << " HaveAlign(" << HaveAlign.value() << ") KnownAlign("
2924 << KA << ")\n");
2925 // Normalize 0 -> ABI alignment of the stored value type (operand 0).
2926 Type *ValTy = In.getOperand(i: 0)->getType();
2927 Align EffA =
2928 (KA > 0) ? Align(KA) : Align(HVC.DL.getABITypeAlign(Ty: ValTy).value());
2929
2930 if (EffA < HaveAlign)
2931 return nullptr;
2932
2933 // Attach/replace the param attribute on pointer param #1.
2934 AttrBuilder AttrB(CI->getContext());
2935 AttrB.addAlignmentAttr(Align: EffA);
2936 CI->setAttributes(
2937 CI->getAttributes().addParamAttributes(C&: CI->getContext(), ArgNo: 1, B: AttrB));
2938 return CI;
2939}
2940
2941Value *HvxIdioms::processMLoad(Instruction &In) const {
2942 [[maybe_unused]] auto *InpTy = dyn_cast<VectorType>(Val: In.getType());
2943 assert(InpTy && "Cannot handle non vector type for llvm.masked.store");
2944 LLVM_DEBUG(dbgs() << "\n[Process mload](" << In << ")\n"
2945 << *In.getParent() << "\n");
2946 LLVM_DEBUG(dbgs() << " Input type(" << *InpTy << ") elements("
2947 << HVC.length(InpTy) << ") VecLen(" << HVC.getSizeOf(InpTy)
2948 << ") type(" << *InpTy->getElementType() << ") of size("
2949 << InpTy->getScalarSizeInBits() << ")bits\n");
2950 auto *CI = dyn_cast<CallBase>(Val: &In);
2951 assert(CI && "Expected to be a call to llvm.masked.load");
2952 // The pointer is operand #0, and its param attribute index is also 0.
2953 Align HaveAlign = CI->getParamAlign(ArgNo: 0).valueOrOne();
2954
2955 // Compute best-known alignment KA from analysis.
2956 uint64_t KA = 1;
2957 if (auto res = getAlignment(In, ptr: In.getOperand(i: 0))) // ptr operand
2958 KA = *res;
2959
2960 // Normalize 0 → ABI alignment of the loaded value type.
2961 Type *ValTy = In.getType();
2962 Align EffA =
2963 (KA > 0) ? Align(KA) : Align(HVC.DL.getABITypeAlign(Ty: ValTy).value());
2964 if (EffA < HaveAlign)
2965 return nullptr;
2966 LLVM_DEBUG(dbgs() << " HaveAlign(" << HaveAlign.value() << ") KnownAlign("
2967 << KA << ")\n");
2968
2969 // Attach/replace the param attribute on pointer param #0.
2970 AttrBuilder AttrB(CI->getContext());
2971 AttrB.addAlignmentAttr(Align: EffA);
2972 CI->setAttributes(
2973 CI->getAttributes().addParamAttributes(C&: CI->getContext(), ArgNo: 0, B: AttrB));
2974 return CI;
2975}
2976
2977auto HvxIdioms::processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
2978 const FxpOp &Op) const -> Value * {
2979 assert(Op.X.Val->getType() == Op.Y.Val->getType());
2980 auto *InpTy = cast<VectorType>(Val: Op.X.Val->getType());
2981 unsigned Width = InpTy->getScalarSizeInBits();
2982 bool Rounding = Op.RoundAt.has_value();
2983
2984 if (!Op.RoundAt || *Op.RoundAt == Op.Frac - 1) {
2985 // The fixed-point intrinsics do signed multiplication.
2986 if (Width == Op.Frac + 1 && Op.X.Sgn != Unsigned && Op.Y.Sgn != Unsigned) {
2987 Value *QMul = nullptr;
2988 if (Width == 16) {
2989 QMul = createMulQ15(Builder, X: Op.X, Y: Op.Y, Rounding);
2990 } else if (Width == 32) {
2991 QMul = createMulQ31(Builder, X: Op.X, Y: Op.Y, Rounding);
2992 }
2993 if (QMul != nullptr)
2994 return QMul;
2995 }
2996 }
2997
2998 assert(Width >= 32 || isPowerOf2_32(Width)); // Width <= 32 => Width is 2^n
2999 assert(Width < 32 || Width % 32 == 0); // Width > 32 => Width is 32*k
3000
3001 // If Width < 32, then it should really be 16.
3002 if (Width < 32) {
3003 if (Width < 16)
3004 return nullptr;
3005 // Getting here with Op.Frac == 0 isn't wrong, but suboptimal: here we
3006 // generate a full precision products, which is unnecessary if there is
3007 // no shift.
3008 assert(Width == 16);
3009 assert(Op.Frac != 0 && "Unshifted mul should have been skipped");
3010 if (Op.Frac == 16) {
3011 // Multiply high
3012 if (Value *MulH = createMulH16(Builder, X: Op.X, Y: Op.Y))
3013 return MulH;
3014 }
3015 // Do full-precision multiply and shift.
3016 Value *Prod32 = createMul16(Builder, X: Op.X, Y: Op.Y);
3017 if (Rounding) {
3018 Value *RoundVal =
3019 ConstantInt::get(Ty: Prod32->getType(), V: 1ull << *Op.RoundAt);
3020 Prod32 = Builder.CreateAdd(LHS: Prod32, RHS: RoundVal, Name: "add");
3021 }
3022
3023 Value *ShiftAmt = ConstantInt::get(Ty: Prod32->getType(), V: Op.Frac);
3024 Value *Shifted = Op.X.Sgn == Signed || Op.Y.Sgn == Signed
3025 ? Builder.CreateAShr(LHS: Prod32, RHS: ShiftAmt, Name: "asr")
3026 : Builder.CreateLShr(LHS: Prod32, RHS: ShiftAmt, Name: "lsr");
3027 return Builder.CreateTrunc(V: Shifted, DestTy: InpTy, Name: "trn");
3028 }
3029
3030 // Width >= 32
3031
3032 // Break up the arguments Op.X and Op.Y into vectors of smaller widths
3033 // in preparation of doing the multiplication by 32-bit parts.
3034 auto WordX = HVC.splitVectorElements(Builder, Vec: Op.X.Val, /*ToWidth=*/32);
3035 auto WordY = HVC.splitVectorElements(Builder, Vec: Op.Y.Val, /*ToWidth=*/32);
3036 auto WordP = createMulLong(Builder, WordX, SgnX: Op.X.Sgn, WordY, SgnY: Op.Y.Sgn);
3037
3038 auto *HvxWordTy = cast<VectorType>(Val: WordP.front()->getType());
3039
3040 // Add the optional rounding to the proper word.
3041 if (Op.RoundAt.has_value()) {
3042 Value *Zero = Constant::getNullValue(Ty: WordX[0]->getType());
3043 SmallVector<Value *> RoundV(WordP.size(), Zero);
3044 RoundV[*Op.RoundAt / 32] =
3045 ConstantInt::get(Ty: HvxWordTy, V: 1ull << (*Op.RoundAt % 32));
3046 WordP = createAddLong(Builder, WordX: WordP, WordY: RoundV);
3047 }
3048
3049 // createRightShiftLong?
3050
3051 // Shift all products right by Op.Frac.
3052 unsigned SkipWords = Op.Frac / 32;
3053 Constant *ShiftAmt = ConstantInt::get(Ty: HvxWordTy, V: Op.Frac % 32);
3054
3055 for (int Dst = 0, End = WordP.size() - SkipWords; Dst != End; ++Dst) {
3056 int Src = Dst + SkipWords;
3057 Value *Lo = WordP[Src];
3058 if (Src + 1 < End) {
3059 Value *Hi = WordP[Src + 1];
3060 WordP[Dst] = Builder.CreateIntrinsic(RetTy: HvxWordTy, ID: Intrinsic::fshr,
3061 Args: {Hi, Lo, ShiftAmt},
3062 /*FMFSource*/ nullptr, Name: "int");
3063 } else {
3064 // The shift of the most significant word.
3065 WordP[Dst] = Builder.CreateAShr(LHS: Lo, RHS: ShiftAmt, Name: "asr");
3066 }
3067 }
3068 if (SkipWords != 0)
3069 WordP.resize(N: WordP.size() - SkipWords);
3070
3071 return HVC.joinVectorElements(Builder, Values: WordP, ToType: Op.ResTy);
3072}
3073
3074auto HvxIdioms::createMulQ15(IRBuilderBase &Builder, SValue X, SValue Y,
3075 bool Rounding) const -> Value * {
3076 assert(X.Val->getType() == Y.Val->getType());
3077 assert(X.Val->getType()->getScalarType() == HVC.getIntTy(16));
3078 assert(HVC.HST.isHVXVectorType(EVT::getEVT(X.Val->getType(), false)));
3079
3080 // There is no non-rounding intrinsic for i16.
3081 if (!Rounding || X.Sgn == Unsigned || Y.Sgn == Unsigned)
3082 return nullptr;
3083
3084 auto V6_vmpyhvsrs = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyhvsrs);
3085 return HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyhvsrs, RetTy: X.Val->getType(),
3086 Args: {X.Val, Y.Val});
3087}
3088
3089auto HvxIdioms::createMulQ31(IRBuilderBase &Builder, SValue X, SValue Y,
3090 bool Rounding) const -> Value * {
3091 Type *InpTy = X.Val->getType();
3092 assert(InpTy == Y.Val->getType());
3093 assert(InpTy->getScalarType() == HVC.getIntTy(32));
3094 assert(HVC.HST.isHVXVectorType(EVT::getEVT(InpTy, false)));
3095
3096 if (X.Sgn == Unsigned || Y.Sgn == Unsigned)
3097 return nullptr;
3098
3099 auto V6_vmpyewuh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyewuh);
3100 auto V6_vmpyo_acc = Rounding
3101 ? HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyowh_rnd_sacc)
3102 : HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyowh_sacc);
3103 Value *V1 =
3104 HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyewuh, RetTy: InpTy, Args: {X.Val, Y.Val});
3105 return HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyo_acc, RetTy: InpTy,
3106 Args: {V1, X.Val, Y.Val});
3107}
3108
3109auto HvxIdioms::createAddCarry(IRBuilderBase &Builder, Value *X, Value *Y,
3110 Value *CarryIn) const
3111 -> std::pair<Value *, Value *> {
3112 assert(X->getType() == Y->getType());
3113 auto VecTy = cast<VectorType>(Val: X->getType());
3114 if (VecTy == HvxI32Ty && HVC.HST.useHVXV62Ops()) {
3115 SmallVector<Value *> Args = {X, Y};
3116 Intrinsic::ID AddCarry;
3117 if (CarryIn == nullptr && HVC.HST.useHVXV66Ops()) {
3118 AddCarry = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaddcarryo);
3119 } else {
3120 AddCarry = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaddcarry);
3121 if (CarryIn == nullptr)
3122 CarryIn = Constant::getNullValue(Ty: HVC.getBoolTy(ElemCount: HVC.length(Ty: VecTy)));
3123 Args.push_back(Elt: CarryIn);
3124 }
3125 Value *Ret = HVC.createHvxIntrinsic(Builder, IntID: AddCarry,
3126 /*RetTy=*/nullptr, Args);
3127 Value *Result = Builder.CreateExtractValue(Agg: Ret, Idxs: {0}, Name: "ext");
3128 Value *CarryOut = Builder.CreateExtractValue(Agg: Ret, Idxs: {1}, Name: "ext");
3129 return {Result, CarryOut};
3130 }
3131
3132 // In other cases, do a regular add, and unsigned compare-less-than.
3133 // The carry-out can originate in two places: adding the carry-in or adding
3134 // the two input values.
3135 Value *Result1 = X; // Result1 = X + CarryIn
3136 if (CarryIn != nullptr) {
3137 unsigned Width = VecTy->getScalarSizeInBits();
3138 uint32_t Mask = 1;
3139 if (Width < 32) {
3140 for (unsigned i = 0, e = 32 / Width; i != e; ++i)
3141 Mask = (Mask << Width) | 1;
3142 }
3143 auto V6_vandqrt = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vandqrt);
3144 Value *ValueIn =
3145 HVC.createHvxIntrinsic(Builder, IntID: V6_vandqrt, /*RetTy=*/nullptr,
3146 Args: {CarryIn, HVC.getConstInt(Val: Mask)});
3147 Result1 = Builder.CreateAdd(LHS: X, RHS: ValueIn, Name: "add");
3148 }
3149
3150 Value *CarryOut1 = Builder.CreateCmp(Pred: CmpInst::ICMP_ULT, LHS: Result1, RHS: X, Name: "cmp");
3151 Value *Result2 = Builder.CreateAdd(LHS: Result1, RHS: Y, Name: "add");
3152 Value *CarryOut2 = Builder.CreateCmp(Pred: CmpInst::ICMP_ULT, LHS: Result2, RHS: Y, Name: "cmp");
3153 return {Result2, Builder.CreateOr(LHS: CarryOut1, RHS: CarryOut2, Name: "orb")};
3154}
3155
3156auto HvxIdioms::createMul16(IRBuilderBase &Builder, SValue X, SValue Y) const
3157 -> Value * {
3158 Intrinsic::ID V6_vmpyh = 0;
3159 std::tie(args&: X, args&: Y) = canonSgn(X, Y);
3160
3161 if (X.Sgn == Signed) {
3162 V6_vmpyh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyhv);
3163 } else if (Y.Sgn == Signed) {
3164 // In vmpyhus the second operand is unsigned
3165 V6_vmpyh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyhus);
3166 } else {
3167 V6_vmpyh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyuhv);
3168 }
3169
3170 // i16*i16 -> i32 / interleaved
3171 Value *P =
3172 HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyh, RetTy: HvxP32Ty, Args: {Y.Val, X.Val});
3173 // Deinterleave
3174 return HVC.vshuff(Builder, Val0: HVC.sublo(Builder, Val: P), Val1: HVC.subhi(Builder, Val: P));
3175}
3176
3177auto HvxIdioms::createMulH16(IRBuilderBase &Builder, SValue X, SValue Y) const
3178 -> Value * {
3179 Type *HvxI16Ty = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), /*Pair=*/false);
3180
3181 if (HVC.HST.useHVXV69Ops()) {
3182 if (X.Sgn != Signed && Y.Sgn != Signed) {
3183 auto V6_vmpyuhvs = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyuhvs);
3184 return HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyuhvs, RetTy: HvxI16Ty,
3185 Args: {X.Val, Y.Val});
3186 }
3187 }
3188
3189 Type *HvxP16Ty = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), /*Pair=*/true);
3190 Value *Pair16 =
3191 Builder.CreateBitCast(V: createMul16(Builder, X, Y), DestTy: HvxP16Ty, Name: "cst");
3192 unsigned Len = HVC.length(Ty: HvxP16Ty) / 2;
3193
3194 SmallVector<int, 128> PickOdd(Len);
3195 for (int i = 0; i != static_cast<int>(Len); ++i)
3196 PickOdd[i] = 2 * i + 1;
3197
3198 return Builder.CreateShuffleVector(
3199 V1: HVC.sublo(Builder, Val: Pair16), V2: HVC.subhi(Builder, Val: Pair16), Mask: PickOdd, Name: "shf");
3200}
3201
3202auto HvxIdioms::createMul32(IRBuilderBase &Builder, SValue X, SValue Y) const
3203 -> std::pair<Value *, Value *> {
3204 assert(X.Val->getType() == Y.Val->getType());
3205 assert(X.Val->getType() == HvxI32Ty);
3206
3207 Intrinsic::ID V6_vmpy_parts;
3208 std::tie(args&: X, args&: Y) = canonSgn(X, Y);
3209
3210 if (X.Sgn == Signed) {
3211 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyss_parts;
3212 } else if (Y.Sgn == Signed) {
3213 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyus_parts;
3214 } else {
3215 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyuu_parts;
3216 }
3217
3218 Value *Parts = HVC.createHvxIntrinsic(Builder, IntID: V6_vmpy_parts, RetTy: nullptr,
3219 Args: {X.Val, Y.Val}, ArgTys: {HvxI32Ty});
3220 Value *Hi = Builder.CreateExtractValue(Agg: Parts, Idxs: {0}, Name: "ext");
3221 Value *Lo = Builder.CreateExtractValue(Agg: Parts, Idxs: {1}, Name: "ext");
3222 return {Lo, Hi};
3223}
3224
3225auto HvxIdioms::createAddLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
3226 ArrayRef<Value *> WordY) const
3227 -> SmallVector<Value *> {
3228 assert(WordX.size() == WordY.size());
3229 unsigned Idx = 0, Length = WordX.size();
3230 SmallVector<Value *> Sum(Length);
3231
3232 while (Idx != Length) {
3233 if (HVC.isZero(Val: WordX[Idx]))
3234 Sum[Idx] = WordY[Idx];
3235 else if (HVC.isZero(Val: WordY[Idx]))
3236 Sum[Idx] = WordX[Idx];
3237 else
3238 break;
3239 ++Idx;
3240 }
3241
3242 Value *Carry = nullptr;
3243 for (; Idx != Length; ++Idx) {
3244 std::tie(args&: Sum[Idx], args&: Carry) =
3245 createAddCarry(Builder, X: WordX[Idx], Y: WordY[Idx], CarryIn: Carry);
3246 }
3247
3248 // This drops the final carry beyond the highest word.
3249 return Sum;
3250}
3251
3252auto HvxIdioms::createMulLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
3253 Signedness SgnX, ArrayRef<Value *> WordY,
3254 Signedness SgnY) const -> SmallVector<Value *> {
3255 SmallVector<SmallVector<Value *>> Products(WordX.size() + WordY.size());
3256
3257 // WordX[i] * WordY[j] produces words i+j and i+j+1 of the results,
3258 // that is halves 2(i+j), 2(i+j)+1, 2(i+j)+2, 2(i+j)+3.
3259 for (int i = 0, e = WordX.size(); i != e; ++i) {
3260 for (int j = 0, f = WordY.size(); j != f; ++j) {
3261 // Check the 4 halves that this multiplication can generate.
3262 Signedness SX = (i + 1 == e) ? SgnX : Unsigned;
3263 Signedness SY = (j + 1 == f) ? SgnY : Unsigned;
3264 auto [Lo, Hi] = createMul32(Builder, X: {.Val: WordX[i], .Sgn: SX}, Y: {.Val: WordY[j], .Sgn: SY});
3265 Products[i + j + 0].push_back(Elt: Lo);
3266 Products[i + j + 1].push_back(Elt: Hi);
3267 }
3268 }
3269
3270 Value *Zero = Constant::getNullValue(Ty: WordX[0]->getType());
3271
3272 auto pop_back_or_zero = [Zero](auto &Vector) -> Value * {
3273 if (Vector.empty())
3274 return Zero;
3275 auto Last = Vector.back();
3276 Vector.pop_back();
3277 return Last;
3278 };
3279
3280 for (int i = 0, e = Products.size(); i != e; ++i) {
3281 while (Products[i].size() > 1) {
3282 Value *Carry = nullptr; // no carry-in
3283 for (int j = i; j != e; ++j) {
3284 auto &ProdJ = Products[j];
3285 auto [Sum, CarryOut] = createAddCarry(Builder, X: pop_back_or_zero(ProdJ),
3286 Y: pop_back_or_zero(ProdJ), CarryIn: Carry);
3287 ProdJ.insert(I: ProdJ.begin(), Elt: Sum);
3288 Carry = CarryOut;
3289 }
3290 }
3291 }
3292
3293 SmallVector<Value *> WordP;
3294 for (auto &P : Products) {
3295 assert(P.size() == 1 && "Should have been added together");
3296 WordP.push_back(Elt: P.front());
3297 }
3298
3299 return WordP;
3300}
3301
3302auto HvxIdioms::run() -> bool {
3303 bool Changed = false;
3304
3305 for (BasicBlock &B : HVC.F) {
3306 for (auto It = B.rbegin(); It != B.rend(); ++It) {
3307 if (auto Fxm = matchFxpMul(In&: *It)) {
3308 Value *New = processFxpMul(In&: *It, Op: *Fxm);
3309 // Always report "changed" for now.
3310 Changed = true;
3311 if (!New)
3312 continue;
3313 bool StartOver = !isa<Instruction>(Val: New);
3314 It->replaceAllUsesWith(V: New);
3315 RecursivelyDeleteTriviallyDeadInstructions(V: &*It, TLI: &HVC.TLI);
3316 It = StartOver ? B.rbegin()
3317 : cast<Instruction>(Val: New)->getReverseIterator();
3318 Changed = true;
3319 } else if (matchGather(In&: *It)) {
3320 Value *New = processVGather(In&: *It);
3321 if (!New)
3322 continue;
3323 LLVM_DEBUG(dbgs() << " Gather : " << *New << "\n");
3324 // We replace original intrinsic with a new pseudo call.
3325 It->eraseFromParent();
3326 It = cast<Instruction>(Val: New)->getReverseIterator();
3327 RecursivelyDeleteTriviallyDeadInstructions(V: &*It, TLI: &HVC.TLI);
3328 Changed = true;
3329 } else if (matchScatter(In&: *It)) {
3330 Value *New = processVScatter(In&: *It);
3331 if (!New)
3332 continue;
3333 LLVM_DEBUG(dbgs() << " Scatter : " << *New << "\n");
3334 // We replace original intrinsic with a new pseudo call.
3335 It->eraseFromParent();
3336 It = cast<Instruction>(Val: New)->getReverseIterator();
3337 RecursivelyDeleteTriviallyDeadInstructions(V: &*It, TLI: &HVC.TLI);
3338 Changed = true;
3339 } else if (matchMLoad(In&: *It)) {
3340 Value *New = processMLoad(In&: *It);
3341 if (!New)
3342 continue;
3343 LLVM_DEBUG(dbgs() << " MLoad : " << *New << "\n");
3344 Changed = true;
3345 } else if (matchMStore(In&: *It)) {
3346 Value *New = processMStore(In&: *It);
3347 if (!New)
3348 continue;
3349 LLVM_DEBUG(dbgs() << " MStore : " << *New << "\n");
3350 Changed = true;
3351 }
3352 }
3353 }
3354
3355 return Changed;
3356}
3357
3358// --- End HvxIdioms
3359
3360auto HexagonVectorCombine::run() -> bool {
3361 if (DumpModule)
3362 dbgs() << "Module before HexagonVectorCombine\n" << *F.getParent();
3363
3364 bool Changed = false;
3365 if (HST.useHVXOps()) {
3366 if (VAEnabled)
3367 Changed |= AlignVectors(*this).run();
3368 if (VIEnabled)
3369 Changed |= HvxIdioms(*this).run();
3370 }
3371
3372 if (DumpModule) {
3373 dbgs() << "Module " << (Changed ? "(modified)" : "(unchanged)")
3374 << " after HexagonVectorCombine\n"
3375 << *F.getParent();
3376 }
3377 return Changed;
3378}
3379
3380auto HexagonVectorCombine::getIntTy(unsigned Width) const -> IntegerType * {
3381 return IntegerType::get(C&: F.getContext(), NumBits: Width);
3382}
3383
3384auto HexagonVectorCombine::getByteTy(int ElemCount) const -> Type * {
3385 assert(ElemCount >= 0);
3386 IntegerType *ByteTy = Type::getInt8Ty(C&: F.getContext());
3387 if (ElemCount == 0)
3388 return ByteTy;
3389 return VectorType::get(ElementType: ByteTy, NumElements: ElemCount, /*Scalable=*/false);
3390}
3391
3392auto HexagonVectorCombine::getBoolTy(int ElemCount) const -> Type * {
3393 assert(ElemCount >= 0);
3394 IntegerType *BoolTy = Type::getInt1Ty(C&: F.getContext());
3395 if (ElemCount == 0)
3396 return BoolTy;
3397 return VectorType::get(ElementType: BoolTy, NumElements: ElemCount, /*Scalable=*/false);
3398}
3399
3400auto HexagonVectorCombine::getConstInt(int Val, unsigned Width) const
3401 -> ConstantInt * {
3402 return ConstantInt::getSigned(Ty: getIntTy(Width), V: Val);
3403}
3404
3405auto HexagonVectorCombine::isZero(const Value *Val) const -> bool {
3406 if (auto *C = dyn_cast<Constant>(Val))
3407 return C->isNullValue();
3408 return false;
3409}
3410
3411auto HexagonVectorCombine::getIntValue(const Value *Val) const
3412 -> std::optional<APInt> {
3413 if (auto *CI = dyn_cast<ConstantInt>(Val))
3414 return CI->getValue();
3415 return std::nullopt;
3416}
3417
3418auto HexagonVectorCombine::isUndef(const Value *Val) const -> bool {
3419 return isa<UndefValue>(Val);
3420}
3421
3422auto HexagonVectorCombine::isTrue(const Value *Val) const -> bool {
3423 return Val == ConstantInt::getTrue(Ty: Val->getType());
3424}
3425
3426auto HexagonVectorCombine::isFalse(const Value *Val) const -> bool {
3427 return isZero(Val);
3428}
3429
3430auto HexagonVectorCombine::getHvxTy(Type *ElemTy, bool Pair) const
3431 -> VectorType * {
3432 EVT ETy = EVT::getEVT(Ty: ElemTy, HandleUnknown: false);
3433 assert(ETy.isSimple() && "Invalid HVX element type");
3434 // Do not allow boolean types here: they don't have a fixed length.
3435 assert(HST.isHVXElementType(ETy.getSimpleVT(), /*IncludeBool=*/false) &&
3436 "Invalid HVX element type");
3437 unsigned HwLen = HST.getVectorLength();
3438 unsigned NumElems = (8 * HwLen) / ETy.getSizeInBits();
3439 return VectorType::get(ElementType: ElemTy, NumElements: Pair ? 2 * NumElems : NumElems,
3440 /*Scalable=*/false);
3441}
3442
3443auto HexagonVectorCombine::getSizeOf(const Value *Val, SizeKind Kind) const
3444 -> int {
3445 return getSizeOf(Ty: Val->getType(), Kind);
3446}
3447
3448auto HexagonVectorCombine::getSizeOf(const Type *Ty, SizeKind Kind) const
3449 -> int {
3450 auto *NcTy = const_cast<Type *>(Ty);
3451 switch (Kind) {
3452 case Store:
3453 return DL.getTypeStoreSize(Ty: NcTy).getFixedValue();
3454 case Alloc:
3455 return DL.getTypeAllocSize(Ty: NcTy).getFixedValue();
3456 }
3457 llvm_unreachable("Unhandled SizeKind enum");
3458}
3459
3460auto HexagonVectorCombine::getTypeAlignment(Type *Ty) const -> int {
3461 // The actual type may be shorter than the HVX vector, so determine
3462 // the alignment based on subtarget info.
3463 if (HST.isTypeForHVX(VecTy: Ty))
3464 return HST.getVectorLength();
3465 return DL.getABITypeAlign(Ty).value();
3466}
3467
3468auto HexagonVectorCombine::length(Value *Val) const -> size_t {
3469 return length(Ty: Val->getType());
3470}
3471
3472auto HexagonVectorCombine::length(Type *Ty) const -> size_t {
3473 auto *VecTy = dyn_cast<VectorType>(Val: Ty);
3474 assert(VecTy && "Must be a vector type");
3475 return VecTy->getElementCount().getFixedValue();
3476}
3477
3478auto HexagonVectorCombine::simplify(Value *V) const -> Value * {
3479 if (auto *In = dyn_cast<Instruction>(Val: V)) {
3480 SimplifyQuery Q(DL, &TLI, &DT, &AC, In);
3481 return simplifyInstruction(I: In, Q);
3482 }
3483 return nullptr;
3484}
3485
3486// Insert bytes [Start..Start+Length) of Src into Dst at byte Where.
3487auto HexagonVectorCombine::insertb(IRBuilderBase &Builder, Value *Dst,
3488 Value *Src, int Start, int Length,
3489 int Where) const -> Value * {
3490 assert(isByteVecTy(Dst->getType()) && isByteVecTy(Src->getType()));
3491 int SrcLen = getSizeOf(Val: Src);
3492 int DstLen = getSizeOf(Val: Dst);
3493 assert(0 <= Start && Start + Length <= SrcLen);
3494 assert(0 <= Where && Where + Length <= DstLen);
3495
3496 int P2Len = PowerOf2Ceil(A: SrcLen | DstLen);
3497 auto *Poison = PoisonValue::get(T: getByteTy());
3498 Value *P2Src = vresize(Builder, Val: Src, NewSize: P2Len, Pad: Poison);
3499 Value *P2Dst = vresize(Builder, Val: Dst, NewSize: P2Len, Pad: Poison);
3500
3501 SmallVector<int, 256> SMask(P2Len);
3502 for (int i = 0; i != P2Len; ++i) {
3503 // If i is in [Where, Where+Length), pick Src[Start+(i-Where)].
3504 // Otherwise, pick Dst[i];
3505 SMask[i] =
3506 (Where <= i && i < Where + Length) ? P2Len + Start + (i - Where) : i;
3507 }
3508
3509 Value *P2Insert = Builder.CreateShuffleVector(V1: P2Dst, V2: P2Src, Mask: SMask, Name: "shf");
3510 return vresize(Builder, Val: P2Insert, NewSize: DstLen, Pad: Poison);
3511}
3512
3513auto HexagonVectorCombine::vlalignb(IRBuilderBase &Builder, Value *Lo,
3514 Value *Hi, Value *Amt) const -> Value * {
3515 assert(Lo->getType() == Hi->getType() && "Argument type mismatch");
3516 if (isZero(Val: Amt))
3517 return Hi;
3518 int VecLen = getSizeOf(Val: Hi);
3519 if (auto IntAmt = getIntValue(Val: Amt))
3520 return getElementRange(Builder, Lo, Hi, Start: VecLen - IntAmt->getSExtValue(),
3521 Length: VecLen);
3522
3523 if (HST.isTypeForHVX(VecTy: Hi->getType())) {
3524 assert(static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
3525 "Expecting an exact HVX type");
3526 return createHvxIntrinsic(Builder, IntID: HST.getIntrinsicId(Opc: Hexagon::V6_vlalignb),
3527 RetTy: Hi->getType(), Args: {Hi, Lo, Amt});
3528 }
3529
3530 if (VecLen == 4) {
3531 Value *Pair = concat(Builder, Vecs: {Lo, Hi});
3532 Value *Shift =
3533 Builder.CreateLShr(LHS: Builder.CreateShl(LHS: Pair, RHS: Amt, Name: "shl"), RHS: 32, Name: "lsr");
3534 Value *Trunc =
3535 Builder.CreateTrunc(V: Shift, DestTy: Type::getInt32Ty(C&: F.getContext()), Name: "trn");
3536 return Builder.CreateBitCast(V: Trunc, DestTy: Hi->getType(), Name: "cst");
3537 }
3538 if (VecLen == 8) {
3539 Value *Sub = Builder.CreateSub(LHS: getConstInt(Val: VecLen), RHS: Amt, Name: "sub");
3540 return vralignb(Builder, Lo, Hi, Amt: Sub);
3541 }
3542 llvm_unreachable("Unexpected vector length");
3543}
3544
3545auto HexagonVectorCombine::vralignb(IRBuilderBase &Builder, Value *Lo,
3546 Value *Hi, Value *Amt) const -> Value * {
3547 assert(Lo->getType() == Hi->getType() && "Argument type mismatch");
3548 if (isZero(Val: Amt))
3549 return Lo;
3550 int VecLen = getSizeOf(Val: Lo);
3551 if (auto IntAmt = getIntValue(Val: Amt))
3552 return getElementRange(Builder, Lo, Hi, Start: IntAmt->getSExtValue(), Length: VecLen);
3553
3554 if (HST.isTypeForHVX(VecTy: Lo->getType())) {
3555 assert(static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
3556 "Expecting an exact HVX type");
3557 return createHvxIntrinsic(Builder, IntID: HST.getIntrinsicId(Opc: Hexagon::V6_valignb),
3558 RetTy: Lo->getType(), Args: {Hi, Lo, Amt});
3559 }
3560
3561 if (VecLen == 4) {
3562 Value *Pair = concat(Builder, Vecs: {Lo, Hi});
3563 Value *Shift = Builder.CreateLShr(LHS: Pair, RHS: Amt, Name: "lsr");
3564 Value *Trunc =
3565 Builder.CreateTrunc(V: Shift, DestTy: Type::getInt32Ty(C&: F.getContext()), Name: "trn");
3566 return Builder.CreateBitCast(V: Trunc, DestTy: Lo->getType(), Name: "cst");
3567 }
3568 if (VecLen == 8) {
3569 Type *Int64Ty = Type::getInt64Ty(C&: F.getContext());
3570 Value *Lo64 = Builder.CreateBitCast(V: Lo, DestTy: Int64Ty, Name: "cst");
3571 Value *Hi64 = Builder.CreateBitCast(V: Hi, DestTy: Int64Ty, Name: "cst");
3572 Value *Call = Builder.CreateIntrinsic(ID: Intrinsic::hexagon_S2_valignrb,
3573 Args: {Hi64, Lo64, Amt},
3574 /*FMFSource=*/nullptr, Name: "cup");
3575 return Builder.CreateBitCast(V: Call, DestTy: Lo->getType(), Name: "cst");
3576 }
3577 llvm_unreachable("Unexpected vector length");
3578}
3579
3580// Concatenates a sequence of vectors of the same type.
3581auto HexagonVectorCombine::concat(IRBuilderBase &Builder,
3582 ArrayRef<Value *> Vecs) const -> Value * {
3583 assert(!Vecs.empty());
3584 SmallVector<int, 256> SMask;
3585 std::vector<Value *> Work[2];
3586 int ThisW = 0, OtherW = 1;
3587
3588 Work[ThisW].assign(first: Vecs.begin(), last: Vecs.end());
3589 while (Work[ThisW].size() > 1) {
3590 auto *Ty = cast<VectorType>(Val: Work[ThisW].front()->getType());
3591 SMask.resize(N: length(Ty) * 2);
3592 std::iota(first: SMask.begin(), last: SMask.end(), value: 0);
3593
3594 Work[OtherW].clear();
3595 if (Work[ThisW].size() % 2 != 0)
3596 Work[ThisW].push_back(x: UndefValue::get(T: Ty));
3597 for (int i = 0, e = Work[ThisW].size(); i < e; i += 2) {
3598 Value *Joined = Builder.CreateShuffleVector(
3599 V1: Work[ThisW][i], V2: Work[ThisW][i + 1], Mask: SMask, Name: "shf");
3600 Work[OtherW].push_back(x: Joined);
3601 }
3602 std::swap(a&: ThisW, b&: OtherW);
3603 }
3604
3605 // Since there may have been some undefs appended to make shuffle operands
3606 // have the same type, perform the last shuffle to only pick the original
3607 // elements.
3608 SMask.resize(N: Vecs.size() * length(Ty: Vecs.front()->getType()));
3609 std::iota(first: SMask.begin(), last: SMask.end(), value: 0);
3610 Value *Total = Work[ThisW].front();
3611 return Builder.CreateShuffleVector(V: Total, Mask: SMask, Name: "shf");
3612}
3613
3614auto HexagonVectorCombine::vresize(IRBuilderBase &Builder, Value *Val,
3615 int NewSize, Value *Pad) const -> Value * {
3616 assert(isa<VectorType>(Val->getType()));
3617 auto *ValTy = cast<VectorType>(Val: Val->getType());
3618 assert(ValTy->getElementType() == Pad->getType());
3619
3620 int CurSize = length(Ty: ValTy);
3621 if (CurSize == NewSize)
3622 return Val;
3623 // Truncate?
3624 if (CurSize > NewSize)
3625 return getElementRange(Builder, Lo: Val, /*Ignored*/ Hi: Val, Start: 0, Length: NewSize);
3626 // Extend.
3627 SmallVector<int, 128> SMask(NewSize);
3628 std::iota(first: SMask.begin(), last: SMask.begin() + CurSize, value: 0);
3629 std::fill(first: SMask.begin() + CurSize, last: SMask.end(), value: CurSize);
3630 Value *PadVec = Builder.CreateVectorSplat(NumElts: CurSize, V: Pad, Name: "spt");
3631 return Builder.CreateShuffleVector(V1: Val, V2: PadVec, Mask: SMask, Name: "shf");
3632}
3633
3634auto HexagonVectorCombine::rescale(IRBuilderBase &Builder, Value *Mask,
3635 Type *FromTy, Type *ToTy) const -> Value * {
3636 // Mask is a vector <N x i1>, where each element corresponds to an
3637 // element of FromTy. Remap it so that each element will correspond
3638 // to an element of ToTy.
3639 assert(isa<VectorType>(Mask->getType()));
3640
3641 Type *FromSTy = FromTy->getScalarType();
3642 Type *ToSTy = ToTy->getScalarType();
3643 if (FromSTy == ToSTy)
3644 return Mask;
3645
3646 int FromSize = getSizeOf(Ty: FromSTy);
3647 int ToSize = getSizeOf(Ty: ToSTy);
3648 assert(FromSize % ToSize == 0 || ToSize % FromSize == 0);
3649
3650 auto *MaskTy = cast<VectorType>(Val: Mask->getType());
3651 int FromCount = length(Ty: MaskTy);
3652 int ToCount = (FromCount * FromSize) / ToSize;
3653 assert((FromCount * FromSize) % ToSize == 0);
3654
3655 auto *FromITy = getIntTy(Width: FromSize * 8);
3656 auto *ToITy = getIntTy(Width: ToSize * 8);
3657
3658 // Mask <N x i1> -> sext to <N x FromTy> -> bitcast to <M x ToTy> ->
3659 // -> trunc to <M x i1>.
3660 Value *Ext = Builder.CreateSExt(
3661 V: Mask, DestTy: VectorType::get(ElementType: FromITy, NumElements: FromCount, /*Scalable=*/false), Name: "sxt");
3662 Value *Cast = Builder.CreateBitCast(
3663 V: Ext, DestTy: VectorType::get(ElementType: ToITy, NumElements: ToCount, /*Scalable=*/false), Name: "cst");
3664 return Builder.CreateTrunc(
3665 V: Cast, DestTy: VectorType::get(ElementType: getBoolTy(), NumElements: ToCount, /*Scalable=*/false), Name: "trn");
3666}
3667
3668// Bitcast to bytes, and return least significant bits.
3669auto HexagonVectorCombine::vlsb(IRBuilderBase &Builder, Value *Val) const
3670 -> Value * {
3671 Type *ScalarTy = Val->getType()->getScalarType();
3672 if (ScalarTy == getBoolTy())
3673 return Val;
3674
3675 Value *Bytes = vbytes(Builder, Val);
3676 if (auto *VecTy = dyn_cast<VectorType>(Val: Bytes->getType()))
3677 return Builder.CreateTrunc(V: Bytes, DestTy: getBoolTy(ElemCount: getSizeOf(Ty: VecTy)), Name: "trn");
3678 // If Bytes is a scalar (i.e. Val was a scalar byte), return i1, not
3679 // <1 x i1>.
3680 return Builder.CreateTrunc(V: Bytes, DestTy: getBoolTy(), Name: "trn");
3681}
3682
3683// Bitcast to bytes for non-bool. For bool, convert i1 -> i8.
3684auto HexagonVectorCombine::vbytes(IRBuilderBase &Builder, Value *Val) const
3685 -> Value * {
3686 Type *ScalarTy = Val->getType()->getScalarType();
3687 if (ScalarTy == getByteTy())
3688 return Val;
3689
3690 if (ScalarTy != getBoolTy())
3691 return Builder.CreateBitCast(V: Val, DestTy: getByteTy(ElemCount: getSizeOf(Val)), Name: "cst");
3692 // For bool, return a sext from i1 to i8.
3693 if (auto *VecTy = dyn_cast<VectorType>(Val: Val->getType()))
3694 return Builder.CreateSExt(V: Val, DestTy: VectorType::get(ElementType: getByteTy(), Other: VecTy), Name: "sxt");
3695 return Builder.CreateSExt(V: Val, DestTy: getByteTy(), Name: "sxt");
3696}
3697
3698auto HexagonVectorCombine::subvector(IRBuilderBase &Builder, Value *Val,
3699 unsigned Start, unsigned Length) const
3700 -> Value * {
3701 assert(Start + Length <= length(Val));
3702 return getElementRange(Builder, Lo: Val, /*Ignored*/ Hi: Val, Start, Length);
3703}
3704
3705auto HexagonVectorCombine::sublo(IRBuilderBase &Builder, Value *Val) const
3706 -> Value * {
3707 size_t Len = length(Val);
3708 assert(Len % 2 == 0 && "Length should be even");
3709 return subvector(Builder, Val, Start: 0, Length: Len / 2);
3710}
3711
3712auto HexagonVectorCombine::subhi(IRBuilderBase &Builder, Value *Val) const
3713 -> Value * {
3714 size_t Len = length(Val);
3715 assert(Len % 2 == 0 && "Length should be even");
3716 return subvector(Builder, Val, Start: Len / 2, Length: Len / 2);
3717}
3718
3719auto HexagonVectorCombine::vdeal(IRBuilderBase &Builder, Value *Val0,
3720 Value *Val1) const -> Value * {
3721 assert(Val0->getType() == Val1->getType());
3722 int Len = length(Val: Val0);
3723 SmallVector<int, 128> Mask(2 * Len);
3724
3725 for (int i = 0; i != Len; ++i) {
3726 Mask[i] = 2 * i; // Even
3727 Mask[i + Len] = 2 * i + 1; // Odd
3728 }
3729 return Builder.CreateShuffleVector(V1: Val0, V2: Val1, Mask, Name: "shf");
3730}
3731
3732auto HexagonVectorCombine::vshuff(IRBuilderBase &Builder, Value *Val0,
3733 Value *Val1) const -> Value * { //
3734 assert(Val0->getType() == Val1->getType());
3735 int Len = length(Val: Val0);
3736 SmallVector<int, 128> Mask(2 * Len);
3737
3738 for (int i = 0; i != Len; ++i) {
3739 Mask[2 * i + 0] = i; // Val0
3740 Mask[2 * i + 1] = i + Len; // Val1
3741 }
3742 return Builder.CreateShuffleVector(V1: Val0, V2: Val1, Mask, Name: "shf");
3743}
3744
3745auto HexagonVectorCombine::createHvxIntrinsic(IRBuilderBase &Builder,
3746 Intrinsic::ID IntID, Type *RetTy,
3747 ArrayRef<Value *> Args,
3748 ArrayRef<Type *> ArgTys,
3749 ArrayRef<Value *> MDSources) const
3750 -> Value * {
3751 auto getCast = [&](IRBuilderBase &Builder, Value *Val,
3752 Type *DestTy) -> Value * {
3753 Type *SrcTy = Val->getType();
3754 if (SrcTy == DestTy)
3755 return Val;
3756
3757 // Non-HVX type. It should be a scalar, and it should already have
3758 // a valid type.
3759 assert(HST.isTypeForHVX(SrcTy, /*IncludeBool=*/true));
3760
3761 Type *BoolTy = Type::getInt1Ty(C&: F.getContext());
3762 if (cast<VectorType>(Val: SrcTy)->getElementType() != BoolTy)
3763 return Builder.CreateBitCast(V: Val, DestTy, Name: "cst");
3764
3765 // Predicate HVX vector.
3766 unsigned HwLen = HST.getVectorLength();
3767 Intrinsic::ID TC = HwLen == 64 ? Intrinsic::hexagon_V6_pred_typecast
3768 : Intrinsic::hexagon_V6_pred_typecast_128B;
3769 return Builder.CreateIntrinsic(ID: TC, OverloadTypes: {DestTy, Val->getType()}, Args: {Val},
3770 /*FMFSource=*/nullptr, Name: "cup");
3771 };
3772
3773 Function *IntrFn =
3774 Intrinsic::getOrInsertDeclaration(M: F.getParent(), id: IntID, OverloadTys: ArgTys);
3775 FunctionType *IntrTy = IntrFn->getFunctionType();
3776
3777 SmallVector<Value *, 4> IntrArgs;
3778 for (int i = 0, e = Args.size(); i != e; ++i) {
3779 Value *A = Args[i];
3780 Type *T = IntrTy->getParamType(i);
3781 if (A->getType() != T) {
3782 IntrArgs.push_back(Elt: getCast(Builder, A, T));
3783 } else {
3784 IntrArgs.push_back(Elt: A);
3785 }
3786 }
3787 StringRef MaybeName = !IntrTy->getReturnType()->isVoidTy() ? "cup" : "";
3788 CallInst *Call = Builder.CreateCall(Callee: IntrFn, Args: IntrArgs, Name: MaybeName);
3789
3790 MemoryEffects ME = Call->getAttributes().getMemoryEffects();
3791 if (!ME.doesNotAccessMemory() && !ME.onlyAccessesInaccessibleMem())
3792 propagateMetadata(I: Call, VL: MDSources);
3793
3794 Type *CallTy = Call->getType();
3795 if (RetTy == nullptr || CallTy == RetTy)
3796 return Call;
3797 // Scalar types should have RetTy matching the call return type.
3798 assert(HST.isTypeForHVX(CallTy, /*IncludeBool=*/true));
3799 return getCast(Builder, Call, RetTy);
3800}
3801
3802auto HexagonVectorCombine::splitVectorElements(IRBuilderBase &Builder,
3803 Value *Vec,
3804 unsigned ToWidth) const
3805 -> SmallVector<Value *> {
3806 // Break a vector of wide elements into a series of vectors with narrow
3807 // elements:
3808 // (...c0:b0:a0, ...c1:b1:a1, ...c2:b2:a2, ...)
3809 // -->
3810 // (a0, a1, a2, ...) // lowest "ToWidth" bits
3811 // (b0, b1, b2, ...) // the next lowest...
3812 // (c0, c1, c2, ...) // ...
3813 // ...
3814 //
3815 // The number of elements in each resulting vector is the same as
3816 // in the original vector.
3817
3818 auto *VecTy = cast<VectorType>(Val: Vec->getType());
3819 assert(VecTy->getElementType()->isIntegerTy());
3820 unsigned FromWidth = VecTy->getScalarSizeInBits();
3821 assert(isPowerOf2_32(ToWidth) && isPowerOf2_32(FromWidth));
3822 assert(ToWidth <= FromWidth && "Breaking up into wider elements?");
3823 unsigned NumResults = FromWidth / ToWidth;
3824
3825 SmallVector<Value *> Results(NumResults);
3826 Results[0] = Vec;
3827 unsigned Length = length(Ty: VecTy);
3828
3829 // Do it by splitting in half, since those operations correspond to deal
3830 // instructions.
3831 auto splitInHalf = [&](unsigned Begin, unsigned End, auto splitFunc) -> void {
3832 // Take V = Results[Begin], split it in L, H.
3833 // Store Results[Begin] = L, Results[(Begin+End)/2] = H
3834 // Call itself recursively split(Begin, Half), split(Half+1, End)
3835 if (Begin + 1 == End)
3836 return;
3837
3838 Value *Val = Results[Begin];
3839 unsigned Width = Val->getType()->getScalarSizeInBits();
3840
3841 auto *VTy = VectorType::get(ElementType: getIntTy(Width: Width / 2), NumElements: 2 * Length, Scalable: false);
3842 Value *VVal = Builder.CreateBitCast(V: Val, DestTy: VTy, Name: "cst");
3843
3844 Value *Res = vdeal(Builder, Val0: sublo(Builder, Val: VVal), Val1: subhi(Builder, Val: VVal));
3845
3846 unsigned Half = (Begin + End) / 2;
3847 Results[Begin] = sublo(Builder, Val: Res);
3848 Results[Half] = subhi(Builder, Val: Res);
3849
3850 splitFunc(Begin, Half, splitFunc);
3851 splitFunc(Half, End, splitFunc);
3852 };
3853
3854 splitInHalf(0, NumResults, splitInHalf);
3855 return Results;
3856}
3857
3858auto HexagonVectorCombine::joinVectorElements(IRBuilderBase &Builder,
3859 ArrayRef<Value *> Values,
3860 VectorType *ToType) const
3861 -> Value * {
3862 assert(ToType->getElementType()->isIntegerTy());
3863
3864 // If the list of values does not have power-of-2 elements, append copies
3865 // of the sign bit to it, to make the size be 2^n.
3866 // The reason for this is that the values will be joined in pairs, because
3867 // otherwise the shuffles will result in convoluted code. With pairwise
3868 // joins, the shuffles will hopefully be folded into a perfect shuffle.
3869 // The output will need to be sign-extended to a type with element width
3870 // being a power-of-2 anyways.
3871 SmallVector<Value *> Inputs(Values);
3872
3873 unsigned ToWidth = ToType->getScalarSizeInBits();
3874 unsigned Width = Inputs.front()->getType()->getScalarSizeInBits();
3875 assert(Width <= ToWidth);
3876 assert(isPowerOf2_32(Width) && isPowerOf2_32(ToWidth));
3877 unsigned Length = length(Ty: Inputs.front()->getType());
3878
3879 unsigned NeedInputs = ToWidth / Width;
3880 if (Inputs.size() != NeedInputs) {
3881 // Having too many inputs is ok: drop the high bits (usual wrap-around).
3882 // If there are too few, fill them with the sign bit.
3883 Value *Last = Inputs.back();
3884 Value *Sign = Builder.CreateAShr(
3885 LHS: Last, RHS: ConstantInt::get(Ty: Last->getType(), V: Width - 1), Name: "asr");
3886 Inputs.resize(N: NeedInputs, NV: Sign);
3887 }
3888
3889 while (Inputs.size() > 1) {
3890 Width *= 2;
3891 auto *VTy = VectorType::get(ElementType: getIntTy(Width), NumElements: Length, Scalable: false);
3892 for (int i = 0, e = Inputs.size(); i < e; i += 2) {
3893 Value *Res = vshuff(Builder, Val0: Inputs[i], Val1: Inputs[i + 1]);
3894 Inputs[i / 2] = Builder.CreateBitCast(V: Res, DestTy: VTy, Name: "cst");
3895 }
3896 Inputs.resize(N: Inputs.size() / 2);
3897 }
3898
3899 assert(Inputs.front()->getType() == ToType);
3900 return Inputs.front();
3901}
3902
3903auto HexagonVectorCombine::calculatePointerDifference(Value *Ptr0,
3904 Value *Ptr1) const
3905 -> std::optional<int> {
3906 // Try SCEV first.
3907 const SCEV *Scev0 = SE.getSCEV(V: Ptr0);
3908 const SCEV *Scev1 = SE.getSCEV(V: Ptr1);
3909 const SCEV *ScevDiff = SE.getMinusSCEV(LHS: Scev0, RHS: Scev1);
3910 if (auto *Const = dyn_cast<SCEVConstant>(Val: ScevDiff)) {
3911 APInt V = Const->getAPInt();
3912 if (V.isSignedIntN(N: 8 * sizeof(int)))
3913 return static_cast<int>(V.getSExtValue());
3914 }
3915
3916 struct Builder : IRBuilder<> {
3917 Builder(BasicBlock *B) : IRBuilder<>(B->getTerminator()) {}
3918 ~Builder() {
3919 for (Instruction *I : llvm::reverse(C&: ToErase))
3920 I->eraseFromParent();
3921 }
3922 SmallVector<Instruction *, 8> ToErase;
3923 };
3924
3925#define CallBuilder(B, F) \
3926 [&](auto &B_) { \
3927 Value *V = B_.F; \
3928 if (auto *I = dyn_cast<Instruction>(V)) \
3929 B_.ToErase.push_back(I); \
3930 return V; \
3931 }(B)
3932
3933 auto Simplify = [this](Value *V) {
3934 if (Value *S = simplify(V))
3935 return S;
3936 return V;
3937 };
3938
3939 auto StripBitCast = [](Value *V) {
3940 while (auto *C = dyn_cast<BitCastInst>(Val: V))
3941 V = C->getOperand(i_nocapture: 0);
3942 return V;
3943 };
3944
3945 Ptr0 = StripBitCast(Ptr0);
3946 Ptr1 = StripBitCast(Ptr1);
3947 if (!isa<GetElementPtrInst>(Val: Ptr0) || !isa<GetElementPtrInst>(Val: Ptr1))
3948 return std::nullopt;
3949
3950 auto *Gep0 = cast<GetElementPtrInst>(Val: Ptr0);
3951 auto *Gep1 = cast<GetElementPtrInst>(Val: Ptr1);
3952 if (Gep0->getPointerOperand() != Gep1->getPointerOperand())
3953 return std::nullopt;
3954 if (Gep0->getSourceElementType() != Gep1->getSourceElementType())
3955 return std::nullopt;
3956
3957 Builder B(Gep0->getParent());
3958 int Scale = getSizeOf(Ty: Gep0->getSourceElementType(), Kind: Alloc);
3959
3960 // FIXME: for now only check GEPs with a single index.
3961 if (Gep0->getNumOperands() != 2 || Gep1->getNumOperands() != 2)
3962 return std::nullopt;
3963
3964 Value *Idx0 = Gep0->getOperand(i_nocapture: 1);
3965 Value *Idx1 = Gep1->getOperand(i_nocapture: 1);
3966
3967 // First, try to simplify the subtraction directly.
3968 if (auto *Diff = dyn_cast<ConstantInt>(
3969 Val: Simplify(CallBuilder(B, CreateSub(Idx0, Idx1)))))
3970 return Diff->getSExtValue() * Scale;
3971
3972 KnownBits Known0 = getKnownBits(V: Idx0, CtxI: Gep0);
3973 KnownBits Known1 = getKnownBits(V: Idx1, CtxI: Gep1);
3974 APInt Unknown = ~(Known0.Zero | Known0.One) | ~(Known1.Zero | Known1.One);
3975 if (Unknown.isAllOnes())
3976 return std::nullopt;
3977
3978 Value *MaskU = ConstantInt::get(Ty: Idx0->getType(), V: Unknown);
3979 Value *AndU0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskU)));
3980 Value *AndU1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskU)));
3981 Value *SubU = Simplify(CallBuilder(B, CreateSub(AndU0, AndU1)));
3982 int Diff0 = 0;
3983 if (auto *C = dyn_cast<ConstantInt>(Val: SubU)) {
3984 Diff0 = C->getSExtValue();
3985 } else {
3986 return std::nullopt;
3987 }
3988
3989 Value *MaskK = ConstantInt::get(Ty: MaskU->getType(), V: ~Unknown);
3990 Value *AndK0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskK)));
3991 Value *AndK1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskK)));
3992 Value *SubK = Simplify(CallBuilder(B, CreateSub(AndK0, AndK1)));
3993 int Diff1 = 0;
3994 if (auto *C = dyn_cast<ConstantInt>(Val: SubK)) {
3995 Diff1 = C->getSExtValue();
3996 } else {
3997 return std::nullopt;
3998 }
3999
4000 return (Diff0 + Diff1) * Scale;
4001
4002#undef CallBuilder
4003}
4004
4005auto HexagonVectorCombine::getNumSignificantBits(const Value *V,
4006 const Instruction *CtxI) const
4007 -> unsigned {
4008 return ComputeMaxSignificantBits(Op: V, DL, AC: &AC, CtxI, DT: &DT);
4009}
4010
4011auto HexagonVectorCombine::getKnownBits(const Value *V,
4012 const Instruction *CtxI) const
4013 -> KnownBits {
4014 return computeKnownBits(V, DL, AC: &AC, CtxI, DT: &DT);
4015}
4016
4017auto HexagonVectorCombine::isSafeToClone(const Instruction &In) const -> bool {
4018 if (In.mayHaveSideEffects() || In.isAtomic() || In.isVolatile() ||
4019 In.isFenceLike() || In.mayReadOrWriteMemory()) {
4020 return false;
4021 }
4022 if (isa<CallBase>(Val: In) || isa<AllocaInst>(Val: In))
4023 return false;
4024 return true;
4025}
4026
4027template <typename T>
4028auto HexagonVectorCombine::isSafeToMoveBeforeInBB(const Instruction &In,
4029 BasicBlock::const_iterator To,
4030 const T &IgnoreInsts) const
4031 -> bool {
4032 auto getLocOrNone =
4033 [this](const Instruction &I) -> std::optional<MemoryLocation> {
4034 if (const auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
4035 switch (II->getIntrinsicID()) {
4036 case Intrinsic::masked_load:
4037 return MemoryLocation::getForArgument(Call: II, ArgIdx: 0, TLI);
4038 case Intrinsic::masked_store:
4039 return MemoryLocation::getForArgument(Call: II, ArgIdx: 1, TLI);
4040 }
4041 }
4042 return MemoryLocation::getOrNone(Inst: &I);
4043 };
4044
4045 // The source and the destination must be in the same basic block.
4046 const BasicBlock &Block = *In.getParent();
4047 assert(Block.begin() == To || Block.end() == To || To->getParent() == &Block);
4048 // No PHIs.
4049 if (isa<PHINode>(Val: In) || (To != Block.end() && isa<PHINode>(Val: *To)))
4050 return false;
4051
4052 if (!mayHaveNonDefUseDependency(I: In))
4053 return true;
4054 bool MayWrite = In.mayWriteToMemory();
4055 auto MaybeLoc = getLocOrNone(In);
4056
4057 auto From = In.getIterator();
4058 if (From == To)
4059 return true;
4060 bool MoveUp = (To != Block.end() && To->comesBefore(Other: &In));
4061 auto Range =
4062 MoveUp ? std::make_pair(x&: To, y&: From) : std::make_pair(x: std::next(x: From), y&: To);
4063 for (auto It = Range.first; It != Range.second; ++It) {
4064 const Instruction &I = *It;
4065 if (llvm::is_contained(IgnoreInsts, &I))
4066 continue;
4067 // assume intrinsic can be ignored
4068 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
4069 if (II->getIntrinsicID() == Intrinsic::assume)
4070 continue;
4071 }
4072 // Parts based on isSafeToMoveBefore from CoveMoverUtils.cpp.
4073 if (I.mayThrow())
4074 return false;
4075 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
4076 if (!CB->hasFnAttr(Kind: Attribute::WillReturn))
4077 return false;
4078 if (!CB->hasFnAttr(Kind: Attribute::NoSync))
4079 return false;
4080 }
4081 if (I.mayReadOrWriteMemory()) {
4082 auto MaybeLocI = getLocOrNone(I);
4083 if (MayWrite || I.mayWriteToMemory()) {
4084 if (!MaybeLoc || !MaybeLocI)
4085 return false;
4086 if (!AA.isNoAlias(*MaybeLoc, *MaybeLocI))
4087 return false;
4088 }
4089 }
4090 }
4091 return true;
4092}
4093
4094auto HexagonVectorCombine::isByteVecTy(Type *Ty) const -> bool {
4095 if (auto *VecTy = dyn_cast<VectorType>(Val: Ty))
4096 return VecTy->getElementType() == getByteTy();
4097 return false;
4098}
4099
4100auto HexagonVectorCombine::getElementRange(IRBuilderBase &Builder, Value *Lo,
4101 Value *Hi, int Start,
4102 int Length) const -> Value * {
4103 assert(0 <= Start && size_t(Start + Length) < length(Lo) + length(Hi));
4104 SmallVector<int, 128> SMask(Length);
4105 std::iota(first: SMask.begin(), last: SMask.end(), value: Start);
4106 return Builder.CreateShuffleVector(V1: Lo, V2: Hi, Mask: SMask, Name: "shf");
4107}
4108
4109// Pass management.
4110
4111namespace {
4112class HexagonVectorCombineLegacy : public FunctionPass {
4113public:
4114 static char ID;
4115
4116 HexagonVectorCombineLegacy() : FunctionPass(ID) {}
4117
4118 StringRef getPassName() const override { return "Hexagon Vector Combine"; }
4119
4120 void getAnalysisUsage(AnalysisUsage &AU) const override {
4121 AU.setPreservesCFG();
4122 AU.addRequired<AAResultsWrapperPass>();
4123 AU.addRequired<AssumptionCacheTracker>();
4124 AU.addRequired<DominatorTreeWrapperPass>();
4125 AU.addRequired<ScalarEvolutionWrapperPass>();
4126 AU.addRequired<TargetLibraryInfoWrapperPass>();
4127 AU.addRequired<TargetPassConfig>();
4128 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4129 FunctionPass::getAnalysisUsage(AU);
4130 }
4131
4132 bool runOnFunction(Function &F) override {
4133 if (skipFunction(F))
4134 return false;
4135 AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
4136 AssumptionCache &AC =
4137 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4138 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4139 ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4140 TargetLibraryInfo &TLI =
4141 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
4142 auto &TM = getAnalysis<TargetPassConfig>().getTM<HexagonTargetMachine>();
4143 auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4144 HexagonVectorCombine HVC(F, AA, AC, DT, SE, TLI, TM, ORE);
4145 return HVC.run();
4146 }
4147};
4148} // namespace
4149
4150char HexagonVectorCombineLegacy::ID = 0;
4151
4152INITIALIZE_PASS_BEGIN(HexagonVectorCombineLegacy, DEBUG_TYPE,
4153 "Hexagon Vector Combine", false, false)
4154INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
4155INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
4156INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
4157INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
4158INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
4159INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
4160INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
4161INITIALIZE_PASS_END(HexagonVectorCombineLegacy, DEBUG_TYPE,
4162 "Hexagon Vector Combine", false, false)
4163
4164FunctionPass *llvm::createHexagonVectorCombineLegacyPass() {
4165 return new HexagonVectorCombineLegacy();
4166}
4167