1//===-- HexagonVectorCombine.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// HexagonVectorCombine is a utility class implementing a variety of functions
9// that assist in vector-based optimizations.
10//
11// AlignVectors: replace unaligned vector loads and stores with aligned ones.
12// HvxIdioms: recognize various opportunities to generate HVX intrinsic code.
13//===----------------------------------------------------------------------===//
14
15#include "llvm/ADT/APInt.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/MapVector.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/Analysis/AliasAnalysis.h"
22#include "llvm/Analysis/AssumeBundleQueries.h"
23#include "llvm/Analysis/AssumptionCache.h"
24#include "llvm/Analysis/InstSimplifyFolder.h"
25#include "llvm/Analysis/InstructionSimplify.h"
26#include "llvm/Analysis/OptimizationRemarkEmitter.h"
27#include "llvm/Analysis/ScalarEvolutionExpressions.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/CodeGen/TargetPassConfig.h"
32#include "llvm/CodeGen/ValueTypes.h"
33#include "llvm/IR/Dominators.h"
34#include "llvm/IR/IRBuilder.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsHexagon.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/PatternMatch.h"
40#include "llvm/InitializePasses.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/KnownBits.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetMachine.h"
47#include "llvm/Transforms/Utils/Local.h"
48
49#include "Hexagon.h"
50#include "HexagonSubtarget.h"
51#include "HexagonTargetMachine.h"
52
53#include <algorithm>
54#include <deque>
55#include <optional>
56#include <set>
57#include <utility>
58#include <vector>
59
60#define DEBUG_TYPE "hexagon-vc"
61
62// This is a const that represents default HVX VTCM page size.
63// It is boot time configurable, so we probably want an API to
64// read it, but for now assume 128KB
65#define DEFAULT_HVX_VTCM_PAGE_SIZE 131072
66
67using namespace llvm;
68
69namespace {
70cl::opt<bool> DumpModule("hvc-dump-module", cl::Hidden);
71cl::opt<bool> VAEnabled("hvc-va", cl::Hidden, cl::init(Val: true)); // Align
72cl::opt<bool> VIEnabled("hvc-vi", cl::Hidden, cl::init(Val: true)); // Idioms
73cl::opt<bool> VADoFullStores("hvc-va-full-stores", cl::Hidden);
74
75cl::opt<unsigned> VAGroupCountLimit("hvc-va-group-count-limit", cl::Hidden,
76 cl::init(Val: ~0));
77cl::opt<unsigned> VAGroupSizeLimit("hvc-va-group-size-limit", cl::Hidden,
78 cl::init(Val: ~0));
79cl::opt<unsigned>
80 MinLoadGroupSizeForAlignment("hvc-ld-min-group-size-for-alignment",
81 cl::Hidden, cl::init(Val: 4));
82
83class HexagonVectorCombine {
84public:
85 HexagonVectorCombine(Function &F_, AliasAnalysis &AA_, AssumptionCache &AC_,
86 DominatorTree &DT_, ScalarEvolution &SE_,
87 TargetLibraryInfo &TLI_, const TargetMachine &TM_,
88 OptimizationRemarkEmitter &ORE_)
89 : F(F_), DL(F.getDataLayout()), AA(AA_), AC(AC_), DT(DT_), SE(SE_),
90 TLI(TLI_),
91 HST(static_cast<const HexagonSubtarget &>(*TM_.getSubtargetImpl(F))),
92 ORE(ORE_) {}
93
94 bool run();
95
96 // Common integer type.
97 IntegerType *getIntTy(unsigned Width = 32) const;
98 // Byte type: either scalar (when Length = 0), or vector with given
99 // element count.
100 Type *getByteTy(int ElemCount = 0) const;
101 // Boolean type: either scalar (when Length = 0), or vector with given
102 // element count.
103 Type *getBoolTy(int ElemCount = 0) const;
104 // Create a ConstantInt of type returned by getIntTy with the value Val.
105 ConstantInt *getConstInt(int Val, unsigned Width = 32) const;
106 // Get the integer value of V, if it exists.
107 std::optional<APInt> getIntValue(const Value *Val) const;
108 // Is Val a constant 0, or a vector of 0s?
109 bool isZero(const Value *Val) const;
110 // Is Val an undef value?
111 bool isUndef(const Value *Val) const;
112 // Is Val a scalar (i1 true) or a vector of (i1 true)?
113 bool isTrue(const Value *Val) const;
114 // Is Val a scalar (i1 false) or a vector of (i1 false)?
115 bool isFalse(const Value *Val) const;
116
117 // Get HVX vector type with the given element type.
118 VectorType *getHvxTy(Type *ElemTy, bool Pair = false) const;
119
120 enum SizeKind {
121 Store, // Store size
122 Alloc, // Alloc size
123 };
124 int getSizeOf(const Value *Val, SizeKind Kind = Store) const;
125 int getSizeOf(const Type *Ty, SizeKind Kind = Store) const;
126 int getTypeAlignment(Type *Ty) const;
127 size_t length(Value *Val) const;
128 size_t length(Type *Ty) const;
129
130 Value *simplify(Value *Val) const;
131
132 Value *insertb(IRBuilderBase &Builder, Value *Dest, Value *Src, int Start,
133 int Length, int Where) const;
134 Value *vlalignb(IRBuilderBase &Builder, Value *Lo, Value *Hi,
135 Value *Amt) const;
136 Value *vralignb(IRBuilderBase &Builder, Value *Lo, Value *Hi,
137 Value *Amt) const;
138 Value *concat(IRBuilderBase &Builder, ArrayRef<Value *> Vecs) const;
139 Value *vresize(IRBuilderBase &Builder, Value *Val, int NewSize,
140 Value *Pad) const;
141 Value *rescale(IRBuilderBase &Builder, Value *Mask, Type *FromTy,
142 Type *ToTy) const;
143 Value *vlsb(IRBuilderBase &Builder, Value *Val) const;
144 Value *vbytes(IRBuilderBase &Builder, Value *Val) const;
145 Value *subvector(IRBuilderBase &Builder, Value *Val, unsigned Start,
146 unsigned Length) const;
147 Value *sublo(IRBuilderBase &Builder, Value *Val) const;
148 Value *subhi(IRBuilderBase &Builder, Value *Val) const;
149 Value *vdeal(IRBuilderBase &Builder, Value *Val0, Value *Val1) const;
150 Value *vshuff(IRBuilderBase &Builder, Value *Val0, Value *Val1) const;
151
152 Value *createHvxIntrinsic(IRBuilderBase &Builder, Intrinsic::ID IntID,
153 Type *RetTy, ArrayRef<Value *> Args,
154 ArrayRef<Type *> ArgTys = {},
155 ArrayRef<Value *> MDSources = {}) const;
156 SmallVector<Value *> splitVectorElements(IRBuilderBase &Builder, Value *Vec,
157 unsigned ToWidth) const;
158 Value *joinVectorElements(IRBuilderBase &Builder, ArrayRef<Value *> Values,
159 VectorType *ToType) const;
160
161 std::optional<int> calculatePointerDifference(Value *Ptr0, Value *Ptr1) const;
162
163 unsigned getNumSignificantBits(const Value *V,
164 const Instruction *CtxI = nullptr) const;
165 KnownBits getKnownBits(const Value *V,
166 const Instruction *CtxI = nullptr) const;
167
168 bool isSafeToClone(const Instruction &In) const;
169
170 template <typename T = std::vector<Instruction *>>
171 bool isSafeToMoveBeforeInBB(const Instruction &In,
172 BasicBlock::const_iterator To,
173 const T &IgnoreInsts = {}) const;
174
175 // This function is only used for assertions at the moment.
176 [[maybe_unused]] bool isByteVecTy(Type *Ty) const;
177
178 Function &F;
179 const DataLayout &DL;
180 AliasAnalysis &AA;
181 AssumptionCache &AC;
182 DominatorTree &DT;
183 ScalarEvolution &SE;
184 TargetLibraryInfo &TLI;
185 const HexagonSubtarget &HST;
186 OptimizationRemarkEmitter &ORE;
187
188private:
189 Value *getElementRange(IRBuilderBase &Builder, Value *Lo, Value *Hi,
190 int Start, int Length) const;
191};
192
193class AlignVectors {
194 // This code tries to replace unaligned vector loads/stores with aligned
195 // ones.
196 // Consider unaligned load:
197 // %v = original_load %some_addr, align <bad>
198 // %user = %v
199 // It will generate
200 // = load ..., align <good>
201 // = load ..., align <good>
202 // = valign
203 // etc.
204 // %synthesize = combine/shuffle the loaded data so that it looks
205 // exactly like what "original_load" has loaded.
206 // %user = %synthesize
207 // Similarly for stores.
208public:
209 AlignVectors(const HexagonVectorCombine &HVC_) : HVC(HVC_) {}
210
211 bool run();
212
213private:
214 using InstList = std::vector<Instruction *>;
215 using InstMap = DenseMap<Instruction *, Instruction *>;
216
217 struct AddrInfo {
218 AddrInfo(const AddrInfo &) = default;
219 AddrInfo &operator=(const AddrInfo &) = default;
220 AddrInfo(const HexagonVectorCombine &HVC, Instruction *I, Value *A, Type *T,
221 Align H)
222 : Inst(I), Addr(A), ValTy(T), HaveAlign(H),
223 NeedAlign(HVC.getTypeAlignment(Ty: ValTy)) {}
224
225 // XXX: add Size member?
226 Instruction *Inst;
227 Value *Addr;
228 Type *ValTy;
229 Align HaveAlign;
230 Align NeedAlign;
231 int Offset = 0; // Offset (in bytes) from the first member of the
232 // containing AddrList.
233 };
234 using AddrList = std::vector<AddrInfo>;
235
236 struct InstrLess {
237 bool operator()(const Instruction *A, const Instruction *B) const {
238 return A->comesBefore(Other: B);
239 }
240 };
241 using DepList = std::set<Instruction *, InstrLess>;
242
243 struct MoveGroup {
244 MoveGroup(const AddrInfo &AI, Instruction *B, bool Hvx, bool Load)
245 : Base(B), Main{AI.Inst}, Clones{}, IsHvx(Hvx), IsLoad(Load) {}
246 MoveGroup() = default;
247 Instruction *Base; // Base instruction of the parent address group.
248 InstList Main; // Main group of instructions.
249 InstList Deps; // List of dependencies.
250 InstMap Clones; // Map from original Deps to cloned ones.
251 bool IsHvx; // Is this group of HVX instructions?
252 bool IsLoad; // Is this a load group?
253 };
254 using MoveList = std::vector<MoveGroup>;
255
256 struct ByteSpan {
257 // A representation of "interesting" bytes within a given span of memory.
258 // These bytes are those that are loaded or stored, and they don't have
259 // to cover the entire span of memory.
260 //
261 // The representation works by picking a contiguous sequence of bytes
262 // from somewhere within a llvm::Value, and placing it at a given offset
263 // within the span.
264 //
265 // The sequence of bytes from llvm:Value is represented by Segment.
266 // Block is Segment, plus where it goes in the span.
267 //
268 // An important feature of ByteSpan is being able to make a "section",
269 // i.e. creating another ByteSpan corresponding to a range of offsets
270 // relative to the source span.
271
272 struct Segment {
273 // Segment of a Value: 'Len' bytes starting at byte 'Begin'.
274 Segment(Value *Val, int Begin, int Len)
275 : Val(Val), Start(Begin), Size(Len) {}
276 Segment(const Segment &Seg) = default;
277 Segment &operator=(const Segment &Seg) = default;
278 Value *Val; // Value representable as a sequence of bytes.
279 int Start; // First byte of the value that belongs to the segment.
280 int Size; // Number of bytes in the segment.
281 };
282
283 struct Block {
284 Block(Value *Val, int Len, int Pos) : Seg(Val, 0, Len), Pos(Pos) {}
285 Block(Value *Val, int Off, int Len, int Pos)
286 : Seg(Val, Off, Len), Pos(Pos) {}
287 Block(const Block &Blk) = default;
288 Block &operator=(const Block &Blk) = default;
289 Segment Seg; // Value segment.
290 int Pos; // Position (offset) of the block in the span.
291 };
292
293 int extent() const;
294 ByteSpan section(int Start, int Length) const;
295 ByteSpan &shift(int Offset);
296 SmallVector<Value *, 8> values() const;
297
298 int size() const { return Blocks.size(); }
299 Block &operator[](int i) { return Blocks[i]; }
300 const Block &operator[](int i) const { return Blocks[i]; }
301
302 std::vector<Block> Blocks;
303
304 using iterator = decltype(Blocks)::iterator;
305 iterator begin() { return Blocks.begin(); }
306 iterator end() { return Blocks.end(); }
307 using const_iterator = decltype(Blocks)::const_iterator;
308 const_iterator begin() const { return Blocks.begin(); }
309 const_iterator end() const { return Blocks.end(); }
310 };
311
312 std::optional<AddrInfo> getAddrInfo(Instruction &In) const;
313 bool isHvx(const AddrInfo &AI) const;
314 // This function is only used for assertions at the moment.
315 [[maybe_unused]] bool isSectorTy(Type *Ty) const;
316
317 Value *getPayload(Value *Val) const;
318 Value *getMask(Value *Val) const;
319 Value *getPassThrough(Value *Val) const;
320
321 Value *createAdjustedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy,
322 int Adjust,
323 const InstMap &CloneMap = InstMap()) const;
324 Value *createAlignedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy,
325 int Alignment,
326 const InstMap &CloneMap = InstMap()) const;
327
328 Value *createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
329 Value *Predicate, int Alignment, Value *Mask,
330 Value *PassThru, ArrayRef<Value *> MDSources = {}) const;
331 Value *createSimpleLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
332 int Alignment,
333 ArrayRef<Value *> MDSources = {}) const;
334
335 Value *createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
336 Value *Predicate, int Alignment, Value *Mask,
337 ArrayRef<Value *> MDSources = {}) const;
338 Value *createSimpleStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
339 int Alignment,
340 ArrayRef<Value *> MDSources = {}) const;
341
342 Value *createPredicatedLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
343 Value *Predicate, int Alignment,
344 ArrayRef<Value *> MDSources = {}) const;
345 Value *createPredicatedStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
346 Value *Predicate, int Alignment,
347 ArrayRef<Value *> MDSources = {}) const;
348
349 DepList getUpwardDeps(Instruction *In, Instruction *Base) const;
350 bool createAddressGroups();
351 MoveList createLoadGroups(const AddrList &Group) const;
352 MoveList createStoreGroups(const AddrList &Group) const;
353 bool moveTogether(MoveGroup &Move) const;
354 template <typename T>
355 InstMap cloneBefore(BasicBlock::iterator To, T &&Insts) const;
356
357 void realignLoadGroup(IRBuilderBase &Builder, const ByteSpan &VSpan,
358 int ScLen, Value *AlignVal, Value *AlignAddr) const;
359 void realignStoreGroup(IRBuilderBase &Builder, const ByteSpan &VSpan,
360 int ScLen, Value *AlignVal, Value *AlignAddr) const;
361 bool realignGroup(const MoveGroup &Move);
362 Value *makeTestIfUnaligned(IRBuilderBase &Builder, Value *AlignVal,
363 int Alignment) const;
364
365 using AddrGroupMap = MapVector<Instruction *, AddrList>;
366 AddrGroupMap AddrGroups;
367
368 friend raw_ostream &operator<<(raw_ostream &OS, const AddrList &L);
369 friend raw_ostream &operator<<(raw_ostream &OS, const AddrInfo &AI);
370 friend raw_ostream &operator<<(raw_ostream &OS, const MoveGroup &MG);
371 friend raw_ostream &operator<<(raw_ostream &OS, const MoveList &L);
372 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan::Block &B);
373 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan &BS);
374 friend raw_ostream &operator<<(raw_ostream &OS, const AddrGroupMap &AG);
375 friend raw_ostream &operator<<(raw_ostream &OS, const AddrList &L);
376 friend raw_ostream &operator<<(raw_ostream &OS, const AddrInfo &AI);
377 friend raw_ostream &operator<<(raw_ostream &OS, const MoveGroup &MG);
378 friend raw_ostream &operator<<(raw_ostream &OS, const MoveList &L);
379 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan::Block &B);
380 friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan &BS);
381 friend raw_ostream &operator<<(raw_ostream &OS, const AddrGroupMap &AG);
382
383 const HexagonVectorCombine &HVC;
384};
385
386[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
387 const AlignVectors::AddrGroupMap &AG) {
388 OS << "Printing AddrGroups:"
389 << "\n";
390 for (auto &It : AG) {
391 OS << "\n\tInstruction: ";
392 It.first->dump();
393 OS << "\n\tAddrInfo: ";
394 for (auto &AI : It.second)
395 OS << AI << "\n";
396 }
397 return OS;
398}
399
400[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
401 const AlignVectors::AddrList &AL) {
402 OS << "\n *** Addr List: ***\n";
403 for (auto &AG : AL) {
404 OS << "\n *** Addr Group: ***\n";
405 OS << AG;
406 OS << "\n";
407 }
408 return OS;
409}
410
411[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
412 const AlignVectors::AddrInfo &AI) {
413 OS << "Inst: " << AI.Inst << " " << *AI.Inst << '\n';
414 OS << "Addr: " << *AI.Addr << '\n';
415 OS << "Type: " << *AI.ValTy << '\n';
416 OS << "HaveAlign: " << AI.HaveAlign.value() << '\n';
417 OS << "NeedAlign: " << AI.NeedAlign.value() << '\n';
418 OS << "Offset: " << AI.Offset;
419 return OS;
420}
421
422[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
423 const AlignVectors::MoveList &ML) {
424 OS << "\n *** Move List: ***\n";
425 for (auto &MG : ML) {
426 OS << "\n *** Move Group: ***\n";
427 OS << MG;
428 OS << "\n";
429 }
430 return OS;
431}
432
433[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
434 const AlignVectors::MoveGroup &MG) {
435 OS << "IsLoad:" << (MG.IsLoad ? "yes" : "no");
436 OS << ", IsHvx:" << (MG.IsHvx ? "yes" : "no") << '\n';
437 OS << "Main\n";
438 for (Instruction *I : MG.Main)
439 OS << " " << *I << '\n';
440 OS << "Deps\n";
441 for (Instruction *I : MG.Deps)
442 OS << " " << *I << '\n';
443 OS << "Clones\n";
444 for (auto [K, V] : MG.Clones) {
445 OS << " ";
446 K->printAsOperand(O&: OS, PrintType: false);
447 OS << "\t-> " << *V << '\n';
448 }
449 return OS;
450}
451
452[[maybe_unused]] raw_ostream &
453operator<<(raw_ostream &OS, const AlignVectors::ByteSpan::Block &B) {
454 OS << " @" << B.Pos << " [" << B.Seg.Start << ',' << B.Seg.Size << "] ";
455 if (B.Seg.Val == reinterpret_cast<const Value *>(&B)) {
456 OS << "(self:" << B.Seg.Val << ')';
457 } else if (B.Seg.Val != nullptr) {
458 OS << *B.Seg.Val;
459 } else {
460 OS << "(null)";
461 }
462 return OS;
463}
464
465[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
466 const AlignVectors::ByteSpan &BS) {
467 OS << "ByteSpan[size=" << BS.size() << ", extent=" << BS.extent() << '\n';
468 for (const AlignVectors::ByteSpan::Block &B : BS)
469 OS << B << '\n';
470 OS << ']';
471 return OS;
472}
473
474class HvxIdioms {
475public:
476 enum DstQualifier {
477 Undefined = 0,
478 Arithmetic,
479 LdSt,
480 LLVM_Gather,
481 LLVM_Scatter,
482 HEX_Gather_Scatter,
483 HEX_Gather,
484 HEX_Scatter,
485 Call
486 };
487
488 HvxIdioms(const HexagonVectorCombine &HVC_) : HVC(HVC_) {
489 auto *Int32Ty = HVC.getIntTy(Width: 32);
490 HvxI32Ty = HVC.getHvxTy(ElemTy: Int32Ty, /*Pair=*/false);
491 HvxP32Ty = HVC.getHvxTy(ElemTy: Int32Ty, /*Pair=*/true);
492 }
493
494 bool run();
495
496private:
497 enum Signedness { Positive, Signed, Unsigned };
498
499 // Value + sign
500 // This is to keep track of whether the value should be treated as signed
501 // or unsigned, or is known to be positive.
502 struct SValue {
503 Value *Val;
504 Signedness Sgn;
505 };
506
507 struct FxpOp {
508 unsigned Opcode;
509 unsigned Frac; // Number of fraction bits
510 SValue X, Y;
511 // If present, add 1 << RoundAt before shift:
512 std::optional<unsigned> RoundAt;
513 VectorType *ResTy;
514 };
515
516 auto getNumSignificantBits(Value *V, Instruction *In) const
517 -> std::pair<unsigned, Signedness>;
518 auto canonSgn(SValue X, SValue Y) const -> std::pair<SValue, SValue>;
519
520 auto matchFxpMul(Instruction &In) const -> std::optional<FxpOp>;
521 auto processFxpMul(Instruction &In, const FxpOp &Op) const -> Value *;
522
523 auto processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
524 const FxpOp &Op) const -> Value *;
525 auto createMulQ15(IRBuilderBase &Builder, SValue X, SValue Y,
526 bool Rounding) const -> Value *;
527 auto createMulQ31(IRBuilderBase &Builder, SValue X, SValue Y,
528 bool Rounding) const -> Value *;
529 // Return {Result, Carry}, where Carry is a vector predicate.
530 auto createAddCarry(IRBuilderBase &Builder, Value *X, Value *Y,
531 Value *CarryIn = nullptr) const
532 -> std::pair<Value *, Value *>;
533 auto createMul16(IRBuilderBase &Builder, SValue X, SValue Y) const -> Value *;
534 auto createMulH16(IRBuilderBase &Builder, SValue X, SValue Y) const
535 -> Value *;
536 auto createMul32(IRBuilderBase &Builder, SValue X, SValue Y) const
537 -> std::pair<Value *, Value *>;
538 auto createAddLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
539 ArrayRef<Value *> WordY) const -> SmallVector<Value *>;
540 auto createMulLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
541 Signedness SgnX, ArrayRef<Value *> WordY,
542 Signedness SgnY) const -> SmallVector<Value *>;
543
544 bool matchMLoad(Instruction &In) const;
545 bool matchMStore(Instruction &In) const;
546 Value *processMLoad(Instruction &In) const;
547 Value *processMStore(Instruction &In) const;
548 std::optional<uint64_t> getAlignment(Instruction &In, Value *ptr) const;
549 std::optional<uint64_t>
550 getAlignmentImpl(Instruction &In, Value *ptr,
551 SmallPtrSet<Value *, 16> &Visited) const;
552 std::optional<uint64_t> getPHIBaseMinAlignment(Instruction &In,
553 PHINode *PN) const;
554
555 // Vector manipulations for Ripple
556 bool matchScatter(Instruction &In) const;
557 bool matchGather(Instruction &In) const;
558 Value *processVScatter(Instruction &In) const;
559 Value *processVGather(Instruction &In) const;
560
561 VectorType *HvxI32Ty;
562 VectorType *HvxP32Ty;
563 const HexagonVectorCombine &HVC;
564
565 friend raw_ostream &operator<<(raw_ostream &, const FxpOp &);
566};
567
568[[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS,
569 const HvxIdioms::FxpOp &Op) {
570 static const char *SgnNames[] = {"Positive", "Signed", "Unsigned"};
571 OS << Instruction::getOpcodeName(Opcode: Op.Opcode) << '.' << Op.Frac;
572 if (Op.RoundAt.has_value()) {
573 if (Op.Frac != 0 && *Op.RoundAt == Op.Frac - 1) {
574 OS << ":rnd";
575 } else {
576 OS << " + 1<<" << *Op.RoundAt;
577 }
578 }
579 OS << "\n X:(" << SgnNames[Op.X.Sgn] << ") " << *Op.X.Val << "\n"
580 << " Y:(" << SgnNames[Op.Y.Sgn] << ") " << *Op.Y.Val;
581 return OS;
582}
583
584} // namespace
585
586namespace {
587
588template <typename T> T *getIfUnordered(T *MaybeT) {
589 return MaybeT && MaybeT->isUnordered() ? MaybeT : nullptr;
590}
591template <typename T> T *isCandidate(Instruction *In) {
592 return dyn_cast<T>(In);
593}
594template <> LoadInst *isCandidate<LoadInst>(Instruction *In) {
595 return getIfUnordered(MaybeT: dyn_cast<LoadInst>(Val: In));
596}
597template <> StoreInst *isCandidate<StoreInst>(Instruction *In) {
598 return getIfUnordered(MaybeT: dyn_cast<StoreInst>(Val: In));
599}
600
601// Forward other erase_ifs to the LLVM implementations.
602template <typename Pred, typename T> void erase_if(T &&container, Pred p) {
603 llvm::erase_if(std::forward<T>(container), p);
604}
605
606} // namespace
607
608// --- Begin AlignVectors
609
610// For brevity, only consider loads. We identify a group of loads where we
611// know the relative differences between their addresses, so we know how they
612// are laid out in memory (relative to one another). These loads can overlap,
613// can be shorter or longer than the desired vector length.
614// Ultimately we want to generate a sequence of aligned loads that will load
615// every byte that the original loads loaded, and have the program use these
616// loaded values instead of the original loads.
617// We consider the contiguous memory area spanned by all these loads.
618//
619// Let's say that a single aligned vector load can load 16 bytes at a time.
620// If the program wanted to use a byte at offset 13 from the beginning of the
621// original span, it will be a byte at offset 13+x in the aligned data for
622// some x>=0. This may happen to be in the first aligned load, or in the load
623// following it. Since we generally don't know what the that alignment value
624// is at compile time, we proactively do valigns on the aligned loads, so that
625// byte that was at offset 13 is still at offset 13 after the valigns.
626//
627// This will be the starting point for making the rest of the program use the
628// data loaded by the new loads.
629// For each original load, and its users:
630// %v = load ...
631// ... = %v
632// ... = %v
633// we create
634// %new_v = extract/combine/shuffle data from loaded/valigned vectors so
635// it contains the same value as %v did before
636// then replace all users of %v with %new_v.
637// ... = %new_v
638// ... = %new_v
639
640auto AlignVectors::ByteSpan::extent() const -> int {
641 if (size() == 0)
642 return 0;
643 int Min = Blocks[0].Pos;
644 int Max = Blocks[0].Pos + Blocks[0].Seg.Size;
645 for (int i = 1, e = size(); i != e; ++i) {
646 Min = std::min(a: Min, b: Blocks[i].Pos);
647 Max = std::max(a: Max, b: Blocks[i].Pos + Blocks[i].Seg.Size);
648 }
649 return Max - Min;
650}
651
652auto AlignVectors::ByteSpan::section(int Start, int Length) const -> ByteSpan {
653 ByteSpan Section;
654 for (const ByteSpan::Block &B : Blocks) {
655 int L = std::max(a: B.Pos, b: Start); // Left end.
656 int R = std::min(a: B.Pos + B.Seg.Size, b: Start + Length); // Right end+1.
657 if (L < R) {
658 // How much to chop off the beginning of the segment:
659 int Off = L > B.Pos ? L - B.Pos : 0;
660 Section.Blocks.emplace_back(args: B.Seg.Val, args: B.Seg.Start + Off, args: R - L, args&: L);
661 }
662 }
663 return Section;
664}
665
666auto AlignVectors::ByteSpan::shift(int Offset) -> ByteSpan & {
667 for (Block &B : Blocks)
668 B.Pos += Offset;
669 return *this;
670}
671
672auto AlignVectors::ByteSpan::values() const -> SmallVector<Value *, 8> {
673 SmallVector<Value *, 8> Values(Blocks.size());
674 for (int i = 0, e = Blocks.size(); i != e; ++i)
675 Values[i] = Blocks[i].Seg.Val;
676 return Values;
677}
678
679// Turn a requested integer alignment into the effective Align to use.
680// If Requested == 0 -> use ABI alignment of the value type (old semantics).
681// 0 means "ABI alignment" in old IR.
682static Align effectiveAlignForValueTy(const DataLayout &DL, Type *ValTy,
683 int Requested) {
684 if (Requested > 0)
685 return Align(static_cast<uint64_t>(Requested));
686 return Align(DL.getABITypeAlign(Ty: ValTy).value());
687}
688
689auto AlignVectors::getAddrInfo(Instruction &In) const
690 -> std::optional<AddrInfo> {
691 if (auto *L = isCandidate<LoadInst>(In: &In))
692 return AddrInfo(HVC, L, L->getPointerOperand(), L->getType(),
693 L->getAlign());
694 if (auto *S = isCandidate<StoreInst>(In: &In))
695 return AddrInfo(HVC, S, S->getPointerOperand(),
696 S->getValueOperand()->getType(), S->getAlign());
697 if (auto *II = isCandidate<IntrinsicInst>(In: &In)) {
698 Intrinsic::ID ID = II->getIntrinsicID();
699 switch (ID) {
700 case Intrinsic::masked_load:
701 return AddrInfo(HVC, II, II->getArgOperand(i: 0), II->getType(),
702 II->getParamAlign(ArgNo: 0).valueOrOne());
703 case Intrinsic::masked_store:
704 return AddrInfo(HVC, II, II->getArgOperand(i: 1),
705 II->getArgOperand(i: 0)->getType(),
706 II->getParamAlign(ArgNo: 1).valueOrOne());
707 }
708 }
709 return std::nullopt;
710}
711
712auto AlignVectors::isHvx(const AddrInfo &AI) const -> bool {
713 return HVC.HST.isTypeForHVX(VecTy: AI.ValTy);
714}
715
716auto AlignVectors::getPayload(Value *Val) const -> Value * {
717 if (auto *In = dyn_cast<Instruction>(Val)) {
718 Intrinsic::ID ID = 0;
719 if (auto *II = dyn_cast<IntrinsicInst>(Val: In))
720 ID = II->getIntrinsicID();
721 if (isa<StoreInst>(Val: In) || ID == Intrinsic::masked_store)
722 return In->getOperand(i: 0);
723 }
724 return Val;
725}
726
727auto AlignVectors::getMask(Value *Val) const -> Value * {
728 if (auto *II = dyn_cast<IntrinsicInst>(Val)) {
729 switch (II->getIntrinsicID()) {
730 case Intrinsic::masked_load:
731 return II->getArgOperand(i: 1);
732 case Intrinsic::masked_store:
733 return II->getArgOperand(i: 2);
734 }
735 }
736
737 Type *ValTy = getPayload(Val)->getType();
738 if (auto *VecTy = dyn_cast<VectorType>(Val: ValTy))
739 return Constant::getAllOnesValue(Ty: HVC.getBoolTy(ElemCount: HVC.length(Ty: VecTy)));
740 return Constant::getAllOnesValue(Ty: HVC.getBoolTy());
741}
742
743auto AlignVectors::getPassThrough(Value *Val) const -> Value * {
744 if (auto *II = dyn_cast<IntrinsicInst>(Val)) {
745 if (II->getIntrinsicID() == Intrinsic::masked_load)
746 return II->getArgOperand(i: 2);
747 }
748 return UndefValue::get(T: getPayload(Val)->getType());
749}
750
751auto AlignVectors::createAdjustedPointer(IRBuilderBase &Builder, Value *Ptr,
752 Type *ValTy, int Adjust,
753 const InstMap &CloneMap) const
754 -> Value * {
755 if (auto *I = dyn_cast<Instruction>(Val: Ptr))
756 if (Instruction *New = CloneMap.lookup(Val: I))
757 Ptr = New;
758 return Builder.CreatePtrAdd(Ptr, Offset: HVC.getConstInt(Val: Adjust), Name: "gep");
759}
760
761auto AlignVectors::createAlignedPointer(IRBuilderBase &Builder, Value *Ptr,
762 Type *ValTy, int Alignment,
763 const InstMap &CloneMap) const
764 -> Value * {
765 auto remap = [&](Value *V) -> Value * {
766 if (auto *I = dyn_cast<Instruction>(Val: V)) {
767 for (auto [Old, New] : CloneMap)
768 I->replaceUsesOfWith(From: Old, To: New);
769 return I;
770 }
771 return V;
772 };
773 Value *AsInt = Builder.CreatePtrToInt(V: Ptr, DestTy: HVC.getIntTy(), Name: "pti");
774 Value *Mask = HVC.getConstInt(Val: -Alignment);
775 Value *And = Builder.CreateAnd(LHS: remap(AsInt), RHS: Mask, Name: "and");
776 return Builder.CreateIntToPtr(
777 V: And, DestTy: PointerType::getUnqual(C&: ValTy->getContext()), Name: "itp");
778}
779
780auto AlignVectors::createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr,
781 Value *Predicate, int Alignment, Value *Mask,
782 Value *PassThru,
783 ArrayRef<Value *> MDSources) const -> Value * {
784 // Predicate is nullptr if not creating predicated load
785 if (Predicate) {
786 assert(!Predicate->getType()->isVectorTy() &&
787 "Expectning scalar predicate");
788 if (HVC.isFalse(Val: Predicate))
789 return UndefValue::get(T: ValTy);
790 if (!HVC.isTrue(Val: Predicate)) {
791 Value *Load = createPredicatedLoad(Builder, ValTy, Ptr, Predicate,
792 Alignment, MDSources);
793 return Builder.CreateSelect(C: Mask, True: Load, False: PassThru);
794 }
795 // Predicate == true here.
796 }
797 assert(!HVC.isUndef(Mask)); // Should this be allowed?
798 if (HVC.isZero(Val: Mask))
799 return PassThru;
800
801 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy, Requested: Alignment);
802 if (HVC.isTrue(Val: Mask))
803 return createSimpleLoad(Builder, ValTy, Ptr, Alignment: EffA.value(), MDSources);
804
805 Instruction *Load =
806 Builder.CreateMaskedLoad(Ty: ValTy, Ptr, Alignment: EffA, Mask, PassThru, Name: "mld");
807 LLVM_DEBUG(dbgs() << "\t[Creating masked Load:] "; Load->dump());
808 propagateMetadata(I: Load, VL: MDSources);
809 return Load;
810}
811
812auto AlignVectors::createSimpleLoad(IRBuilderBase &Builder, Type *ValTy,
813 Value *Ptr, int Alignment,
814 ArrayRef<Value *> MDSources) const
815 -> Value * {
816 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy, Requested: Alignment);
817 Instruction *Load = Builder.CreateAlignedLoad(Ty: ValTy, Ptr, Align: EffA, Name: "ald");
818 propagateMetadata(I: Load, VL: MDSources);
819 LLVM_DEBUG(dbgs() << "\t[Creating Load:] "; Load->dump());
820 return Load;
821}
822
823auto AlignVectors::createPredicatedLoad(IRBuilderBase &Builder, Type *ValTy,
824 Value *Ptr, Value *Predicate,
825 int Alignment,
826 ArrayRef<Value *> MDSources) const
827 -> Value * {
828 assert(HVC.HST.isTypeForHVX(ValTy) &&
829 "Predicates 'scalar' vector loads not yet supported");
830 assert(Predicate);
831 assert(!Predicate->getType()->isVectorTy() && "Expectning scalar predicate");
832 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy, Requested: Alignment);
833 assert(HVC.getSizeOf(ValTy, HVC.Alloc) % EffA.value() == 0);
834
835 if (HVC.isFalse(Val: Predicate))
836 return UndefValue::get(T: ValTy);
837 if (HVC.isTrue(Val: Predicate))
838 return createSimpleLoad(Builder, ValTy, Ptr, Alignment: EffA.value(), MDSources);
839
840 auto V6_vL32b_pred_ai = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vL32b_pred_ai);
841 // FIXME: This may not put the offset from Ptr into the vmem offset.
842 return HVC.createHvxIntrinsic(Builder, IntID: V6_vL32b_pred_ai, RetTy: ValTy,
843 Args: {Predicate, Ptr, HVC.getConstInt(Val: 0)}, ArgTys: {},
844 MDSources);
845}
846
847auto AlignVectors::createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr,
848 Value *Predicate, int Alignment, Value *Mask,
849 ArrayRef<Value *> MDSources) const -> Value * {
850 if (HVC.isZero(Val: Mask) || HVC.isUndef(Val) || HVC.isUndef(Val: Mask))
851 return UndefValue::get(T: Val->getType());
852 assert(!Predicate || (!Predicate->getType()->isVectorTy() &&
853 "Expectning scalar predicate"));
854 if (Predicate) {
855 if (HVC.isFalse(Val: Predicate))
856 return UndefValue::get(T: Val->getType());
857 if (HVC.isTrue(Val: Predicate))
858 Predicate = nullptr;
859 }
860 // Here both Predicate and Mask are true or unknown.
861
862 if (HVC.isTrue(Val: Mask)) {
863 if (Predicate) { // Predicate unknown
864 return createPredicatedStore(Builder, Val, Ptr, Predicate, Alignment,
865 MDSources);
866 }
867 // Predicate is true:
868 return createSimpleStore(Builder, Val, Ptr, Alignment, MDSources);
869 }
870
871 // Mask is unknown
872 if (!Predicate) {
873 Instruction *Store =
874 Builder.CreateMaskedStore(Val, Ptr, Alignment: Align(Alignment), Mask);
875 propagateMetadata(I: Store, VL: MDSources);
876 return Store;
877 }
878
879 // Both Predicate and Mask are unknown.
880 // Emulate masked store with predicated-load + mux + predicated-store.
881 Value *PredLoad = createPredicatedLoad(Builder, ValTy: Val->getType(), Ptr,
882 Predicate, Alignment, MDSources);
883 Value *Mux = Builder.CreateSelect(C: Mask, True: Val, False: PredLoad);
884 return createPredicatedStore(Builder, Val: Mux, Ptr, Predicate, Alignment,
885 MDSources);
886}
887
888auto AlignVectors::createSimpleStore(IRBuilderBase &Builder, Value *Val,
889 Value *Ptr, int Alignment,
890 ArrayRef<Value *> MDSources) const
891 -> Value * {
892 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy: Val->getType(), Requested: Alignment);
893 Instruction *Store = Builder.CreateAlignedStore(Val, Ptr, Align: EffA);
894 LLVM_DEBUG(dbgs() << "\t[Creating store:] "; Store->dump());
895 propagateMetadata(I: Store, VL: MDSources);
896 return Store;
897}
898
899auto AlignVectors::createPredicatedStore(IRBuilderBase &Builder, Value *Val,
900 Value *Ptr, Value *Predicate,
901 int Alignment,
902 ArrayRef<Value *> MDSources) const
903 -> Value * {
904 Align EffA = effectiveAlignForValueTy(DL: HVC.DL, ValTy: Val->getType(), Requested: Alignment);
905 assert(HVC.HST.isTypeForHVX(Val->getType()) &&
906 "Predicates 'scalar' vector stores not yet supported");
907 assert(Predicate);
908 if (HVC.isFalse(Val: Predicate))
909 return UndefValue::get(T: Val->getType());
910 if (HVC.isTrue(Val: Predicate))
911 return createSimpleStore(Builder, Val, Ptr, Alignment: EffA.value(), MDSources);
912
913 assert(HVC.getSizeOf(Val, HVC.Alloc) % EffA.value() == 0);
914 auto V6_vS32b_pred_ai = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vS32b_pred_ai);
915 // FIXME: This may not put the offset from Ptr into the vmem offset.
916 return HVC.createHvxIntrinsic(Builder, IntID: V6_vS32b_pred_ai, RetTy: nullptr,
917 Args: {Predicate, Ptr, HVC.getConstInt(Val: 0), Val}, ArgTys: {},
918 MDSources);
919}
920
921auto AlignVectors::getUpwardDeps(Instruction *In, Instruction *Base) const
922 -> DepList {
923 BasicBlock *Parent = Base->getParent();
924 assert(In->getParent() == Parent &&
925 "Base and In should be in the same block");
926 assert(Base->comesBefore(In) && "Base should come before In");
927
928 DepList Deps;
929 std::deque<Instruction *> WorkQ = {In};
930 while (!WorkQ.empty()) {
931 Instruction *D = WorkQ.front();
932 WorkQ.pop_front();
933 if (D != In)
934 Deps.insert(x: D);
935 for (Value *Op : D->operands()) {
936 if (auto *I = dyn_cast<Instruction>(Val: Op)) {
937 if (I->getParent() == Parent && Base->comesBefore(Other: I))
938 WorkQ.push_back(x: I);
939 }
940 }
941 }
942 return Deps;
943}
944
945auto AlignVectors::createAddressGroups() -> bool {
946 // An address group created here may contain instructions spanning
947 // multiple basic blocks.
948 AddrList WorkStack;
949
950 auto findBaseAndOffset = [&](AddrInfo &AI) -> std::pair<Instruction *, int> {
951 for (AddrInfo &W : WorkStack) {
952 if (auto D = HVC.calculatePointerDifference(Ptr0: AI.Addr, Ptr1: W.Addr))
953 return std::make_pair(x&: W.Inst, y&: *D);
954 }
955 return std::make_pair(x: nullptr, y: 0);
956 };
957
958 auto traverseBlock = [&](DomTreeNode *DomN, auto Visit) -> void {
959 BasicBlock &Block = *DomN->getBlock();
960 for (Instruction &I : Block) {
961 auto AI = this->getAddrInfo(In&: I); // Use this-> for gcc6.
962 if (!AI)
963 continue;
964 auto F = findBaseAndOffset(*AI);
965 Instruction *GroupInst;
966 if (Instruction *BI = F.first) {
967 AI->Offset = F.second;
968 GroupInst = BI;
969 } else {
970 WorkStack.push_back(x: *AI);
971 GroupInst = AI->Inst;
972 }
973 AddrGroups[GroupInst].push_back(x: *AI);
974 }
975
976 for (DomTreeNode *C : DomN->children())
977 Visit(C, Visit);
978
979 while (!WorkStack.empty() && WorkStack.back().Inst->getParent() == &Block)
980 WorkStack.pop_back();
981 };
982
983 traverseBlock(HVC.DT.getRootNode(), traverseBlock);
984 assert(WorkStack.empty());
985
986 // AddrGroups are formed.
987 // Remove groups of size 1.
988 AddrGroups.remove_if(Pred: [](auto &G) { return G.second.size() == 1; });
989 // Remove groups that don't use HVX types.
990 AddrGroups.remove_if(Pred: [&](auto &G) {
991 return llvm::none_of(
992 G.second, [&](auto &I) { return HVC.HST.isTypeForHVX(VecTy: I.ValTy); });
993 });
994
995 LLVM_DEBUG(dbgs() << AddrGroups);
996 return !AddrGroups.empty();
997}
998
999auto AlignVectors::createLoadGroups(const AddrList &Group) const -> MoveList {
1000 // Form load groups.
1001 // To avoid complications with moving code across basic blocks, only form
1002 // groups that are contained within a single basic block.
1003 unsigned SizeLimit = VAGroupSizeLimit;
1004 if (SizeLimit == 0)
1005 return {};
1006
1007 auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) {
1008 assert(!Move.Main.empty() && "Move group should have non-empty Main");
1009 if (Move.Main.size() >= SizeLimit) {
1010 HVC.ORE.emit(RemarkBuilder: [&]() {
1011 return OptimizationRemarkMissed(DEBUG_TYPE, "GroupSizeLimitExceeded",
1012 Info.Inst->getDebugLoc(),
1013 Info.Inst->getParent())
1014 << "alignment group exceeds size limit";
1015 });
1016 return false;
1017 }
1018 // Don't mix HVX and non-HVX instructions.
1019 if (Move.IsHvx != isHvx(AI: Info))
1020 return false;
1021 // Leading instruction in the load group.
1022 Instruction *Base = Move.Main.front();
1023 if (Base->getParent() != Info.Inst->getParent())
1024 return false;
1025 // Check if it's safe to move the load.
1026 if (!HVC.isSafeToMoveBeforeInBB(In: *Info.Inst, To: Base->getIterator())) {
1027 HVC.ORE.emit(RemarkBuilder: [&]() {
1028 return OptimizationRemarkMissed(DEBUG_TYPE, "UnsafeToRelocate",
1029 Info.Inst->getDebugLoc(),
1030 Info.Inst->getParent())
1031 << "unsafe to relocate memory access for alignment";
1032 });
1033 return false;
1034 }
1035 // And if it's safe to clone the dependencies.
1036 auto isSafeToCopyAtBase = [&](const Instruction *I) {
1037 return HVC.isSafeToMoveBeforeInBB(In: *I, To: Base->getIterator()) &&
1038 HVC.isSafeToClone(In: *I);
1039 };
1040 DepList Deps = getUpwardDeps(In: Info.Inst, Base);
1041 if (!llvm::all_of(Range&: Deps, P: isSafeToCopyAtBase))
1042 return false;
1043
1044 Move.Main.push_back(x: Info.Inst);
1045 llvm::append_range(C&: Move.Deps, R&: Deps);
1046 return true;
1047 };
1048
1049 MoveList LoadGroups;
1050
1051 for (const AddrInfo &Info : Group) {
1052 if (!Info.Inst->mayReadFromMemory())
1053 continue;
1054 if (LoadGroups.empty() || !tryAddTo(Info, LoadGroups.back()))
1055 LoadGroups.emplace_back(args: Info, args: Group.front().Inst, args: isHvx(AI: Info), args: true);
1056 }
1057
1058 // Erase groups smaller than the minimum load group size.
1059 unsigned LoadGroupSizeLimit = MinLoadGroupSizeForAlignment;
1060 erase_if(container&: LoadGroups, p: [LoadGroupSizeLimit](const MoveGroup &G) {
1061 return G.Main.size() < LoadGroupSizeLimit;
1062 });
1063
1064 // Erase HVX groups on targets < HvxV62 (due to lack of predicated loads).
1065 if (!HVC.HST.useHVXV62Ops()) {
1066 bool HadHvx =
1067 llvm::any_of(Range&: LoadGroups, P: [](const MoveGroup &G) { return G.IsHvx; });
1068 erase_if(container&: LoadGroups, p: [](const MoveGroup &G) { return G.IsHvx; });
1069 if (HadHvx) {
1070 HVC.ORE.emit(RemarkBuilder: [&]() {
1071 return OptimizationRemarkMissed(DEBUG_TYPE, "HvxVersionTooLow",
1072 HVC.F.getSubprogram(), &HVC.F.front())
1073 << "HVX version too low for predicated load operations";
1074 });
1075 }
1076 }
1077
1078 LLVM_DEBUG(dbgs() << "LoadGroups list: " << LoadGroups);
1079 return LoadGroups;
1080}
1081
1082auto AlignVectors::createStoreGroups(const AddrList &Group) const -> MoveList {
1083 // Form store groups.
1084 // To avoid complications with moving code across basic blocks, only form
1085 // groups that are contained within a single basic block.
1086 unsigned SizeLimit = VAGroupSizeLimit;
1087 if (SizeLimit == 0)
1088 return {};
1089
1090 auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) {
1091 assert(!Move.Main.empty() && "Move group should have non-empty Main");
1092 if (Move.Main.size() >= SizeLimit) {
1093 HVC.ORE.emit(RemarkBuilder: [&]() {
1094 return OptimizationRemarkMissed(DEBUG_TYPE, "GroupSizeLimitExceeded",
1095 Info.Inst->getDebugLoc(),
1096 Info.Inst->getParent())
1097 << "alignment group exceeds size limit";
1098 });
1099 return false;
1100 }
1101 // For stores with return values we'd have to collect downward dependencies.
1102 // There are no such stores that we handle at the moment, so omit that.
1103 assert(Info.Inst->getType()->isVoidTy() &&
1104 "Not handling stores with return values");
1105 // Don't mix HVX and non-HVX instructions.
1106 if (Move.IsHvx != isHvx(AI: Info))
1107 return false;
1108 // For stores we need to be careful whether it's safe to move them.
1109 // Stores that are otherwise safe to move together may not appear safe
1110 // to move over one another (i.e. isSafeToMoveBefore may return false).
1111 Instruction *Base = Move.Main.front();
1112 if (Base->getParent() != Info.Inst->getParent())
1113 return false;
1114 if (!HVC.isSafeToMoveBeforeInBB(In: *Info.Inst, To: Base->getIterator(),
1115 IgnoreInsts: Move.Main)) {
1116 HVC.ORE.emit(RemarkBuilder: [&]() {
1117 return OptimizationRemarkMissed(DEBUG_TYPE, "UnsafeToRelocate",
1118 Info.Inst->getDebugLoc(),
1119 Info.Inst->getParent())
1120 << "unsafe to relocate memory access for alignment";
1121 });
1122 return false;
1123 }
1124 Move.Main.push_back(x: Info.Inst);
1125 return true;
1126 };
1127
1128 MoveList StoreGroups;
1129
1130 for (auto I = Group.rbegin(), E = Group.rend(); I != E; ++I) {
1131 const AddrInfo &Info = *I;
1132 if (!Info.Inst->mayWriteToMemory())
1133 continue;
1134 if (StoreGroups.empty() || !tryAddTo(Info, StoreGroups.back()))
1135 StoreGroups.emplace_back(args: Info, args: Group.front().Inst, args: isHvx(AI: Info), args: false);
1136 }
1137
1138 // Erase singleton groups.
1139 erase_if(container&: StoreGroups, p: [](const MoveGroup &G) { return G.Main.size() <= 1; });
1140
1141 // Erase HVX groups on targets < HvxV62 (due to lack of predicated loads).
1142 if (!HVC.HST.useHVXV62Ops()) {
1143 bool HadHvx =
1144 llvm::any_of(Range&: StoreGroups, P: [](const MoveGroup &G) { return G.IsHvx; });
1145 erase_if(container&: StoreGroups, p: [](const MoveGroup &G) { return G.IsHvx; });
1146 if (HadHvx) {
1147 HVC.ORE.emit(RemarkBuilder: [&]() {
1148 return OptimizationRemarkMissed(DEBUG_TYPE, "HvxVersionTooLow",
1149 HVC.F.getSubprogram(), &HVC.F.front())
1150 << "HVX version too low for predicated store operations";
1151 });
1152 }
1153 }
1154
1155 // Erase groups where every store is a full HVX vector. The reason is that
1156 // aligning predicated stores generates complex code that may be less
1157 // efficient than a sequence of unaligned vector stores.
1158 if (!VADoFullStores) {
1159 erase_if(container&: StoreGroups, p: [this](const MoveGroup &G) {
1160 return G.IsHvx && llvm::all_of(Range: G.Main, P: [this](Instruction *S) {
1161 auto MaybeInfo = this->getAddrInfo(In&: *S);
1162 assert(MaybeInfo.has_value());
1163 return HVC.HST.isHVXVectorType(
1164 VecTy: EVT::getEVT(Ty: MaybeInfo->ValTy, HandleUnknown: false));
1165 });
1166 });
1167 }
1168
1169 return StoreGroups;
1170}
1171
1172auto AlignVectors::moveTogether(MoveGroup &Move) const -> bool {
1173 // Move all instructions to be adjacent.
1174 assert(!Move.Main.empty() && "Move group should have non-empty Main");
1175 Instruction *Where = Move.Main.front();
1176
1177 if (Move.IsLoad) {
1178 // Move all the loads (and dependencies) to where the first load is.
1179 // Clone all deps to before Where, keeping order.
1180 Move.Clones = cloneBefore(To: Where->getIterator(), Insts&: Move.Deps);
1181 // Move all main instructions to after Where, keeping order.
1182 ArrayRef<Instruction *> Main(Move.Main);
1183 for (Instruction *M : Main) {
1184 if (M != Where)
1185 M->moveAfter(MovePos: Where);
1186 for (auto [Old, New] : Move.Clones)
1187 M->replaceUsesOfWith(From: Old, To: New);
1188 Where = M;
1189 }
1190 // Replace Deps with the clones.
1191 for (int i = 0, e = Move.Deps.size(); i != e; ++i)
1192 Move.Deps[i] = Move.Clones[Move.Deps[i]];
1193 } else {
1194 // Move all the stores to where the last store is.
1195 // NOTE: Deps are empty for "store" groups. If they need to be
1196 // non-empty, decide on the order.
1197 assert(Move.Deps.empty());
1198 // Move all main instructions to before Where, inverting order.
1199 ArrayRef<Instruction *> Main(Move.Main);
1200 for (Instruction *M : Main.drop_front(N: 1)) {
1201 M->moveBefore(InsertPos: Where->getIterator());
1202 Where = M;
1203 }
1204 }
1205
1206 return Move.Main.size() + Move.Deps.size() > 1;
1207}
1208
1209template <typename T>
1210auto AlignVectors::cloneBefore(BasicBlock::iterator To, T &&Insts) const
1211 -> InstMap {
1212 InstMap Map;
1213
1214 for (Instruction *I : Insts) {
1215 assert(HVC.isSafeToClone(*I));
1216 Instruction *C = I->clone();
1217 C->setName(Twine("c.") + I->getName() + ".");
1218 C->insertBefore(InsertPos: To);
1219
1220 for (auto [Old, New] : Map)
1221 C->replaceUsesOfWith(From: Old, To: New);
1222 Map.insert(KV: std::make_pair(x&: I, y&: C));
1223 }
1224 return Map;
1225}
1226
1227auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder,
1228 const ByteSpan &VSpan, int ScLen,
1229 Value *AlignVal, Value *AlignAddr) const
1230 -> void {
1231 LLVM_DEBUG(dbgs() << __func__ << "\n");
1232
1233 Type *SecTy = HVC.getByteTy(ElemCount: ScLen);
1234 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1235 bool DoAlign = !HVC.isZero(Val: AlignVal);
1236 BasicBlock::iterator BasePos = Builder.GetInsertPoint();
1237 BasicBlock *BaseBlock = Builder.GetInsertBlock();
1238
1239 ByteSpan ASpan;
1240 auto *True = Constant::getAllOnesValue(Ty: HVC.getBoolTy(ElemCount: ScLen));
1241 auto *Undef = UndefValue::get(T: SecTy);
1242
1243 // Created load does not have to be "Instruction" (e.g. "undef").
1244 SmallVector<Value *> Loads(NumSectors + DoAlign, nullptr);
1245
1246 // We could create all of the aligned loads, and generate the valigns
1247 // at the location of the first load, but for large load groups, this
1248 // could create highly suboptimal code (there have been groups of 140+
1249 // loads in real code).
1250 // Instead, place the loads/valigns as close to the users as possible.
1251 // In any case we need to have a mapping from the blocks of VSpan (the
1252 // span covered by the pre-existing loads) to ASpan (the span covered
1253 // by the aligned loads). There is a small problem, though: ASpan needs
1254 // to have pointers to the loads/valigns, but we don't have these loads
1255 // because we don't know where to put them yet. We find out by creating
1256 // a section of ASpan that corresponds to values (blocks) from VSpan,
1257 // and checking where the new load should be placed. We need to attach
1258 // this location information to each block in ASpan somehow, so we put
1259 // distincts values for Seg.Val in each ASpan.Blocks[i], and use a map
1260 // to store the location for each Seg.Val.
1261 // The distinct values happen to be Blocks[i].Seg.Val = &Blocks[i],
1262 // which helps with printing ByteSpans without crashing when printing
1263 // Segments with these temporary identifiers in place of Val.
1264
1265 // Populate the blocks first, to avoid reallocations of the vector
1266 // interfering with generating the placeholder addresses.
1267 for (int Index = 0; Index != NumSectors; ++Index)
1268 ASpan.Blocks.emplace_back(args: nullptr, args&: ScLen, args: Index * ScLen);
1269 for (int Index = 0; Index != NumSectors; ++Index) {
1270 ASpan.Blocks[Index].Seg.Val =
1271 reinterpret_cast<Value *>(&ASpan.Blocks[Index]);
1272 }
1273
1274 // Multiple values from VSpan can map to the same value in ASpan. Since we
1275 // try to create loads lazily, we need to find the earliest use for each
1276 // value from ASpan.
1277 DenseMap<void *, Instruction *> EarliestUser;
1278 auto isEarlier = [](Instruction *A, Instruction *B) {
1279 if (B == nullptr)
1280 return true;
1281 if (A == nullptr)
1282 return false;
1283 assert(A->getParent() == B->getParent());
1284 return A->comesBefore(Other: B);
1285 };
1286 auto earliestUser = [&](const auto &Uses) {
1287 Instruction *User = nullptr;
1288 for (const Use &U : Uses) {
1289 auto *I = dyn_cast<Instruction>(Val: U.getUser());
1290 assert(I != nullptr && "Load used in a non-instruction?");
1291 // Make sure we only consider users in this block, but we need
1292 // to remember if there were users outside the block too. This is
1293 // because if no users are found, aligned loads will not be created.
1294 if (I->getParent() == BaseBlock) {
1295 if (!isa<PHINode>(Val: I))
1296 User = std::min(a: User, b: I, comp: isEarlier);
1297 } else {
1298 User = std::min(a: User, b: BaseBlock->getTerminator(), comp: isEarlier);
1299 }
1300 }
1301 return User;
1302 };
1303
1304 for (const ByteSpan::Block &B : VSpan) {
1305 ByteSpan ASection = ASpan.section(Start: B.Pos, Length: B.Seg.Size);
1306 for (const ByteSpan::Block &S : ASection) {
1307 auto &EU = EarliestUser[S.Seg.Val];
1308 EU = std::min(a: EU, b: earliestUser(B.Seg.Val->uses()), comp: isEarlier);
1309 }
1310 }
1311
1312 LLVM_DEBUG({
1313 dbgs() << "ASpan:\n" << ASpan << '\n';
1314 dbgs() << "Earliest users of ASpan:\n";
1315 for (auto &[Val, User] : EarliestUser) {
1316 dbgs() << Val << "\n ->" << *User << '\n';
1317 }
1318 });
1319
1320 auto createLoad = [&](IRBuilderBase &Builder, const ByteSpan &VSpan,
1321 int Index, bool MakePred) {
1322 Value *Ptr =
1323 createAdjustedPointer(Builder, Ptr: AlignAddr, ValTy: SecTy, Adjust: Index * ScLen);
1324 Value *Predicate =
1325 MakePred ? makeTestIfUnaligned(Builder, AlignVal, Alignment: ScLen) : nullptr;
1326
1327 // If vector shifting is potentially needed, accumulate metadata
1328 // from source sections of twice the load width.
1329 int Start = (Index - DoAlign) * ScLen;
1330 int Width = (1 + DoAlign) * ScLen;
1331 return this->createLoad(Builder, ValTy: SecTy, Ptr, Predicate, Alignment: ScLen, Mask: True, PassThru: Undef,
1332 MDSources: VSpan.section(Start, Length: Width).values());
1333 };
1334
1335 auto moveBefore = [this](BasicBlock::iterator In, BasicBlock::iterator To) {
1336 // Move In and its upward dependencies to before To.
1337 assert(In->getParent() == To->getParent());
1338 DepList Deps = getUpwardDeps(In: &*In, Base: &*To);
1339 In->moveBefore(InsertPos: To);
1340 // DepList is sorted with respect to positions in the basic block.
1341 InstMap Map = cloneBefore(To: In, Insts&: Deps);
1342 for (auto [Old, New] : Map)
1343 In->replaceUsesOfWith(From: Old, To: New);
1344 };
1345
1346 // Generate necessary loads at appropriate locations.
1347 LLVM_DEBUG(dbgs() << "Creating loads for ASpan sectors\n");
1348 for (int Index = 0; Index != NumSectors + 1; ++Index) {
1349 // In ASpan, each block will be either a single aligned load, or a
1350 // valign of a pair of loads. In the latter case, an aligned load j
1351 // will belong to the current valign, and the one in the previous
1352 // block (for j > 0).
1353 // Place the load at a location which will dominate the valign, assuming
1354 // the valign will be placed right before the earliest user.
1355 Instruction *PrevAt =
1356 DoAlign && Index > 0 ? EarliestUser[&ASpan[Index - 1]] : nullptr;
1357 Instruction *ThisAt =
1358 Index < NumSectors ? EarliestUser[&ASpan[Index]] : nullptr;
1359 if (auto *Where = std::min(a: PrevAt, b: ThisAt, comp: isEarlier)) {
1360 Builder.SetInsertPoint(Where);
1361 Loads[Index] =
1362 createLoad(Builder, VSpan, Index, DoAlign && Index == NumSectors);
1363 // We know it's safe to put the load at BasePos, but we'd prefer to put
1364 // it at "Where". To see if the load is safe to be placed at Where, put
1365 // it there first and then check if it's safe to move it to BasePos.
1366 // If not, then the load needs to be placed at BasePos.
1367 // We can't do this check proactively because we need the load to exist
1368 // in order to check legality.
1369 if (auto *Load = dyn_cast<Instruction>(Val: Loads[Index])) {
1370 if (!HVC.isSafeToMoveBeforeInBB(In: *Load, To: BasePos))
1371 moveBefore(Load->getIterator(), BasePos);
1372 }
1373 LLVM_DEBUG(dbgs() << "Loads[" << Index << "]:" << *Loads[Index] << '\n');
1374 }
1375 }
1376
1377 // Generate valigns if needed, and fill in proper values in ASpan
1378 LLVM_DEBUG(dbgs() << "Creating values for ASpan sectors\n");
1379 for (int Index = 0; Index != NumSectors; ++Index) {
1380 ASpan[Index].Seg.Val = nullptr;
1381 if (auto *Where = EarliestUser[&ASpan[Index]]) {
1382 Builder.SetInsertPoint(Where);
1383 Value *Val = Loads[Index];
1384 assert(Val != nullptr);
1385 if (DoAlign) {
1386 Value *NextLoad = Loads[Index + 1];
1387 assert(NextLoad != nullptr);
1388 Val = HVC.vralignb(Builder, Lo: Val, Hi: NextLoad, Amt: AlignVal);
1389 }
1390 ASpan[Index].Seg.Val = Val;
1391 LLVM_DEBUG(dbgs() << "ASpan[" << Index << "]:" << *Val << '\n');
1392 }
1393 }
1394
1395 for (const ByteSpan::Block &B : VSpan) {
1396 ByteSpan ASection = ASpan.section(Start: B.Pos, Length: B.Seg.Size).shift(Offset: -B.Pos);
1397 Value *Accum = UndefValue::get(T: HVC.getByteTy(ElemCount: B.Seg.Size));
1398 Builder.SetInsertPoint(cast<Instruction>(Val: B.Seg.Val));
1399
1400 // We're generating a reduction, where each instruction depends on
1401 // the previous one, so we need to order them according to the position
1402 // of their inputs in the code.
1403 std::vector<ByteSpan::Block *> ABlocks;
1404 for (ByteSpan::Block &S : ASection) {
1405 if (S.Seg.Val != nullptr)
1406 ABlocks.push_back(x: &S);
1407 }
1408 llvm::sort(C&: ABlocks,
1409 Comp: [&](const ByteSpan::Block *A, const ByteSpan::Block *B) {
1410 return isEarlier(cast<Instruction>(Val: A->Seg.Val),
1411 cast<Instruction>(Val: B->Seg.Val));
1412 });
1413 for (ByteSpan::Block *S : ABlocks) {
1414 // The processing of the data loaded by the aligned loads
1415 // needs to be inserted after the data is available.
1416 Instruction *SegI = cast<Instruction>(Val: S->Seg.Val);
1417 Builder.SetInsertPoint(&*std::next(x: SegI->getIterator()));
1418 Value *Pay = HVC.vbytes(Builder, Val: getPayload(Val: S->Seg.Val));
1419 Accum =
1420 HVC.insertb(Builder, Dest: Accum, Src: Pay, Start: S->Seg.Start, Length: S->Seg.Size, Where: S->Pos);
1421 }
1422 // Instead of casting everything to bytes for the vselect, cast to the
1423 // original value type. This will avoid complications with casting masks.
1424 // For example, in cases when the original mask applied to i32, it could
1425 // be converted to a mask applicable to i8 via pred_typecast intrinsic,
1426 // but if the mask is not exactly of HVX length, extra handling would be
1427 // needed to make it work.
1428 Type *ValTy = getPayload(Val: B.Seg.Val)->getType();
1429 Value *Cast = Builder.CreateBitCast(V: Accum, DestTy: ValTy, Name: "cst");
1430 Value *Sel = Builder.CreateSelect(C: getMask(Val: B.Seg.Val), True: Cast,
1431 False: getPassThrough(Val: B.Seg.Val), Name: "sel");
1432 B.Seg.Val->replaceAllUsesWith(V: Sel);
1433 }
1434}
1435
1436auto AlignVectors::realignStoreGroup(IRBuilderBase &Builder,
1437 const ByteSpan &VSpan, int ScLen,
1438 Value *AlignVal, Value *AlignAddr) const
1439 -> void {
1440 LLVM_DEBUG(dbgs() << __func__ << "\n");
1441
1442 Type *SecTy = HVC.getByteTy(ElemCount: ScLen);
1443 int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
1444 bool DoAlign = !HVC.isZero(Val: AlignVal);
1445
1446 // Stores.
1447 ByteSpan ASpanV, ASpanM;
1448
1449 // Return a vector value corresponding to the input value Val:
1450 // either <1 x Val> for scalar Val, or Val itself for vector Val.
1451 auto MakeVec = [](IRBuilderBase &Builder, Value *Val) -> Value * {
1452 Type *Ty = Val->getType();
1453 if (Ty->isVectorTy())
1454 return Val;
1455 auto *VecTy = VectorType::get(ElementType: Ty, NumElements: 1, /*Scalable=*/false);
1456 return Builder.CreateBitCast(V: Val, DestTy: VecTy, Name: "cst");
1457 };
1458
1459 // Create an extra "undef" sector at the beginning and at the end.
1460 // They will be used as the left/right filler in the vlalign step.
1461 for (int Index = (DoAlign ? -1 : 0); Index != NumSectors + DoAlign; ++Index) {
1462 // For stores, the size of each section is an aligned vector length.
1463 // Adjust the store offsets relative to the section start offset.
1464 ByteSpan VSection =
1465 VSpan.section(Start: Index * ScLen, Length: ScLen).shift(Offset: -Index * ScLen);
1466 Value *Undef = UndefValue::get(T: SecTy);
1467 Value *Zero = Constant::getNullValue(Ty: SecTy);
1468 Value *AccumV = Undef;
1469 Value *AccumM = Zero;
1470 for (ByteSpan::Block &S : VSection) {
1471 Value *Pay = getPayload(Val: S.Seg.Val);
1472 Value *Mask = HVC.rescale(Builder, Mask: MakeVec(Builder, getMask(Val: S.Seg.Val)),
1473 FromTy: Pay->getType(), ToTy: HVC.getByteTy());
1474 Value *PartM = HVC.insertb(Builder, Dest: Zero, Src: HVC.vbytes(Builder, Val: Mask),
1475 Start: S.Seg.Start, Length: S.Seg.Size, Where: S.Pos);
1476 AccumM = Builder.CreateOr(LHS: AccumM, RHS: PartM);
1477
1478 Value *PartV = HVC.insertb(Builder, Dest: Undef, Src: HVC.vbytes(Builder, Val: Pay),
1479 Start: S.Seg.Start, Length: S.Seg.Size, Where: S.Pos);
1480
1481 AccumV = Builder.CreateSelect(
1482 C: Builder.CreateICmp(P: CmpInst::ICMP_NE, LHS: PartM, RHS: Zero), True: PartV, False: AccumV);
1483 }
1484 ASpanV.Blocks.emplace_back(args&: AccumV, args&: ScLen, args: Index * ScLen);
1485 ASpanM.Blocks.emplace_back(args&: AccumM, args&: ScLen, args: Index * ScLen);
1486 }
1487
1488 LLVM_DEBUG({
1489 dbgs() << "ASpanV before vlalign:\n" << ASpanV << '\n';
1490 dbgs() << "ASpanM before vlalign:\n" << ASpanM << '\n';
1491 });
1492
1493 // vlalign
1494 if (DoAlign) {
1495 for (int Index = 1; Index != NumSectors + 2; ++Index) {
1496 Value *PrevV = ASpanV[Index - 1].Seg.Val, *ThisV = ASpanV[Index].Seg.Val;
1497 Value *PrevM = ASpanM[Index - 1].Seg.Val, *ThisM = ASpanM[Index].Seg.Val;
1498 assert(isSectorTy(PrevV->getType()) && isSectorTy(PrevM->getType()));
1499 ASpanV[Index - 1].Seg.Val = HVC.vlalignb(Builder, Lo: PrevV, Hi: ThisV, Amt: AlignVal);
1500 ASpanM[Index - 1].Seg.Val = HVC.vlalignb(Builder, Lo: PrevM, Hi: ThisM, Amt: AlignVal);
1501 }
1502 }
1503
1504 LLVM_DEBUG({
1505 dbgs() << "ASpanV after vlalign:\n" << ASpanV << '\n';
1506 dbgs() << "ASpanM after vlalign:\n" << ASpanM << '\n';
1507 });
1508
1509 auto createStore = [&](IRBuilderBase &Builder, const ByteSpan &ASpanV,
1510 const ByteSpan &ASpanM, int Index, bool MakePred) {
1511 Value *Val = ASpanV[Index].Seg.Val;
1512 Value *Mask = ASpanM[Index].Seg.Val; // bytes
1513 if (HVC.isUndef(Val) || HVC.isZero(Val: Mask))
1514 return;
1515 Value *Ptr =
1516 createAdjustedPointer(Builder, Ptr: AlignAddr, ValTy: SecTy, Adjust: Index * ScLen);
1517 Value *Predicate =
1518 MakePred ? makeTestIfUnaligned(Builder, AlignVal, Alignment: ScLen) : nullptr;
1519
1520 // If vector shifting is potentially needed, accumulate metadata
1521 // from source sections of twice the store width.
1522 int Start = (Index - DoAlign) * ScLen;
1523 int Width = (1 + DoAlign) * ScLen;
1524 this->createStore(Builder, Val, Ptr, Predicate, Alignment: ScLen,
1525 Mask: HVC.vlsb(Builder, Val: Mask),
1526 MDSources: VSpan.section(Start, Length: Width).values());
1527 };
1528
1529 for (int Index = 0; Index != NumSectors + DoAlign; ++Index) {
1530 createStore(Builder, ASpanV, ASpanM, Index, DoAlign && Index == NumSectors);
1531 }
1532}
1533
1534auto AlignVectors::realignGroup(const MoveGroup &Move) -> bool {
1535 LLVM_DEBUG(dbgs() << "Realigning group:\n" << Move << '\n');
1536
1537 // TODO: Needs support for masked loads/stores of "scalar" vectors.
1538 if (!Move.IsHvx)
1539 return false;
1540
1541 // Return the element with the maximum alignment from Range,
1542 // where GetValue obtains the value to compare from an element.
1543 auto getMaxOf = [](auto Range, auto GetValue) {
1544 return *llvm::max_element(Range, [&GetValue](auto &A, auto &B) {
1545 return GetValue(A) < GetValue(B);
1546 });
1547 };
1548
1549 AddrList &BaseInfos = AddrGroups[Move.Base];
1550
1551 // Conceptually, there is a vector of N bytes covering the addresses
1552 // starting from the minimum offset (i.e. Base.Addr+Start). This vector
1553 // represents a contiguous memory region that spans all accessed memory
1554 // locations.
1555 // The correspondence between loaded or stored values will be expressed
1556 // in terms of this vector. For example, the 0th element of the vector
1557 // from the Base address info will start at byte Start from the beginning
1558 // of this conceptual vector.
1559 //
1560 // This vector will be loaded/stored starting at the nearest down-aligned
1561 // address and the amount of the down-alignment will be AlignVal:
1562 // valign(load_vector(align_down(Base+Start)), AlignVal)
1563
1564 std::set<Instruction *> TestSet(Move.Main.begin(), Move.Main.end());
1565 AddrList MoveInfos;
1566
1567 llvm::copy_if(
1568 Range&: BaseInfos, Out: std::back_inserter(x&: MoveInfos),
1569 P: [&TestSet](const AddrInfo &AI) { return TestSet.count(x: AI.Inst); });
1570
1571 // Maximum alignment present in the whole address group.
1572 const AddrInfo &WithMaxAlign =
1573 getMaxOf(MoveInfos, [](const AddrInfo &AI) { return AI.HaveAlign; });
1574 Align MaxGiven = WithMaxAlign.HaveAlign;
1575
1576 // Minimum alignment present in the move address group.
1577 const AddrInfo &WithMinOffset =
1578 getMaxOf(MoveInfos, [](const AddrInfo &AI) { return -AI.Offset; });
1579
1580 const AddrInfo &WithMaxNeeded =
1581 getMaxOf(MoveInfos, [](const AddrInfo &AI) { return AI.NeedAlign; });
1582 Align MinNeeded = WithMaxNeeded.NeedAlign;
1583
1584 // Set the builder's insertion point right before the load group, or
1585 // immediately after the store group. (Instructions in a store group are
1586 // listed in reverse order.)
1587 Instruction *InsertAt = Move.Main.front();
1588 if (!Move.IsLoad) {
1589 // There should be a terminator (which store isn't, but check anyways).
1590 assert(InsertAt->getIterator() != InsertAt->getParent()->end());
1591 InsertAt = &*std::next(x: InsertAt->getIterator());
1592 }
1593
1594 IRBuilder Builder(InsertAt->getParent(), InsertAt->getIterator(),
1595 InstSimplifyFolder(HVC.DL));
1596 Value *AlignAddr = nullptr; // Actual aligned address.
1597 Value *AlignVal = nullptr; // Right-shift amount (for valign).
1598
1599 if (MinNeeded <= MaxGiven) {
1600 int Start = WithMinOffset.Offset;
1601 int OffAtMax = WithMaxAlign.Offset;
1602 // Shift the offset of the maximally aligned instruction (OffAtMax)
1603 // back by just enough multiples of the required alignment to cover the
1604 // distance from Start to OffAtMax.
1605 // Calculate the address adjustment amount based on the address with the
1606 // maximum alignment. This is to allow a simple gep instruction instead
1607 // of potential bitcasts to i8*.
1608 int Adjust = -alignTo(Value: OffAtMax - Start, Align: MinNeeded.value());
1609 AlignAddr = createAdjustedPointer(Builder, Ptr: WithMaxAlign.Addr,
1610 ValTy: WithMaxAlign.ValTy, Adjust, CloneMap: Move.Clones);
1611 int Diff = Start - (OffAtMax + Adjust);
1612 AlignVal = HVC.getConstInt(Val: Diff);
1613 assert(Diff >= 0);
1614 assert(static_cast<decltype(MinNeeded.value())>(Diff) < MinNeeded.value());
1615 } else {
1616 // WithMinOffset is the lowest address in the group,
1617 // WithMinOffset.Addr = Base+Start.
1618 // Align instructions for both HVX (V6_valign) and scalar (S2_valignrb)
1619 // mask off unnecessary bits, so it's ok to just the original pointer as
1620 // the alignment amount.
1621 // Do an explicit down-alignment of the address to avoid creating an
1622 // aligned instruction with an address that is not really aligned.
1623 AlignAddr =
1624 createAlignedPointer(Builder, Ptr: WithMinOffset.Addr, ValTy: WithMinOffset.ValTy,
1625 Alignment: MinNeeded.value(), CloneMap: Move.Clones);
1626 AlignVal =
1627 Builder.CreatePtrToInt(V: WithMinOffset.Addr, DestTy: HVC.getIntTy(), Name: "pti");
1628 if (auto *I = dyn_cast<Instruction>(Val: AlignVal)) {
1629 for (auto [Old, New] : Move.Clones)
1630 I->replaceUsesOfWith(From: Old, To: New);
1631 }
1632 }
1633
1634 ByteSpan VSpan;
1635 for (const AddrInfo &AI : MoveInfos) {
1636 VSpan.Blocks.emplace_back(args: AI.Inst, args: HVC.getSizeOf(Ty: AI.ValTy),
1637 args: AI.Offset - WithMinOffset.Offset);
1638 }
1639
1640 // The aligned loads/stores will use blocks that are either scalars,
1641 // or HVX vectors. Let "sector" be the unified term for such a block.
1642 // blend(scalar, vector) -> sector...
1643 int ScLen = Move.IsHvx ? HVC.HST.getVectorLength()
1644 : std::max<int>(a: MinNeeded.value(), b: 4);
1645 assert(!Move.IsHvx || ScLen == 64 || ScLen == 128);
1646 assert(Move.IsHvx || ScLen == 4 || ScLen == 8);
1647
1648 LLVM_DEBUG({
1649 dbgs() << "ScLen: " << ScLen << "\n";
1650 dbgs() << "AlignVal:" << *AlignVal << "\n";
1651 dbgs() << "AlignAddr:" << *AlignAddr << "\n";
1652 dbgs() << "VSpan:\n" << VSpan << '\n';
1653 });
1654
1655 if (Move.IsLoad)
1656 realignLoadGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1657 else
1658 realignStoreGroup(Builder, VSpan, ScLen, AlignVal, AlignAddr);
1659
1660 Instruction *Front = Move.Main.front();
1661 HVC.ORE.emit(RemarkBuilder: [&]() {
1662 return OptimizationRemark(DEBUG_TYPE, "VectorsAligned",
1663 Front->getDebugLoc(), Front->getParent())
1664 << "aligned vector memory operations";
1665 });
1666
1667 for (auto *Inst : Move.Main)
1668 Inst->eraseFromParent();
1669
1670 return true;
1671}
1672
1673auto AlignVectors::makeTestIfUnaligned(IRBuilderBase &Builder, Value *AlignVal,
1674 int Alignment) const -> Value * {
1675 auto *AlignTy = AlignVal->getType();
1676 Value *And = Builder.CreateAnd(
1677 LHS: AlignVal, RHS: ConstantInt::get(Ty: AlignTy, V: Alignment - 1), Name: "and");
1678 Value *Zero = ConstantInt::get(Ty: AlignTy, V: 0);
1679 return Builder.CreateICmpNE(LHS: And, RHS: Zero, Name: "isz");
1680}
1681
1682auto AlignVectors::isSectorTy(Type *Ty) const -> bool {
1683 if (!HVC.isByteVecTy(Ty))
1684 return false;
1685 int Size = HVC.getSizeOf(Ty);
1686 if (HVC.HST.isTypeForHVX(VecTy: Ty))
1687 return Size == static_cast<int>(HVC.HST.getVectorLength());
1688 return Size == 4 || Size == 8;
1689}
1690
1691auto AlignVectors::run() -> bool {
1692 LLVM_DEBUG(dbgs() << "\nRunning HVC::AlignVectors on " << HVC.F.getName()
1693 << '\n');
1694 if (!createAddressGroups())
1695 return false;
1696
1697 LLVM_DEBUG({
1698 dbgs() << "Address groups(" << AddrGroups.size() << "):\n";
1699 for (auto &[In, AL] : AddrGroups) {
1700 for (const AddrInfo &AI : AL)
1701 dbgs() << "---\n" << AI << '\n';
1702 }
1703 });
1704
1705 bool Changed = false;
1706 MoveList LoadGroups, StoreGroups;
1707
1708 for (auto &G : AddrGroups) {
1709 llvm::append_range(C&: LoadGroups, R: createLoadGroups(Group: G.second));
1710 llvm::append_range(C&: StoreGroups, R: createStoreGroups(Group: G.second));
1711 }
1712
1713 LLVM_DEBUG({
1714 dbgs() << "\nLoad groups(" << LoadGroups.size() << "):\n";
1715 for (const MoveGroup &G : LoadGroups)
1716 dbgs() << G << "\n";
1717 dbgs() << "Store groups(" << StoreGroups.size() << "):\n";
1718 for (const MoveGroup &G : StoreGroups)
1719 dbgs() << G << "\n";
1720 });
1721
1722 // Cumulative limit on the number of groups.
1723 unsigned CountLimit = VAGroupCountLimit;
1724 if (CountLimit == 0)
1725 return false;
1726
1727 if (LoadGroups.size() > CountLimit) {
1728 LoadGroups.resize(new_size: CountLimit);
1729 StoreGroups.clear();
1730 } else {
1731 unsigned StoreLimit = CountLimit - LoadGroups.size();
1732 if (StoreGroups.size() > StoreLimit)
1733 StoreGroups.resize(new_size: StoreLimit);
1734 }
1735
1736 for (auto &M : LoadGroups)
1737 Changed |= moveTogether(Move&: M);
1738 for (auto &M : StoreGroups)
1739 Changed |= moveTogether(Move&: M);
1740
1741 LLVM_DEBUG(dbgs() << "After moveTogether:\n" << HVC.F);
1742
1743 for (auto &M : LoadGroups)
1744 Changed |= realignGroup(Move: M);
1745 for (auto &M : StoreGroups)
1746 Changed |= realignGroup(Move: M);
1747
1748 return Changed;
1749}
1750
1751// --- End AlignVectors
1752
1753// --- Begin HvxIdioms
1754
1755auto HvxIdioms::getNumSignificantBits(Value *V, Instruction *In) const
1756 -> std::pair<unsigned, Signedness> {
1757 unsigned Bits = HVC.getNumSignificantBits(V, CtxI: In);
1758 // The significant bits are calculated including the sign bit. This may
1759 // add an extra bit for zero-extended values, e.g. (zext i32 to i64) may
1760 // result in 33 significant bits. To avoid extra words, skip the extra
1761 // sign bit, but keep information that the value is to be treated as
1762 // unsigned.
1763 KnownBits Known = HVC.getKnownBits(V, CtxI: In);
1764 Signedness Sign = Signed;
1765 unsigned NumToTest = 0; // Number of bits used in test for unsignedness.
1766 if (isPowerOf2_32(Value: Bits))
1767 NumToTest = Bits;
1768 else if (Bits > 1 && isPowerOf2_32(Value: Bits - 1))
1769 NumToTest = Bits - 1;
1770
1771 if (NumToTest != 0 && Known.Zero.ashr(ShiftAmt: NumToTest).isAllOnes()) {
1772 Sign = Unsigned;
1773 Bits = NumToTest;
1774 }
1775
1776 // If the top bit of the nearest power-of-2 is zero, this value is
1777 // positive. It could be treated as either signed or unsigned.
1778 if (unsigned Pow2 = PowerOf2Ceil(A: Bits); Pow2 != Bits) {
1779 if (Known.Zero.ashr(ShiftAmt: Pow2 - 1).isAllOnes())
1780 Sign = Positive;
1781 }
1782 return {Bits, Sign};
1783}
1784
1785auto HvxIdioms::canonSgn(SValue X, SValue Y) const
1786 -> std::pair<SValue, SValue> {
1787 // Canonicalize the signedness of X and Y, so that the result is one of:
1788 // S, S
1789 // U/P, S
1790 // U/P, U/P
1791 if (X.Sgn == Signed && Y.Sgn != Signed)
1792 std::swap(a&: X, b&: Y);
1793 return {X, Y};
1794}
1795
1796// Match
1797// (X * Y) [>> N], or
1798// ((X * Y) + (1 << M)) >> N
1799auto HvxIdioms::matchFxpMul(Instruction &In) const -> std::optional<FxpOp> {
1800 using namespace PatternMatch;
1801 auto *Ty = In.getType();
1802
1803 if (!Ty->isVectorTy() || !Ty->getScalarType()->isIntegerTy())
1804 return std::nullopt;
1805
1806 unsigned Width = cast<IntegerType>(Val: Ty->getScalarType())->getBitWidth();
1807
1808 FxpOp Op;
1809 Value *Exp = &In;
1810
1811 // Fixed-point multiplication is always shifted right (except when the
1812 // fraction is 0 bits).
1813 auto m_Shr = [](auto &&V, auto &&S) {
1814 return m_CombineOr(m_LShr(V, S), m_AShr(V, S));
1815 };
1816
1817 uint64_t Qn = 0;
1818 if (Value *T; match(V: Exp, P: m_Shr(m_Value(V&: T), m_ConstantInt(V&: Qn)))) {
1819 Op.Frac = Qn;
1820 Exp = T;
1821 } else {
1822 Op.Frac = 0;
1823 }
1824
1825 if (Op.Frac > Width)
1826 return std::nullopt;
1827
1828 // Check if there is rounding added.
1829 uint64_t CV;
1830 if (Value *T;
1831 Op.Frac > 0 && match(V: Exp, P: m_Add(L: m_Value(V&: T), R: m_ConstantInt(V&: CV)))) {
1832 if (CV != 0 && !isPowerOf2_64(Value: CV))
1833 return std::nullopt;
1834 if (CV != 0)
1835 Op.RoundAt = Log2_64(Value: CV);
1836 Exp = T;
1837 }
1838
1839 // Check if the rest is a multiplication.
1840 if (match(V: Exp, P: m_Mul(L: m_Value(V&: Op.X.Val), R: m_Value(V&: Op.Y.Val)))) {
1841 Op.Opcode = Instruction::Mul;
1842 // FIXME: The information below is recomputed.
1843 Op.X.Sgn = getNumSignificantBits(V: Op.X.Val, In: &In).second;
1844 Op.Y.Sgn = getNumSignificantBits(V: Op.Y.Val, In: &In).second;
1845 Op.ResTy = cast<VectorType>(Val: Ty);
1846 return Op;
1847 }
1848
1849 return std::nullopt;
1850}
1851
1852auto HvxIdioms::processFxpMul(Instruction &In, const FxpOp &Op) const
1853 -> Value * {
1854 assert(Op.X.Val->getType() == Op.Y.Val->getType());
1855
1856 auto *VecTy = dyn_cast<VectorType>(Val: Op.X.Val->getType());
1857 if (VecTy == nullptr)
1858 return nullptr;
1859 auto *ElemTy = cast<IntegerType>(Val: VecTy->getElementType());
1860 unsigned ElemWidth = ElemTy->getBitWidth();
1861
1862 // TODO: This can be relaxed after legalization is done pre-isel.
1863 if ((HVC.length(Ty: VecTy) * ElemWidth) % (8 * HVC.HST.getVectorLength()) != 0)
1864 return nullptr;
1865
1866 // There are no special intrinsics that should be used for multiplying
1867 // signed 8-bit values, so just skip them. Normal codegen should handle
1868 // this just fine.
1869 if (ElemWidth <= 8)
1870 return nullptr;
1871 // Similarly, if this is just a multiplication that can be handled without
1872 // intervention, then leave it alone.
1873 if (ElemWidth <= 32 && Op.Frac == 0)
1874 return nullptr;
1875
1876 auto [BitsX, SignX] = getNumSignificantBits(V: Op.X.Val, In: &In);
1877 auto [BitsY, SignY] = getNumSignificantBits(V: Op.Y.Val, In: &In);
1878
1879 // TODO: Add multiplication of vectors by scalar registers (up to 4 bytes).
1880
1881 Value *X = Op.X.Val, *Y = Op.Y.Val;
1882 IRBuilder Builder(In.getParent(), In.getIterator(),
1883 InstSimplifyFolder(HVC.DL));
1884
1885 auto roundUpWidth = [](unsigned Width) -> unsigned {
1886 if (Width <= 32 && !isPowerOf2_32(Value: Width)) {
1887 // If the element width is not a power of 2, round it up
1888 // to the next one. Do this for widths not exceeding 32.
1889 return PowerOf2Ceil(A: Width);
1890 }
1891 if (Width > 32 && Width % 32 != 0) {
1892 // For wider elements, round it up to the multiple of 32.
1893 return alignTo(Value: Width, Align: 32u);
1894 }
1895 return Width;
1896 };
1897
1898 BitsX = roundUpWidth(BitsX);
1899 BitsY = roundUpWidth(BitsY);
1900
1901 // For elementwise multiplication vectors must have the same lengths, so
1902 // resize the elements of both inputs to the same width, the max of the
1903 // calculated significant bits.
1904 unsigned Width = std::max(a: BitsX, b: BitsY);
1905
1906 auto *ResizeTy = VectorType::get(ElementType: HVC.getIntTy(Width), Other: VecTy);
1907 if (Width < ElemWidth) {
1908 X = Builder.CreateTrunc(V: X, DestTy: ResizeTy, Name: "trn");
1909 Y = Builder.CreateTrunc(V: Y, DestTy: ResizeTy, Name: "trn");
1910 } else if (Width > ElemWidth) {
1911 X = SignX == Signed ? Builder.CreateSExt(V: X, DestTy: ResizeTy, Name: "sxt")
1912 : Builder.CreateZExt(V: X, DestTy: ResizeTy, Name: "zxt");
1913 Y = SignY == Signed ? Builder.CreateSExt(V: Y, DestTy: ResizeTy, Name: "sxt")
1914 : Builder.CreateZExt(V: Y, DestTy: ResizeTy, Name: "zxt");
1915 };
1916
1917 assert(X->getType() == Y->getType() && X->getType() == ResizeTy);
1918
1919 unsigned VecLen = HVC.length(Ty: ResizeTy);
1920 unsigned ChopLen = (8 * HVC.HST.getVectorLength()) / std::min(a: Width, b: 32u);
1921
1922 SmallVector<Value *> Results;
1923 FxpOp ChopOp = Op;
1924 ChopOp.ResTy = VectorType::get(ElementType: Op.ResTy->getElementType(), NumElements: ChopLen, Scalable: false);
1925
1926 for (unsigned V = 0; V != VecLen / ChopLen; ++V) {
1927 ChopOp.X.Val = HVC.subvector(Builder, Val: X, Start: V * ChopLen, Length: ChopLen);
1928 ChopOp.Y.Val = HVC.subvector(Builder, Val: Y, Start: V * ChopLen, Length: ChopLen);
1929 Results.push_back(Elt: processFxpMulChopped(Builder, In, Op: ChopOp));
1930 if (Results.back() == nullptr)
1931 break;
1932 }
1933
1934 if (Results.empty() || Results.back() == nullptr)
1935 return nullptr;
1936
1937 Value *Cat = HVC.concat(Builder, Vecs: Results);
1938 Value *Ext = SignX == Signed || SignY == Signed
1939 ? Builder.CreateSExt(V: Cat, DestTy: VecTy, Name: "sxt")
1940 : Builder.CreateZExt(V: Cat, DestTy: VecTy, Name: "zxt");
1941 return Ext;
1942}
1943
1944inline bool HvxIdioms::matchScatter(Instruction &In) const {
1945 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1946 if (!II)
1947 return false;
1948 return (II->getIntrinsicID() == Intrinsic::masked_scatter);
1949}
1950
1951inline bool HvxIdioms::matchGather(Instruction &In) const {
1952 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1953 if (!II)
1954 return false;
1955 return (II->getIntrinsicID() == Intrinsic::masked_gather);
1956}
1957
1958inline bool HvxIdioms::matchMLoad(Instruction &In) const {
1959 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1960 if (!II)
1961 return false;
1962 return (II->getIntrinsicID() == Intrinsic::masked_load);
1963}
1964
1965inline bool HvxIdioms::matchMStore(Instruction &In) const {
1966 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &In);
1967 if (!II)
1968 return false;
1969 return (II->getIntrinsicID() == Intrinsic::masked_store);
1970}
1971
1972Instruction *locateDestination(Instruction *In, HvxIdioms::DstQualifier &Qual);
1973
1974// Binary instructions we want to handle as users of gather/scatter.
1975inline bool isArithmetic(unsigned Opc) {
1976 switch (Opc) {
1977 case Instruction::Add:
1978 case Instruction::Sub:
1979 case Instruction::Mul:
1980 case Instruction::And:
1981 case Instruction::Or:
1982 case Instruction::Xor:
1983 case Instruction::AShr:
1984 case Instruction::LShr:
1985 case Instruction::Shl:
1986 case Instruction::UDiv:
1987 return true;
1988 }
1989 return false;
1990}
1991
1992// TODO: Maybe use MemoryLocation for this. See getLocOrNone above.
1993inline Value *getPointer(Value *Ptr) {
1994 assert(Ptr && "Unable to extract pointer");
1995 if (isa<AllocaInst>(Val: Ptr) || isa<Argument>(Val: Ptr) || isa<GlobalValue>(Val: Ptr))
1996 return Ptr;
1997 if (isa<LoadInst>(Val: Ptr) || isa<StoreInst>(Val: Ptr))
1998 return getLoadStorePointerOperand(V: Ptr);
1999 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
2000 if (II->getIntrinsicID() == Intrinsic::masked_store)
2001 return II->getOperand(i_nocapture: 1);
2002 }
2003 return nullptr;
2004}
2005
2006static Instruction *selectDestination(Instruction *In,
2007 HvxIdioms::DstQualifier &Qual) {
2008 Instruction *Destination = nullptr;
2009 if (!In)
2010 return Destination;
2011 if (isa<StoreInst>(Val: In)) {
2012 Destination = In;
2013 Qual = HvxIdioms::LdSt;
2014 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: In)) {
2015 if (II->getIntrinsicID() == Intrinsic::masked_gather) {
2016 Destination = In;
2017 Qual = HvxIdioms::LLVM_Gather;
2018 } else if (II->getIntrinsicID() == Intrinsic::masked_scatter) {
2019 Destination = In;
2020 Qual = HvxIdioms::LLVM_Scatter;
2021 } else if (II->getIntrinsicID() == Intrinsic::masked_store) {
2022 Destination = In;
2023 Qual = HvxIdioms::LdSt;
2024 } else if (II->getIntrinsicID() ==
2025 Intrinsic::hexagon_V6_vgather_vscattermh) {
2026 Destination = In;
2027 Qual = HvxIdioms::HEX_Gather_Scatter;
2028 } else if (II->getIntrinsicID() == Intrinsic::hexagon_V6_vscattermh_128B) {
2029 Destination = In;
2030 Qual = HvxIdioms::HEX_Scatter;
2031 } else if (II->getIntrinsicID() == Intrinsic::hexagon_V6_vgathermh_128B) {
2032 Destination = In;
2033 Qual = HvxIdioms::HEX_Gather;
2034 }
2035 } else if (isa<ZExtInst>(Val: In)) {
2036 return locateDestination(In, Qual);
2037 } else if (isa<CastInst>(Val: In)) {
2038 return locateDestination(In, Qual);
2039 } else if (isa<CallInst>(Val: In)) {
2040 Destination = In;
2041 Qual = HvxIdioms::Call;
2042 } else if (isa<GetElementPtrInst>(Val: In)) {
2043 return locateDestination(In, Qual);
2044 } else if (isArithmetic(Opc: In->getOpcode())) {
2045 Destination = In;
2046 Qual = HvxIdioms::Arithmetic;
2047 } else {
2048 LLVM_DEBUG(dbgs() << "Unhandled destination : " << *In << "\n");
2049 }
2050 return Destination;
2051}
2052
2053// This method attempts to find destination (user) for a given intrinsic.
2054// Given that these are produced only by Ripple, the number of options is
2055// limited. Simplest case is explicit store which in fact is redundant (since
2056// HVX gater creates its own store during packetization). Nevertheless we need
2057// to figure address where we storing. Other cases are more complicated, but
2058// still few.
2059Instruction *locateDestination(Instruction *In, HvxIdioms::DstQualifier &Qual) {
2060 Instruction *Destination = nullptr;
2061 if (!In)
2062 return Destination;
2063 // Get all possible destinations
2064 SmallVector<Instruction *> Users;
2065 // Iterate over the uses of the instruction
2066 for (auto &U : In->uses()) {
2067 if (auto *UI = dyn_cast<Instruction>(Val: U.getUser())) {
2068 Destination = selectDestination(In: UI, Qual);
2069 if (Destination)
2070 Users.push_back(Elt: Destination);
2071 }
2072 }
2073 // Now see which of the users (if any) is a memory destination.
2074 for (auto *I : Users)
2075 if (getPointer(Ptr: I))
2076 return I;
2077 return Destination;
2078}
2079
2080// The two intrinsics we handle here have GEP in a different position.
2081inline GetElementPtrInst *locateGepFromIntrinsic(Instruction *In) {
2082 assert(In && "Bad instruction");
2083 IntrinsicInst *IIn = dyn_cast<IntrinsicInst>(Val: In);
2084 assert((IIn && (IIn->getIntrinsicID() == Intrinsic::masked_gather ||
2085 IIn->getIntrinsicID() == Intrinsic::masked_scatter)) &&
2086 "Not a gather Intrinsic");
2087 GetElementPtrInst *GEPIndex = nullptr;
2088 if (IIn->getIntrinsicID() == Intrinsic::masked_gather)
2089 GEPIndex = dyn_cast<GetElementPtrInst>(Val: IIn->getOperand(i_nocapture: 0));
2090 else
2091 GEPIndex = dyn_cast<GetElementPtrInst>(Val: IIn->getOperand(i_nocapture: 1));
2092 return GEPIndex;
2093}
2094
2095// Given the intrinsic find its GEP argument and extract base address it uses.
2096// The method relies on the way how Ripple typically forms the GEP for
2097// scatter/gather.
2098static Value *locateAddressFromIntrinsic(Instruction *In) {
2099 GetElementPtrInst *GEPIndex = locateGepFromIntrinsic(In);
2100 if (!GEPIndex) {
2101 LLVM_DEBUG(dbgs() << " No GEP in intrinsic\n");
2102 return nullptr;
2103 }
2104 Value *BaseAddress = GEPIndex->getPointerOperand();
2105 auto *IndexLoad = dyn_cast<LoadInst>(Val: BaseAddress);
2106 if (IndexLoad)
2107 return IndexLoad;
2108
2109 auto *IndexZEx = dyn_cast<ZExtInst>(Val: BaseAddress);
2110 if (IndexZEx) {
2111 IndexLoad = dyn_cast<LoadInst>(Val: IndexZEx->getOperand(i_nocapture: 0));
2112 if (IndexLoad)
2113 return IndexLoad;
2114 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: IndexZEx->getOperand(i_nocapture: 0));
2115 if (II && II->getIntrinsicID() == Intrinsic::masked_gather)
2116 return locateAddressFromIntrinsic(In: II);
2117 }
2118 auto *BaseShuffle = dyn_cast<ShuffleVectorInst>(Val: BaseAddress);
2119 if (BaseShuffle) {
2120 IndexLoad = dyn_cast<LoadInst>(Val: BaseShuffle->getOperand(i_nocapture: 0));
2121 if (IndexLoad)
2122 return IndexLoad;
2123 auto *IE = dyn_cast<InsertElementInst>(Val: BaseShuffle->getOperand(i_nocapture: 0));
2124 if (IE) {
2125 auto *Src = IE->getOperand(i_nocapture: 1);
2126 IndexLoad = dyn_cast<LoadInst>(Val: Src);
2127 if (IndexLoad)
2128 return IndexLoad;
2129 auto *Alloca = dyn_cast<AllocaInst>(Val: Src);
2130 if (Alloca)
2131 return Alloca;
2132 if (isa<Argument>(Val: Src)) {
2133 return Src;
2134 }
2135 if (isa<GlobalValue>(Val: Src)) {
2136 return Src;
2137 }
2138 }
2139 }
2140 LLVM_DEBUG(dbgs() << " Unable to locate Address from intrinsic\n");
2141 return nullptr;
2142}
2143
2144static Type *getIndexType(Value *In) {
2145 if (!In)
2146 return nullptr;
2147
2148 if (isa<LoadInst>(Val: In) || isa<StoreInst>(Val: In))
2149 return getLoadStoreType(I: In);
2150
2151 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: In)) {
2152 if (II->getIntrinsicID() == Intrinsic::masked_load)
2153 return II->getType();
2154 if (II->getIntrinsicID() == Intrinsic::masked_store)
2155 return II->getOperand(i_nocapture: 0)->getType();
2156 }
2157 return In->getType();
2158}
2159
2160static Value *locateIndexesFromGEP(Value *In) {
2161 if (!In)
2162 return nullptr;
2163 if (isa<LoadInst>(Val: In))
2164 return In;
2165 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: In)) {
2166 if (II->getIntrinsicID() == Intrinsic::masked_load)
2167 return In;
2168 if (II->getIntrinsicID() == Intrinsic::masked_gather)
2169 return In;
2170 }
2171 if (auto *IndexZEx = dyn_cast<ZExtInst>(Val: In))
2172 return locateIndexesFromGEP(In: IndexZEx->getOperand(i_nocapture: 0));
2173 if (auto *IndexSEx = dyn_cast<SExtInst>(Val: In))
2174 return locateIndexesFromGEP(In: IndexSEx->getOperand(i_nocapture: 0));
2175 if (auto *BaseShuffle = dyn_cast<ShuffleVectorInst>(Val: In))
2176 return locateIndexesFromGEP(In: BaseShuffle->getOperand(i_nocapture: 0));
2177 if (auto *IE = dyn_cast<InsertElementInst>(Val: In))
2178 return locateIndexesFromGEP(In: IE->getOperand(i_nocapture: 1));
2179 if (auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: In))
2180 return cstDataVector;
2181 if (auto *GEPIndex = dyn_cast<GetElementPtrInst>(Val: In))
2182 return GEPIndex->getOperand(i_nocapture: 0);
2183 return nullptr;
2184}
2185
2186// Given the intrinsic find its GEP argument and extract offsetts from the base
2187// address it uses.
2188static Value *locateIndexesFromIntrinsic(Instruction *In) {
2189 GetElementPtrInst *GEPIndex = locateGepFromIntrinsic(In);
2190 if (!GEPIndex) {
2191 LLVM_DEBUG(dbgs() << " No GEP in intrinsic\n");
2192 return nullptr;
2193 }
2194 Value *Indexes = GEPIndex->getOperand(i_nocapture: 1);
2195 if (auto *IndexLoad = locateIndexesFromGEP(In: Indexes))
2196 return IndexLoad;
2197
2198 LLVM_DEBUG(dbgs() << " Unable to locate Index from intrinsic\n");
2199 return nullptr;
2200}
2201
2202// Because of aukward definition of many Hex intrinsics we often have to
2203// reinterprete HVX native <64 x i16> as <32 x i32> which in practice is a NOP
2204// for all use cases, so this only exist to make IR builder happy.
2205inline Value *getReinterpretiveCast_i16_to_i32(const HexagonVectorCombine &HVC,
2206 IRBuilderBase &Builder,
2207 LLVMContext &Ctx, Value *I) {
2208 assert(I && "Unable to reinterprete cast");
2209 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2210 std::vector<unsigned> shuffleMask;
2211 for (unsigned i = 0; i < 64; ++i)
2212 shuffleMask.push_back(x: i);
2213 Constant *Mask = llvm::ConstantDataVector::get(Context&: Ctx, Elts: shuffleMask);
2214 Value *CastShuffle =
2215 Builder.CreateShuffleVector(V1: I, V2: I, Mask, Name: "identity_shuffle");
2216 return Builder.CreateBitCast(V: CastShuffle, DestTy: NT, Name: "cst64_i16_to_32_i32");
2217}
2218
2219// Recast <128 x i8> as <32 x i32>
2220inline Value *getReinterpretiveCast_i8_to_i32(const HexagonVectorCombine &HVC,
2221 IRBuilderBase &Builder,
2222 LLVMContext &Ctx, Value *I) {
2223 assert(I && "Unable to reinterprete cast");
2224 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2225 std::vector<unsigned> shuffleMask;
2226 for (unsigned i = 0; i < 128; ++i)
2227 shuffleMask.push_back(x: i);
2228 Constant *Mask = llvm::ConstantDataVector::get(Context&: Ctx, Elts: shuffleMask);
2229 Value *CastShuffle =
2230 Builder.CreateShuffleVector(V1: I, V2: I, Mask, Name: "identity_shuffle");
2231 return Builder.CreateBitCast(V: CastShuffle, DestTy: NT, Name: "cst128_i8_to_32_i32");
2232}
2233
2234// Create <32 x i32> mask reinterpreted as <128 x i1> with a given pattern
2235inline Value *get_i32_Mask(const HexagonVectorCombine &HVC,
2236 IRBuilderBase &Builder, LLVMContext &Ctx,
2237 unsigned int pattern) {
2238 std::vector<unsigned int> byteMask;
2239 for (unsigned i = 0; i < 32; ++i)
2240 byteMask.push_back(x: pattern);
2241
2242 return Builder.CreateIntrinsic(
2243 RetTy: HVC.getBoolTy(ElemCount: 128), ID: HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vandvrt),
2244 Args: {llvm::ConstantDataVector::get(Context&: Ctx, Elts: byteMask), HVC.getConstInt(Val: ~0)},
2245 FMFSource: nullptr);
2246}
2247
2248Value *HvxIdioms::processVScatter(Instruction &In) const {
2249 auto *InpTy = dyn_cast<VectorType>(Val: In.getOperand(i: 0)->getType());
2250 assert(InpTy && "Cannot handle no vector type for llvm.scatter/gather");
2251 unsigned InpSize = HVC.getSizeOf(Ty: InpTy);
2252 auto *F = In.getFunction();
2253 LLVMContext &Ctx = F->getContext();
2254 auto *ElemTy = dyn_cast<IntegerType>(Val: InpTy->getElementType());
2255 assert(ElemTy && "llvm.scatter needs integer type argument");
2256 unsigned ElemWidth = HVC.DL.getTypeAllocSize(Ty: ElemTy);
2257 LLVM_DEBUG({
2258 unsigned Elements = HVC.length(InpTy);
2259 dbgs() << "\n[Process scatter](" << In << ")\n" << *In.getParent() << "\n";
2260 dbgs() << " Input type(" << *InpTy << ") elements(" << Elements
2261 << ") VecLen(" << InpSize << ") type(" << *ElemTy << ") ElemWidth("
2262 << ElemWidth << ")\n";
2263 });
2264
2265 IRBuilder Builder(In.getParent(), In.getIterator(),
2266 InstSimplifyFolder(HVC.DL));
2267
2268 auto *ValueToScatter = In.getOperand(i: 0);
2269 LLVM_DEBUG(dbgs() << " ValueToScatter : " << *ValueToScatter << "\n");
2270
2271 if (HVC.HST.getVectorLength() != InpSize) {
2272 LLVM_DEBUG(dbgs() << "Unhandled vector size(" << InpSize
2273 << ") for vscatter\n");
2274 return nullptr;
2275 }
2276
2277 // Base address of indexes.
2278 auto *IndexLoad = locateAddressFromIntrinsic(In: &In);
2279 if (!IndexLoad)
2280 return nullptr;
2281 LLVM_DEBUG(dbgs() << " IndexLoad : " << *IndexLoad << "\n");
2282
2283 // Address of destination. Must be in VTCM.
2284 auto *Ptr = getPointer(Ptr: IndexLoad);
2285 if (!Ptr)
2286 return nullptr;
2287 LLVM_DEBUG(dbgs() << " Ptr : " << *Ptr << "\n");
2288 // Indexes/offsets
2289 auto *Indexes = locateIndexesFromIntrinsic(In: &In);
2290 if (!Indexes)
2291 return nullptr;
2292 LLVM_DEBUG(dbgs() << " Indexes : " << *Indexes << "\n");
2293 Value *CastedDst = Builder.CreateBitOrPointerCast(V: Ptr, DestTy: Type::getInt32Ty(C&: Ctx),
2294 Name: "cst_ptr_to_i32");
2295 LLVM_DEBUG(dbgs() << " CastedDst : " << *CastedDst << "\n");
2296 // Adjust Indexes
2297 auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: Indexes);
2298 Value *CastIndex = nullptr;
2299 if (cstDataVector) {
2300 // Our indexes are represented as a constant. We need it in a reg.
2301 Type *IndexVectorType = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2302 AllocaInst *IndexesAlloca = Builder.CreateAlloca(Ty: IndexVectorType);
2303 [[maybe_unused]] auto *StoreIndexes =
2304 Builder.CreateStore(Val: cstDataVector, Ptr: IndexesAlloca);
2305 LLVM_DEBUG(dbgs() << " StoreIndexes : " << *StoreIndexes << "\n");
2306 CastIndex =
2307 Builder.CreateLoad(Ty: IndexVectorType, Ptr: IndexesAlloca, Name: "reload_index");
2308 } else {
2309 if (ElemWidth == 2)
2310 CastIndex = getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: Indexes);
2311 else
2312 CastIndex = Indexes;
2313 }
2314 LLVM_DEBUG(dbgs() << " Cast index : " << *CastIndex << ")\n");
2315
2316 if (ElemWidth == 1) {
2317 // v128i8 There is no native instruction for this.
2318 // Do this as two Hi/Lo gathers with masking.
2319 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2320 // Extend indexes. We assume that indexes are in 128i8 format - need to
2321 // expand them to Hi/Lo 64i16
2322 Value *CastIndexes = Builder.CreateBitCast(V: CastIndex, DestTy: NT, Name: "cast_to_32i32");
2323 auto V6_vunpack = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vunpackub);
2324 auto *UnpackedIndexes = Builder.CreateIntrinsic(
2325 RetTy: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: true), ID: V6_vunpack, Args: CastIndexes, FMFSource: nullptr);
2326 LLVM_DEBUG(dbgs() << " UnpackedIndexes : " << *UnpackedIndexes << ")\n");
2327
2328 auto V6_hi = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_hi);
2329 auto V6_lo = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_lo);
2330 [[maybe_unused]] Value *IndexHi =
2331 HVC.createHvxIntrinsic(Builder, IntID: V6_hi, RetTy: NT, Args: UnpackedIndexes);
2332 [[maybe_unused]] Value *IndexLo =
2333 HVC.createHvxIntrinsic(Builder, IntID: V6_lo, RetTy: NT, Args: UnpackedIndexes);
2334 LLVM_DEBUG(dbgs() << " UnpackedIndHi : " << *IndexHi << ")\n");
2335 LLVM_DEBUG(dbgs() << " UnpackedIndLo : " << *IndexLo << ")\n");
2336 // Now unpack values to scatter
2337 Value *CastSrc =
2338 getReinterpretiveCast_i8_to_i32(HVC, Builder, Ctx, I: ValueToScatter);
2339 LLVM_DEBUG(dbgs() << " CastSrc : " << *CastSrc << ")\n");
2340 auto *UnpackedValueToScatter = Builder.CreateIntrinsic(
2341 RetTy: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: true), ID: V6_vunpack, Args: CastSrc, FMFSource: nullptr);
2342 LLVM_DEBUG(dbgs() << " UnpackedValToScat: " << *UnpackedValueToScatter
2343 << ")\n");
2344
2345 [[maybe_unused]] Value *UVSHi =
2346 HVC.createHvxIntrinsic(Builder, IntID: V6_hi, RetTy: NT, Args: UnpackedValueToScatter);
2347 [[maybe_unused]] Value *UVSLo =
2348 HVC.createHvxIntrinsic(Builder, IntID: V6_lo, RetTy: NT, Args: UnpackedValueToScatter);
2349 LLVM_DEBUG(dbgs() << " UVSHi : " << *UVSHi << ")\n");
2350 LLVM_DEBUG(dbgs() << " UVSLo : " << *UVSLo << ")\n");
2351
2352 // Create the mask for individual bytes
2353 auto *QByteMask = get_i32_Mask(HVC, Builder, Ctx, pattern: 0x00ff00ff);
2354 LLVM_DEBUG(dbgs() << " QByteMask : " << *QByteMask << "\n");
2355 [[maybe_unused]] auto *ResHi = Builder.CreateIntrinsic(
2356 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermhq_128B,
2357 Args: {QByteMask, CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2358 IndexHi, UVSHi},
2359 FMFSource: nullptr);
2360 LLVM_DEBUG(dbgs() << " ResHi : " << *ResHi << ")\n");
2361 return Builder.CreateIntrinsic(
2362 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermhq_128B,
2363 Args: {QByteMask, CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2364 IndexLo, UVSLo},
2365 FMFSource: nullptr);
2366 } else if (ElemWidth == 2) {
2367 Value *CastSrc =
2368 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: ValueToScatter);
2369 LLVM_DEBUG(dbgs() << " CastSrc : " << *CastSrc << ")\n");
2370 return Builder.CreateIntrinsic(
2371 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermh_128B,
2372 Args: {CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), CastIndex,
2373 CastSrc},
2374 FMFSource: nullptr);
2375 } else if (ElemWidth == 4) {
2376 return Builder.CreateIntrinsic(
2377 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vscattermw_128B,
2378 Args: {CastedDst, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), CastIndex,
2379 ValueToScatter},
2380 FMFSource: nullptr);
2381 } else {
2382 LLVM_DEBUG(dbgs() << "Unhandled element type for vscatter\n");
2383 return nullptr;
2384 }
2385}
2386
2387Value *HvxIdioms::processVGather(Instruction &In) const {
2388 [[maybe_unused]] auto *InpTy =
2389 dyn_cast<VectorType>(Val: In.getOperand(i: 0)->getType());
2390 assert(InpTy && "Cannot handle no vector type for llvm.gather");
2391 [[maybe_unused]] auto *ElemTy =
2392 dyn_cast<PointerType>(Val: InpTy->getElementType());
2393 assert(ElemTy && "llvm.gather needs vector of ptr argument");
2394 auto *F = In.getFunction();
2395 LLVMContext &Ctx = F->getContext();
2396 LLVM_DEBUG(dbgs() << "\n[Process gather](" << In << ")\n"
2397 << *In.getParent() << "\n");
2398 LLVM_DEBUG(dbgs() << " Input type(" << *InpTy << ") elements("
2399 << HVC.length(InpTy) << ") VecLen(" << HVC.getSizeOf(InpTy)
2400 << ") type(" << *ElemTy << ") Access alignment("
2401 << *In.getOperand(1) << ") AddressSpace("
2402 << ElemTy->getAddressSpace() << ")\n");
2403
2404 // TODO: Handle masking of elements.
2405 assert(dyn_cast<VectorType>(In.getOperand(2)->getType()) &&
2406 "llvm.gather needs vector for mask");
2407 IRBuilder Builder(In.getParent(), In.getIterator(),
2408 InstSimplifyFolder(HVC.DL));
2409
2410 // See who is using the result. The difference between LLVM and HVX vgather
2411 // Intrinsic makes it impossible to handle all cases with temp storage. Alloca
2412 // in VTCM is not yet supported, so for now we just bail out for those cases.
2413 HvxIdioms::DstQualifier Qual = HvxIdioms::Undefined;
2414 Instruction *Dst = locateDestination(In: &In, Qual);
2415 if (!Dst) {
2416 LLVM_DEBUG(dbgs() << " Unable to locate vgather destination\n");
2417 return nullptr;
2418 }
2419 LLVM_DEBUG(dbgs() << " Destination : " << *Dst << " Qual(" << Qual
2420 << ")\n");
2421
2422 // Address of destination. Must be in VTCM.
2423 auto *Ptr = getPointer(Ptr: Dst);
2424 if (!Ptr) {
2425 LLVM_DEBUG(dbgs() << "Could not locate vgather destination ptr\n");
2426 return nullptr;
2427 }
2428
2429 // Result type. Assume it is a vector type.
2430 auto *DstType = cast<VectorType>(Val: getIndexType(In: Dst));
2431 assert(DstType && "Cannot handle non vector dst type for llvm.gather");
2432
2433 // Base address for sources to be loaded
2434 auto *IndexLoad = locateAddressFromIntrinsic(In: &In);
2435 if (!IndexLoad)
2436 return nullptr;
2437 LLVM_DEBUG(dbgs() << " IndexLoad : " << *IndexLoad << "\n");
2438
2439 // Gather indexes/offsets
2440 auto *Indexes = locateIndexesFromIntrinsic(In: &In);
2441 if (!Indexes)
2442 return nullptr;
2443 LLVM_DEBUG(dbgs() << " Indexes : " << *Indexes << "\n");
2444
2445 Value *Gather = nullptr;
2446 Type *NT = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false);
2447 if (Qual == HvxIdioms::LdSt || Qual == HvxIdioms::Arithmetic) {
2448 // We fully assume the address space is in VTCM. We also assume that all
2449 // pointers in Operand(0) have the same base(!).
2450 // This is the most basic case of all the above.
2451 unsigned OutputSize = HVC.getSizeOf(Ty: DstType);
2452 auto *DstElemTy = cast<IntegerType>(Val: DstType->getElementType());
2453 unsigned ElemWidth = HVC.DL.getTypeAllocSize(Ty: DstElemTy);
2454 LLVM_DEBUG(dbgs() << " Buffer type : " << *Ptr->getType()
2455 << " Address space ("
2456 << Ptr->getType()->getPointerAddressSpace() << ")\n"
2457 << " Result type : " << *DstType
2458 << "\n Size in bytes : " << OutputSize
2459 << " element type(" << *DstElemTy
2460 << ")\n ElemWidth : " << ElemWidth << " bytes\n");
2461
2462 auto *IndexType = cast<VectorType>(Val: getIndexType(In: Indexes));
2463 assert(IndexType && "Cannot handle non vector index type for llvm.gather");
2464 unsigned IndexWidth = HVC.DL.getTypeAllocSize(Ty: IndexType->getElementType());
2465 LLVM_DEBUG(dbgs() << " IndexWidth(" << IndexWidth << ")\n");
2466
2467 // Intrinsic takes i32 instead of pointer so cast.
2468 Value *CastedPtr = Builder.CreateBitOrPointerCast(
2469 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2470 // [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, ...]
2471 // int_hexagon_V6_vgathermh [... , llvm_v16i32_ty]
2472 // int_hexagon_V6_vgathermh_128B [... , llvm_v32i32_ty]
2473 // int_hexagon_V6_vgathermhw [... , llvm_v32i32_ty]
2474 // int_hexagon_V6_vgathermhw_128B [... , llvm_v64i32_ty]
2475 // int_hexagon_V6_vgathermw [... , llvm_v16i32_ty]
2476 // int_hexagon_V6_vgathermw_128B [... , llvm_v32i32_ty]
2477 if (HVC.HST.getVectorLength() == OutputSize) {
2478 if (ElemWidth == 1) {
2479 // v128i8 There is no native instruction for this.
2480 // Do this as two Hi/Lo gathers with masking.
2481 // Unpack indexes. We assume that indexes are in 128i8 format - need to
2482 // expand them to Hi/Lo 64i16
2483 Value *CastIndexes =
2484 Builder.CreateBitCast(V: Indexes, DestTy: NT, Name: "cast_to_32i32");
2485 auto V6_vunpack = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vunpackub);
2486 auto *UnpackedIndexes =
2487 Builder.CreateIntrinsic(RetTy: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: true),
2488 ID: V6_vunpack, Args: CastIndexes, FMFSource: nullptr);
2489 LLVM_DEBUG(dbgs() << " UnpackedIndexes : " << *UnpackedIndexes
2490 << ")\n");
2491
2492 auto V6_hi = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_hi);
2493 auto V6_lo = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_lo);
2494 [[maybe_unused]] Value *IndexHi =
2495 HVC.createHvxIntrinsic(Builder, IntID: V6_hi, RetTy: NT, Args: UnpackedIndexes);
2496 [[maybe_unused]] Value *IndexLo =
2497 HVC.createHvxIntrinsic(Builder, IntID: V6_lo, RetTy: NT, Args: UnpackedIndexes);
2498 LLVM_DEBUG(dbgs() << " UnpackedIndHi : " << *IndexHi << ")\n");
2499 LLVM_DEBUG(dbgs() << " UnpackedIndLo : " << *IndexLo << ")\n");
2500 // Create the mask for individual bytes
2501 auto *QByteMask = get_i32_Mask(HVC, Builder, Ctx, pattern: 0x00ff00ff);
2502 LLVM_DEBUG(dbgs() << " QByteMask : " << *QByteMask << "\n");
2503 // We use our destination allocation as a temp storage
2504 // This is unlikely to work properly for masked gather.
2505 auto V6_vgather = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vgathermhq);
2506 [[maybe_unused]] auto GatherHi = Builder.CreateIntrinsic(
2507 RetTy: Type::getVoidTy(C&: Ctx), ID: V6_vgather,
2508 Args: {Ptr, QByteMask, CastedPtr,
2509 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), IndexHi},
2510 FMFSource: nullptr);
2511 LLVM_DEBUG(dbgs() << " GatherHi : " << *GatherHi << ")\n");
2512 // Rematerialize the result
2513 [[maybe_unused]] Value *LoadedResultHi = Builder.CreateLoad(
2514 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false), Ptr, Name: "temp_result_hi");
2515 LLVM_DEBUG(dbgs() << " LoadedResultHi : " << *LoadedResultHi << "\n");
2516 // Same for the low part. Here we use Gather to return non-NULL result
2517 // from this function and continue to iterate. We also are deleting Dst
2518 // store below.
2519 Gather = Builder.CreateIntrinsic(
2520 RetTy: Type::getVoidTy(C&: Ctx), ID: V6_vgather,
2521 Args: {Ptr, QByteMask, CastedPtr,
2522 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), IndexLo},
2523 FMFSource: nullptr);
2524 LLVM_DEBUG(dbgs() << " GatherLo : " << *Gather << ")\n");
2525 Value *LoadedResultLo = Builder.CreateLoad(
2526 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 32), Pair: false), Ptr, Name: "temp_result_lo");
2527 LLVM_DEBUG(dbgs() << " LoadedResultLo : " << *LoadedResultLo << "\n");
2528 // Now we have properly sized bytes in every other position
2529 // B b A a c a A b B c f F g G h H is presented as
2530 // B . b . A . a . c . a . A . b . B . c . f . F . g . G . h . H
2531 // Use vpack to gather them
2532 auto V6_vpackeb = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vpackeb);
2533 [[maybe_unused]] auto Res = Builder.CreateIntrinsic(
2534 RetTy: NT, ID: V6_vpackeb, Args: {LoadedResultHi, LoadedResultLo}, FMFSource: nullptr);
2535 LLVM_DEBUG(dbgs() << " ScaledRes : " << *Res << "\n");
2536 [[maybe_unused]] auto *StoreRes = Builder.CreateStore(Val: Res, Ptr);
2537 LLVM_DEBUG(dbgs() << " StoreRes : " << *StoreRes << "\n");
2538 } else if (ElemWidth == 2) {
2539 // v32i16
2540 if (IndexWidth == 2) {
2541 // Reinterprete 64i16 as 32i32. Only needed for syntactic IR match.
2542 Value *CastIndex =
2543 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: Indexes);
2544 LLVM_DEBUG(dbgs() << " Cast index: " << *CastIndex << ")\n");
2545 // shift all i16 left by 1 to match short addressing mode instead of
2546 // byte.
2547 auto V6_vaslh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaslh);
2548 Value *AdjustedIndex = HVC.createHvxIntrinsic(
2549 Builder, IntID: V6_vaslh, RetTy: NT, Args: {CastIndex, HVC.getConstInt(Val: 1)});
2550 LLVM_DEBUG(dbgs()
2551 << " Shifted half index: " << *AdjustedIndex << ")\n");
2552
2553 auto V6_vgather = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vgathermh);
2554 // The 3rd argument is the size of the region to gather from. Probably
2555 // want to set it to max VTCM size.
2556 Gather = Builder.CreateIntrinsic(
2557 RetTy: Type::getVoidTy(C&: Ctx), ID: V6_vgather,
2558 Args: {Ptr, CastedPtr, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2559 AdjustedIndex},
2560 FMFSource: nullptr);
2561 for (auto &U : Dst->uses()) {
2562 if (auto *UI = dyn_cast<Instruction>(Val: U.getUser()))
2563 dbgs() << " dst used by: " << *UI << "\n";
2564 }
2565 for (auto &U : In.uses()) {
2566 if (auto *UI = dyn_cast<Instruction>(Val: U.getUser()))
2567 dbgs() << " In used by : " << *UI << "\n";
2568 }
2569 // Create temp load from result in case the result is used by any
2570 // other instruction.
2571 Value *LoadedResult = Builder.CreateLoad(
2572 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr, Name: "temp_result");
2573 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2574 In.replaceAllUsesWith(V: LoadedResult);
2575 } else {
2576 dbgs() << " Unhandled index type for vgather\n";
2577 return nullptr;
2578 }
2579 } else if (ElemWidth == 4) {
2580 if (IndexWidth == 4) {
2581 // v32i32
2582 auto V6_vaslh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaslh);
2583 Value *AdjustedIndex = HVC.createHvxIntrinsic(
2584 Builder, IntID: V6_vaslh, RetTy: NT, Args: {Indexes, HVC.getConstInt(Val: 2)});
2585 LLVM_DEBUG(dbgs()
2586 << " Shifted word index: " << *AdjustedIndex << ")\n");
2587 Gather = Builder.CreateIntrinsic(
2588 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermw_128B,
2589 Args: {Ptr, CastedPtr, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2590 AdjustedIndex},
2591 FMFSource: nullptr);
2592 } else {
2593 LLVM_DEBUG(dbgs() << " Unhandled index type for vgather\n");
2594 return nullptr;
2595 }
2596 } else {
2597 LLVM_DEBUG(dbgs() << " Unhandled element type for vgather\n");
2598 return nullptr;
2599 }
2600 } else if (HVC.HST.getVectorLength() == OutputSize * 2) {
2601 // This is half of the reg width, duplicate low in high
2602 LLVM_DEBUG(dbgs() << " Unhandled half of register size\n");
2603 return nullptr;
2604 } else if (HVC.HST.getVectorLength() * 2 == OutputSize) {
2605 LLVM_DEBUG(dbgs() << " Unhandle twice the register size\n");
2606 return nullptr;
2607 }
2608 // Erase the original intrinsic and store that consumes it.
2609 // HVX will create a pseudo for gather that is expanded to gather + store
2610 // during packetization.
2611 Dst->eraseFromParent();
2612 } else if (Qual == HvxIdioms::LLVM_Scatter) {
2613 // Gather feeds directly into scatter.
2614 LLVM_DEBUG({
2615 auto *DstInpTy = cast<VectorType>(Dst->getOperand(1)->getType());
2616 assert(DstInpTy && "Cannot handle no vector type for llvm.scatter");
2617 unsigned DstInpSize = HVC.getSizeOf(DstInpTy);
2618 unsigned DstElements = HVC.length(DstInpTy);
2619 auto *DstElemTy = cast<PointerType>(DstInpTy->getElementType());
2620 assert(DstElemTy && "llvm.scatter needs vector of ptr argument");
2621 dbgs() << " Gather feeds into scatter\n Values to scatter : "
2622 << *Dst->getOperand(0) << "\n";
2623 dbgs() << " Dst type(" << *DstInpTy << ") elements(" << DstElements
2624 << ") VecLen(" << DstInpSize << ") type(" << *DstElemTy
2625 << ") Access alignment(" << *Dst->getOperand(2) << ")\n";
2626 });
2627 // Address of source
2628 auto *Src = getPointer(Ptr: IndexLoad);
2629 if (!Src)
2630 return nullptr;
2631 LLVM_DEBUG(dbgs() << " Src : " << *Src << "\n");
2632
2633 if (!isa<PointerType>(Val: Src->getType())) {
2634 LLVM_DEBUG(dbgs() << " Source is not a pointer type...\n");
2635 return nullptr;
2636 }
2637
2638 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2639 V: Src, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2640 LLVM_DEBUG(dbgs() << " CastedSrc: " << *CastedSrc << "\n");
2641
2642 auto *DstLoad = locateAddressFromIntrinsic(In: Dst);
2643 if (!DstLoad) {
2644 LLVM_DEBUG(dbgs() << " Unable to locate DstLoad\n");
2645 return nullptr;
2646 }
2647 LLVM_DEBUG(dbgs() << " DstLoad : " << *DstLoad << "\n");
2648
2649 Value *Ptr = getPointer(Ptr: DstLoad);
2650 if (!Ptr)
2651 return nullptr;
2652 LLVM_DEBUG(dbgs() << " Ptr : " << *Ptr << "\n");
2653 Value *CastIndex =
2654 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: IndexLoad);
2655 LLVM_DEBUG(dbgs() << " Cast index: " << *CastIndex << ")\n");
2656 // Shift all i16 left by 1 to match short addressing mode instead of
2657 // byte.
2658 auto V6_vaslh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaslh);
2659 Value *AdjustedIndex = HVC.createHvxIntrinsic(
2660 Builder, IntID: V6_vaslh, RetTy: NT, Args: {CastIndex, HVC.getConstInt(Val: 1)});
2661 LLVM_DEBUG(dbgs() << " Shifted half index: " << *AdjustedIndex << ")\n");
2662
2663 return Builder.CreateIntrinsic(
2664 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2665 Args: {Ptr, CastedSrc, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2666 AdjustedIndex},
2667 FMFSource: nullptr);
2668 } else if (Qual == HvxIdioms::HEX_Gather_Scatter) {
2669 // Gather feeds into previously inserted pseudo intrinsic.
2670 // These could not be in the same packet, so we need to generate another
2671 // pseudo that is expanded to .tmp + store V6_vgathermh_pseudo
2672 // V6_vgathermh_pseudo (ins IntRegs:$_dst_, s4_0Imm:$Ii, IntRegs:$Rt,
2673 // ModRegs:$Mu, HvxVR:$Vv)
2674 if (isa<AllocaInst>(Val: IndexLoad)) {
2675 auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: Indexes);
2676 if (cstDataVector) {
2677 // Our indexes are represented as a constant. We need THEM in a reg.
2678 // This most likely will not work properly since alloca gives us DDR
2679 // stack location. This will be fixed once we teach compiler about VTCM.
2680 AllocaInst *IndexesAlloca = Builder.CreateAlloca(Ty: NT);
2681 [[maybe_unused]] auto *StoreIndexes =
2682 Builder.CreateStore(Val: cstDataVector, Ptr: IndexesAlloca);
2683 LLVM_DEBUG(dbgs() << " StoreIndexes : " << *StoreIndexes << "\n");
2684 Value *LoadedIndex =
2685 Builder.CreateLoad(Ty: NT, Ptr: IndexesAlloca, Name: "reload_index");
2686 AllocaInst *ResultAlloca = Builder.CreateAlloca(Ty: NT);
2687 LLVM_DEBUG(dbgs() << " ResultAlloca : " << *ResultAlloca << "\n");
2688
2689 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2690 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2691 LLVM_DEBUG(dbgs() << " CastedSrc : " << *CastedSrc << "\n");
2692
2693 Gather = Builder.CreateIntrinsic(
2694 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2695 Args: {ResultAlloca, CastedSrc,
2696 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), LoadedIndex},
2697 FMFSource: nullptr);
2698 Value *LoadedResult = Builder.CreateLoad(
2699 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr: ResultAlloca, Name: "temp_result");
2700 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2701 LLVM_DEBUG(dbgs() << " Gather : " << *Gather << "\n");
2702 In.replaceAllUsesWith(V: LoadedResult);
2703 }
2704 } else {
2705 // Address of source
2706 auto *Src = getPointer(Ptr: IndexLoad);
2707 if (!Src)
2708 return nullptr;
2709 LLVM_DEBUG(dbgs() << " Src : " << *Src << "\n");
2710
2711 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2712 V: Src, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2713 LLVM_DEBUG(dbgs() << " CastedSrc: " << *CastedSrc << "\n");
2714
2715 auto *DstLoad = locateAddressFromIntrinsic(In: Dst);
2716 if (!DstLoad)
2717 return nullptr;
2718 LLVM_DEBUG(dbgs() << " DstLoad : " << *DstLoad << "\n");
2719 auto *Ptr = getPointer(Ptr: DstLoad);
2720 if (!Ptr)
2721 return nullptr;
2722 LLVM_DEBUG(dbgs() << " Ptr : " << *Ptr << "\n");
2723
2724 Gather = Builder.CreateIntrinsic(
2725 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgather_vscattermh,
2726 Args: {Ptr, CastedSrc, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2727 Indexes},
2728 FMFSource: nullptr);
2729 }
2730 return Gather;
2731 } else if (Qual == HvxIdioms::HEX_Scatter) {
2732 // This is the case when result of a gather is used as an argument to
2733 // Intrinsic::hexagon_V6_vscattermh_128B. Most likely we just inserted it
2734 // ourselves. We have to create alloca, store to it, and replace all uses
2735 // with that.
2736 AllocaInst *ResultAlloca = Builder.CreateAlloca(Ty: NT);
2737 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2738 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2739 LLVM_DEBUG(dbgs() << " CastedSrc : " << *CastedSrc << "\n");
2740 Value *CastIndex =
2741 getReinterpretiveCast_i16_to_i32(HVC, Builder, Ctx, I: Indexes);
2742 LLVM_DEBUG(dbgs() << " Cast index : " << *CastIndex << ")\n");
2743
2744 Gather = Builder.CreateIntrinsic(
2745 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2746 Args: {ResultAlloca, CastedSrc, HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE),
2747 CastIndex},
2748 FMFSource: nullptr);
2749 Value *LoadedResult = Builder.CreateLoad(
2750 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr: ResultAlloca, Name: "temp_result");
2751 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2752 In.replaceAllUsesWith(V: LoadedResult);
2753 } else if (Qual == HvxIdioms::HEX_Gather) {
2754 // Gather feeds to another gather but already replaced with
2755 // hexagon_V6_vgathermh_128B
2756 if (isa<AllocaInst>(Val: IndexLoad)) {
2757 auto *cstDataVector = dyn_cast<ConstantDataVector>(Val: Indexes);
2758 if (cstDataVector) {
2759 // Our indexes are represented as a constant. We need it in a reg.
2760 AllocaInst *IndexesAlloca = Builder.CreateAlloca(Ty: NT);
2761
2762 [[maybe_unused]] auto *StoreIndexes =
2763 Builder.CreateStore(Val: cstDataVector, Ptr: IndexesAlloca);
2764 LLVM_DEBUG(dbgs() << " StoreIndexes : " << *StoreIndexes << "\n");
2765 Value *LoadedIndex =
2766 Builder.CreateLoad(Ty: NT, Ptr: IndexesAlloca, Name: "reload_index");
2767 AllocaInst *ResultAlloca = Builder.CreateAlloca(Ty: NT);
2768 LLVM_DEBUG(dbgs() << " ResultAlloca : " << *ResultAlloca
2769 << "\n AddressSpace: "
2770 << ResultAlloca->getAddressSpace() << "\n";);
2771
2772 Value *CastedSrc = Builder.CreateBitOrPointerCast(
2773 V: IndexLoad, DestTy: Type::getInt32Ty(C&: Ctx), Name: "cst_ptr_to_i32");
2774 LLVM_DEBUG(dbgs() << " CastedSrc : " << *CastedSrc << "\n");
2775
2776 Gather = Builder.CreateIntrinsic(
2777 RetTy: Type::getVoidTy(C&: Ctx), ID: Intrinsic::hexagon_V6_vgathermh_128B,
2778 Args: {ResultAlloca, CastedSrc,
2779 HVC.getConstInt(DEFAULT_HVX_VTCM_PAGE_SIZE), LoadedIndex},
2780 FMFSource: nullptr);
2781 Value *LoadedResult = Builder.CreateLoad(
2782 Ty: HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), Pair: false), Ptr: ResultAlloca, Name: "temp_result");
2783 LLVM_DEBUG(dbgs() << " LoadedResult : " << *LoadedResult << "\n");
2784 LLVM_DEBUG(dbgs() << " Gather : " << *Gather << "\n");
2785 In.replaceAllUsesWith(V: LoadedResult);
2786 }
2787 }
2788 } else if (Qual == HvxIdioms::LLVM_Gather) {
2789 // Gather feeds into another gather
2790 errs() << " Underimplemented vgather to vgather sequence\n";
2791 return nullptr;
2792 } else
2793 llvm_unreachable("Unhandled Qual enum");
2794
2795 return Gather;
2796}
2797
2798// Go through all PHI incomming values and find minimal alignment for non GEP
2799// members.
2800std::optional<uint64_t> HvxIdioms::getPHIBaseMinAlignment(Instruction &In,
2801 PHINode *PN) const {
2802 if (!PN)
2803 return std::nullopt;
2804
2805 SmallVector<Value *, 16> Worklist;
2806 SmallPtrSet<Value *, 16> Visited;
2807 uint64_t minPHIAlignment = Value::MaximumAlignment;
2808 Worklist.push_back(Elt: PN);
2809
2810 while (!Worklist.empty()) {
2811 Value *V = Worklist.back();
2812 Worklist.pop_back();
2813 if (!Visited.insert(Ptr: V).second)
2814 continue;
2815
2816 if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
2817 for (unsigned i = 0; i < PN->getNumIncomingValues(); ++i) {
2818 Worklist.push_back(Elt: PN->getIncomingValue(i));
2819 }
2820 } else if (isa<GetElementPtrInst>(Val: V)) {
2821 // Ignore geps for now.
2822 continue;
2823 } else {
2824 Align KnownAlign = getKnownAlignment(V, DL: HVC.DL, CxtI: &In, AC: &HVC.AC, DT: &HVC.DT);
2825 if (KnownAlign.value() < minPHIAlignment)
2826 minPHIAlignment = KnownAlign.value();
2827 }
2828 }
2829 if (minPHIAlignment != Value::MaximumAlignment)
2830 return minPHIAlignment;
2831 return std::nullopt;
2832}
2833
2834// Helper function to discover alignment for a ptr.
2835std::optional<uint64_t> HvxIdioms::getAlignment(Instruction &In,
2836 Value *ptr) const {
2837 SmallPtrSet<Value *, 16> Visited;
2838 return getAlignmentImpl(In, ptr, Visited);
2839}
2840
2841std::optional<uint64_t>
2842HvxIdioms::getAlignmentImpl(Instruction &In, Value *ptr,
2843 SmallPtrSet<Value *, 16> &Visited) const {
2844 LLVM_DEBUG(dbgs() << "[getAlignment] for : " << *ptr << "\n");
2845 // Prevent infinite recursion
2846 if (!Visited.insert(Ptr: ptr).second)
2847 return std::nullopt;
2848 // Try AssumptionCache.
2849 Align KnownAlign = getKnownAlignment(V: ptr, DL: HVC.DL, CxtI: &In, AC: &HVC.AC, DT: &HVC.DT);
2850 // This is the most formal and reliable source of information.
2851 if (KnownAlign.value() > 1) {
2852 LLVM_DEBUG(dbgs() << " VC align(" << KnownAlign.value() << ")\n");
2853 return KnownAlign.value();
2854 }
2855
2856 // If it is a PHI try to iterate through inputs
2857 if (PHINode *PN = dyn_cast<PHINode>(Val: ptr)) {
2858 // See if we have a common base to which we know alignment.
2859 auto baseAlignmentOpt = getPHIBaseMinAlignment(In, PN);
2860 if (!baseAlignmentOpt)
2861 return std::nullopt;
2862
2863 uint64_t minBaseAlignment = *baseAlignmentOpt;
2864 // If it is 1, there is no point to keep on looking.
2865 if (minBaseAlignment == 1)
2866 return 1;
2867 // No see if all other incomming phi nodes are just loop carried constants.
2868 uint64_t minPHIAlignment = minBaseAlignment;
2869 LLVM_DEBUG(dbgs() << " It is a PHI with(" << PN->getNumIncomingValues()
2870 << ")nodes and min base aligned to (" << minBaseAlignment
2871 << ")\n");
2872 for (unsigned i = 0; i < PN->getNumIncomingValues(); ++i) {
2873 Value *IV = PN->getIncomingValue(i);
2874 // We have already looked at all other values.
2875 if (!isa<GetElementPtrInst>(Val: IV))
2876 continue;
2877 uint64_t MemberAlignment = Value::MaximumAlignment;
2878 if (auto res = getAlignment(In&: *PN, ptr: IV))
2879 MemberAlignment = *res;
2880 else
2881 return std::nullopt;
2882 // Adjust total PHI alignment.
2883 if (minPHIAlignment > MemberAlignment)
2884 minPHIAlignment = MemberAlignment;
2885 }
2886 LLVM_DEBUG(dbgs() << " total PHI alignment(" << minPHIAlignment << ")\n");
2887 return minPHIAlignment;
2888 }
2889
2890 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: ptr)) {
2891 auto *GEPPtr = GEP->getPointerOperand();
2892 // Only if this is the induction variable with const offset
2893 // Implicit assumption is that induction variable itself is a PHI
2894 if (&In == GEPPtr) {
2895 APInt Offset(HVC.DL.getPointerSizeInBits(
2896 AS: GEPPtr->getType()->getPointerAddressSpace()),
2897 0);
2898 if (GEP->accumulateConstantOffset(DL: HVC.DL, Offset)) {
2899 LLVM_DEBUG(dbgs() << " Induction GEP with const step of ("
2900 << Offset.getZExtValue() << ")\n");
2901 return Offset.getZExtValue();
2902 }
2903 }
2904 }
2905
2906 return std::nullopt;
2907}
2908
2909Value *HvxIdioms::processMStore(Instruction &In) const {
2910 [[maybe_unused]] auto *InpTy =
2911 dyn_cast<VectorType>(Val: In.getOperand(i: 0)->getType());
2912 assert(InpTy && "Cannot handle no vector type for llvm.masked.store");
2913
2914 LLVM_DEBUG(dbgs() << "\n[Process mstore](" << In << ")\n"
2915 << *In.getParent() << "\n");
2916 LLVM_DEBUG(dbgs() << " Input type(" << *InpTy << ") elements("
2917 << HVC.length(InpTy) << ") VecLen(" << HVC.getSizeOf(InpTy)
2918 << ") type(" << *InpTy->getElementType() << ") of size("
2919 << InpTy->getScalarSizeInBits() << ")bits\n");
2920 auto *CI = dyn_cast<CallBase>(Val: &In);
2921 assert(CI && "Expected llvm.masked.store to be a call");
2922 Align HaveAlign = CI->getParamAlign(ArgNo: 1).valueOrOne();
2923
2924 uint64_t KA = 1;
2925 if (auto res = getAlignment(In, ptr: In.getOperand(i: 1))) // ptr operand
2926 KA = *res;
2927 LLVM_DEBUG(dbgs() << " HaveAlign(" << HaveAlign.value() << ") KnownAlign("
2928 << KA << ")\n");
2929 // Normalize 0 -> ABI alignment of the stored value type (operand 0).
2930 Type *ValTy = In.getOperand(i: 0)->getType();
2931 Align EffA =
2932 (KA > 0) ? Align(KA) : Align(HVC.DL.getABITypeAlign(Ty: ValTy).value());
2933
2934 if (EffA < HaveAlign)
2935 return nullptr;
2936
2937 // Attach/replace the param attribute on pointer param #1.
2938 AttrBuilder AttrB(CI->getContext());
2939 AttrB.addAlignmentAttr(Align: EffA);
2940 CI->setAttributes(
2941 CI->getAttributes().addParamAttributes(C&: CI->getContext(), ArgNo: 1, B: AttrB));
2942 return CI;
2943}
2944
2945Value *HvxIdioms::processMLoad(Instruction &In) const {
2946 [[maybe_unused]] auto *InpTy = dyn_cast<VectorType>(Val: In.getType());
2947 assert(InpTy && "Cannot handle non vector type for llvm.masked.store");
2948 LLVM_DEBUG(dbgs() << "\n[Process mload](" << In << ")\n"
2949 << *In.getParent() << "\n");
2950 LLVM_DEBUG(dbgs() << " Input type(" << *InpTy << ") elements("
2951 << HVC.length(InpTy) << ") VecLen(" << HVC.getSizeOf(InpTy)
2952 << ") type(" << *InpTy->getElementType() << ") of size("
2953 << InpTy->getScalarSizeInBits() << ")bits\n");
2954 auto *CI = dyn_cast<CallBase>(Val: &In);
2955 assert(CI && "Expected to be a call to llvm.masked.load");
2956 // The pointer is operand #0, and its param attribute index is also 0.
2957 Align HaveAlign = CI->getParamAlign(ArgNo: 0).valueOrOne();
2958
2959 // Compute best-known alignment KA from analysis.
2960 uint64_t KA = 1;
2961 if (auto res = getAlignment(In, ptr: In.getOperand(i: 0))) // ptr operand
2962 KA = *res;
2963
2964 // Normalize 0 → ABI alignment of the loaded value type.
2965 Type *ValTy = In.getType();
2966 Align EffA =
2967 (KA > 0) ? Align(KA) : Align(HVC.DL.getABITypeAlign(Ty: ValTy).value());
2968 if (EffA < HaveAlign)
2969 return nullptr;
2970 LLVM_DEBUG(dbgs() << " HaveAlign(" << HaveAlign.value() << ") KnownAlign("
2971 << KA << ")\n");
2972
2973 // Attach/replace the param attribute on pointer param #0.
2974 AttrBuilder AttrB(CI->getContext());
2975 AttrB.addAlignmentAttr(Align: EffA);
2976 CI->setAttributes(
2977 CI->getAttributes().addParamAttributes(C&: CI->getContext(), ArgNo: 0, B: AttrB));
2978 return CI;
2979}
2980
2981auto HvxIdioms::processFxpMulChopped(IRBuilderBase &Builder, Instruction &In,
2982 const FxpOp &Op) const -> Value * {
2983 assert(Op.X.Val->getType() == Op.Y.Val->getType());
2984 auto *InpTy = cast<VectorType>(Val: Op.X.Val->getType());
2985 unsigned Width = InpTy->getScalarSizeInBits();
2986 bool Rounding = Op.RoundAt.has_value();
2987
2988 if (!Op.RoundAt || *Op.RoundAt == Op.Frac - 1) {
2989 // The fixed-point intrinsics do signed multiplication.
2990 if (Width == Op.Frac + 1 && Op.X.Sgn != Unsigned && Op.Y.Sgn != Unsigned) {
2991 Value *QMul = nullptr;
2992 if (Width == 16) {
2993 QMul = createMulQ15(Builder, X: Op.X, Y: Op.Y, Rounding);
2994 } else if (Width == 32) {
2995 QMul = createMulQ31(Builder, X: Op.X, Y: Op.Y, Rounding);
2996 }
2997 if (QMul != nullptr)
2998 return QMul;
2999 }
3000 }
3001
3002 assert(Width >= 32 || isPowerOf2_32(Width)); // Width <= 32 => Width is 2^n
3003 assert(Width < 32 || Width % 32 == 0); // Width > 32 => Width is 32*k
3004
3005 // If Width < 32, then it should really be 16.
3006 if (Width < 32) {
3007 if (Width < 16)
3008 return nullptr;
3009 // Getting here with Op.Frac == 0 isn't wrong, but suboptimal: here we
3010 // generate a full precision products, which is unnecessary if there is
3011 // no shift.
3012 assert(Width == 16);
3013 assert(Op.Frac != 0 && "Unshifted mul should have been skipped");
3014 if (Op.Frac == 16) {
3015 // Multiply high
3016 if (Value *MulH = createMulH16(Builder, X: Op.X, Y: Op.Y))
3017 return MulH;
3018 }
3019 // Do full-precision multiply and shift.
3020 Value *Prod32 = createMul16(Builder, X: Op.X, Y: Op.Y);
3021 if (Rounding) {
3022 Value *RoundVal =
3023 ConstantInt::get(Ty: Prod32->getType(), V: 1ull << *Op.RoundAt);
3024 Prod32 = Builder.CreateAdd(LHS: Prod32, RHS: RoundVal, Name: "add");
3025 }
3026
3027 Value *ShiftAmt = ConstantInt::get(Ty: Prod32->getType(), V: Op.Frac);
3028 Value *Shifted = Op.X.Sgn == Signed || Op.Y.Sgn == Signed
3029 ? Builder.CreateAShr(LHS: Prod32, RHS: ShiftAmt, Name: "asr")
3030 : Builder.CreateLShr(LHS: Prod32, RHS: ShiftAmt, Name: "lsr");
3031 return Builder.CreateTrunc(V: Shifted, DestTy: InpTy, Name: "trn");
3032 }
3033
3034 // Width >= 32
3035
3036 // Break up the arguments Op.X and Op.Y into vectors of smaller widths
3037 // in preparation of doing the multiplication by 32-bit parts.
3038 auto WordX = HVC.splitVectorElements(Builder, Vec: Op.X.Val, /*ToWidth=*/32);
3039 auto WordY = HVC.splitVectorElements(Builder, Vec: Op.Y.Val, /*ToWidth=*/32);
3040 auto WordP = createMulLong(Builder, WordX, SgnX: Op.X.Sgn, WordY, SgnY: Op.Y.Sgn);
3041
3042 auto *HvxWordTy = cast<VectorType>(Val: WordP.front()->getType());
3043
3044 // Add the optional rounding to the proper word.
3045 if (Op.RoundAt.has_value()) {
3046 Value *Zero = Constant::getNullValue(Ty: WordX[0]->getType());
3047 SmallVector<Value *> RoundV(WordP.size(), Zero);
3048 RoundV[*Op.RoundAt / 32] =
3049 ConstantInt::get(Ty: HvxWordTy, V: 1ull << (*Op.RoundAt % 32));
3050 WordP = createAddLong(Builder, WordX: WordP, WordY: RoundV);
3051 }
3052
3053 // createRightShiftLong?
3054
3055 // Shift all products right by Op.Frac.
3056 unsigned SkipWords = Op.Frac / 32;
3057 Constant *ShiftAmt = ConstantInt::get(Ty: HvxWordTy, V: Op.Frac % 32);
3058
3059 for (int Dst = 0, End = WordP.size() - SkipWords; Dst != End; ++Dst) {
3060 int Src = Dst + SkipWords;
3061 Value *Lo = WordP[Src];
3062 if (Src + 1 < End) {
3063 Value *Hi = WordP[Src + 1];
3064 WordP[Dst] = Builder.CreateIntrinsic(RetTy: HvxWordTy, ID: Intrinsic::fshr,
3065 Args: {Hi, Lo, ShiftAmt},
3066 /*FMFSource*/ nullptr, Name: "int");
3067 } else {
3068 // The shift of the most significant word.
3069 WordP[Dst] = Builder.CreateAShr(LHS: Lo, RHS: ShiftAmt, Name: "asr");
3070 }
3071 }
3072 if (SkipWords != 0)
3073 WordP.resize(N: WordP.size() - SkipWords);
3074
3075 return HVC.joinVectorElements(Builder, Values: WordP, ToType: Op.ResTy);
3076}
3077
3078auto HvxIdioms::createMulQ15(IRBuilderBase &Builder, SValue X, SValue Y,
3079 bool Rounding) const -> Value * {
3080 assert(X.Val->getType() == Y.Val->getType());
3081 assert(X.Val->getType()->getScalarType() == HVC.getIntTy(16));
3082 assert(HVC.HST.isHVXVectorType(EVT::getEVT(X.Val->getType(), false)));
3083
3084 // There is no non-rounding intrinsic for i16.
3085 if (!Rounding || X.Sgn == Unsigned || Y.Sgn == Unsigned)
3086 return nullptr;
3087
3088 auto V6_vmpyhvsrs = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyhvsrs);
3089 return HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyhvsrs, RetTy: X.Val->getType(),
3090 Args: {X.Val, Y.Val});
3091}
3092
3093auto HvxIdioms::createMulQ31(IRBuilderBase &Builder, SValue X, SValue Y,
3094 bool Rounding) const -> Value * {
3095 Type *InpTy = X.Val->getType();
3096 assert(InpTy == Y.Val->getType());
3097 assert(InpTy->getScalarType() == HVC.getIntTy(32));
3098 assert(HVC.HST.isHVXVectorType(EVT::getEVT(InpTy, false)));
3099
3100 if (X.Sgn == Unsigned || Y.Sgn == Unsigned)
3101 return nullptr;
3102
3103 auto V6_vmpyewuh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyewuh);
3104 auto V6_vmpyo_acc = Rounding
3105 ? HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyowh_rnd_sacc)
3106 : HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyowh_sacc);
3107 Value *V1 =
3108 HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyewuh, RetTy: InpTy, Args: {X.Val, Y.Val});
3109 return HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyo_acc, RetTy: InpTy,
3110 Args: {V1, X.Val, Y.Val});
3111}
3112
3113auto HvxIdioms::createAddCarry(IRBuilderBase &Builder, Value *X, Value *Y,
3114 Value *CarryIn) const
3115 -> std::pair<Value *, Value *> {
3116 assert(X->getType() == Y->getType());
3117 auto VecTy = cast<VectorType>(Val: X->getType());
3118 if (VecTy == HvxI32Ty && HVC.HST.useHVXV62Ops()) {
3119 SmallVector<Value *> Args = {X, Y};
3120 Intrinsic::ID AddCarry;
3121 if (CarryIn == nullptr && HVC.HST.useHVXV66Ops()) {
3122 AddCarry = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaddcarryo);
3123 } else {
3124 AddCarry = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vaddcarry);
3125 if (CarryIn == nullptr)
3126 CarryIn = Constant::getNullValue(Ty: HVC.getBoolTy(ElemCount: HVC.length(Ty: VecTy)));
3127 Args.push_back(Elt: CarryIn);
3128 }
3129 Value *Ret = HVC.createHvxIntrinsic(Builder, IntID: AddCarry,
3130 /*RetTy=*/nullptr, Args);
3131 Value *Result = Builder.CreateExtractValue(Agg: Ret, Idxs: {0}, Name: "ext");
3132 Value *CarryOut = Builder.CreateExtractValue(Agg: Ret, Idxs: {1}, Name: "ext");
3133 return {Result, CarryOut};
3134 }
3135
3136 // In other cases, do a regular add, and unsigned compare-less-than.
3137 // The carry-out can originate in two places: adding the carry-in or adding
3138 // the two input values.
3139 Value *Result1 = X; // Result1 = X + CarryIn
3140 if (CarryIn != nullptr) {
3141 unsigned Width = VecTy->getScalarSizeInBits();
3142 uint32_t Mask = 1;
3143 if (Width < 32) {
3144 for (unsigned i = 0, e = 32 / Width; i != e; ++i)
3145 Mask = (Mask << Width) | 1;
3146 }
3147 auto V6_vandqrt = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vandqrt);
3148 Value *ValueIn =
3149 HVC.createHvxIntrinsic(Builder, IntID: V6_vandqrt, /*RetTy=*/nullptr,
3150 Args: {CarryIn, HVC.getConstInt(Val: Mask)});
3151 Result1 = Builder.CreateAdd(LHS: X, RHS: ValueIn, Name: "add");
3152 }
3153
3154 Value *CarryOut1 = Builder.CreateCmp(Pred: CmpInst::ICMP_ULT, LHS: Result1, RHS: X, Name: "cmp");
3155 Value *Result2 = Builder.CreateAdd(LHS: Result1, RHS: Y, Name: "add");
3156 Value *CarryOut2 = Builder.CreateCmp(Pred: CmpInst::ICMP_ULT, LHS: Result2, RHS: Y, Name: "cmp");
3157 return {Result2, Builder.CreateOr(LHS: CarryOut1, RHS: CarryOut2, Name: "orb")};
3158}
3159
3160auto HvxIdioms::createMul16(IRBuilderBase &Builder, SValue X, SValue Y) const
3161 -> Value * {
3162 Intrinsic::ID V6_vmpyh = 0;
3163 std::tie(args&: X, args&: Y) = canonSgn(X, Y);
3164
3165 if (X.Sgn == Signed) {
3166 V6_vmpyh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyhv);
3167 } else if (Y.Sgn == Signed) {
3168 // In vmpyhus the second operand is unsigned
3169 V6_vmpyh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyhus);
3170 } else {
3171 V6_vmpyh = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyuhv);
3172 }
3173
3174 // i16*i16 -> i32 / interleaved
3175 Value *P =
3176 HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyh, RetTy: HvxP32Ty, Args: {Y.Val, X.Val});
3177 // Deinterleave
3178 return HVC.vshuff(Builder, Val0: HVC.sublo(Builder, Val: P), Val1: HVC.subhi(Builder, Val: P));
3179}
3180
3181auto HvxIdioms::createMulH16(IRBuilderBase &Builder, SValue X, SValue Y) const
3182 -> Value * {
3183 Type *HvxI16Ty = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), /*Pair=*/false);
3184
3185 if (HVC.HST.useHVXV69Ops()) {
3186 if (X.Sgn != Signed && Y.Sgn != Signed) {
3187 auto V6_vmpyuhvs = HVC.HST.getIntrinsicId(Opc: Hexagon::V6_vmpyuhvs);
3188 return HVC.createHvxIntrinsic(Builder, IntID: V6_vmpyuhvs, RetTy: HvxI16Ty,
3189 Args: {X.Val, Y.Val});
3190 }
3191 }
3192
3193 Type *HvxP16Ty = HVC.getHvxTy(ElemTy: HVC.getIntTy(Width: 16), /*Pair=*/true);
3194 Value *Pair16 =
3195 Builder.CreateBitCast(V: createMul16(Builder, X, Y), DestTy: HvxP16Ty, Name: "cst");
3196 unsigned Len = HVC.length(Ty: HvxP16Ty) / 2;
3197
3198 SmallVector<int, 128> PickOdd(Len);
3199 for (int i = 0; i != static_cast<int>(Len); ++i)
3200 PickOdd[i] = 2 * i + 1;
3201
3202 return Builder.CreateShuffleVector(
3203 V1: HVC.sublo(Builder, Val: Pair16), V2: HVC.subhi(Builder, Val: Pair16), Mask: PickOdd, Name: "shf");
3204}
3205
3206auto HvxIdioms::createMul32(IRBuilderBase &Builder, SValue X, SValue Y) const
3207 -> std::pair<Value *, Value *> {
3208 assert(X.Val->getType() == Y.Val->getType());
3209 assert(X.Val->getType() == HvxI32Ty);
3210
3211 Intrinsic::ID V6_vmpy_parts;
3212 std::tie(args&: X, args&: Y) = canonSgn(X, Y);
3213
3214 if (X.Sgn == Signed) {
3215 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyss_parts;
3216 } else if (Y.Sgn == Signed) {
3217 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyus_parts;
3218 } else {
3219 V6_vmpy_parts = Intrinsic::hexagon_V6_vmpyuu_parts;
3220 }
3221
3222 Value *Parts = HVC.createHvxIntrinsic(Builder, IntID: V6_vmpy_parts, RetTy: nullptr,
3223 Args: {X.Val, Y.Val}, ArgTys: {HvxI32Ty});
3224 Value *Hi = Builder.CreateExtractValue(Agg: Parts, Idxs: {0}, Name: "ext");
3225 Value *Lo = Builder.CreateExtractValue(Agg: Parts, Idxs: {1}, Name: "ext");
3226 return {Lo, Hi};
3227}
3228
3229auto HvxIdioms::createAddLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
3230 ArrayRef<Value *> WordY) const
3231 -> SmallVector<Value *> {
3232 assert(WordX.size() == WordY.size());
3233 unsigned Idx = 0, Length = WordX.size();
3234 SmallVector<Value *> Sum(Length);
3235
3236 while (Idx != Length) {
3237 if (HVC.isZero(Val: WordX[Idx]))
3238 Sum[Idx] = WordY[Idx];
3239 else if (HVC.isZero(Val: WordY[Idx]))
3240 Sum[Idx] = WordX[Idx];
3241 else
3242 break;
3243 ++Idx;
3244 }
3245
3246 Value *Carry = nullptr;
3247 for (; Idx != Length; ++Idx) {
3248 std::tie(args&: Sum[Idx], args&: Carry) =
3249 createAddCarry(Builder, X: WordX[Idx], Y: WordY[Idx], CarryIn: Carry);
3250 }
3251
3252 // This drops the final carry beyond the highest word.
3253 return Sum;
3254}
3255
3256auto HvxIdioms::createMulLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX,
3257 Signedness SgnX, ArrayRef<Value *> WordY,
3258 Signedness SgnY) const -> SmallVector<Value *> {
3259 SmallVector<SmallVector<Value *>> Products(WordX.size() + WordY.size());
3260
3261 // WordX[i] * WordY[j] produces words i+j and i+j+1 of the results,
3262 // that is halves 2(i+j), 2(i+j)+1, 2(i+j)+2, 2(i+j)+3.
3263 for (int i = 0, e = WordX.size(); i != e; ++i) {
3264 for (int j = 0, f = WordY.size(); j != f; ++j) {
3265 // Check the 4 halves that this multiplication can generate.
3266 Signedness SX = (i + 1 == e) ? SgnX : Unsigned;
3267 Signedness SY = (j + 1 == f) ? SgnY : Unsigned;
3268 auto [Lo, Hi] = createMul32(Builder, X: {.Val: WordX[i], .Sgn: SX}, Y: {.Val: WordY[j], .Sgn: SY});
3269 Products[i + j + 0].push_back(Elt: Lo);
3270 Products[i + j + 1].push_back(Elt: Hi);
3271 }
3272 }
3273
3274 Value *Zero = Constant::getNullValue(Ty: WordX[0]->getType());
3275
3276 auto pop_back_or_zero = [Zero](auto &Vector) -> Value * {
3277 if (Vector.empty())
3278 return Zero;
3279 auto Last = Vector.back();
3280 Vector.pop_back();
3281 return Last;
3282 };
3283
3284 for (int i = 0, e = Products.size(); i != e; ++i) {
3285 while (Products[i].size() > 1) {
3286 Value *Carry = nullptr; // no carry-in
3287 for (int j = i; j != e; ++j) {
3288 auto &ProdJ = Products[j];
3289 auto [Sum, CarryOut] = createAddCarry(Builder, X: pop_back_or_zero(ProdJ),
3290 Y: pop_back_or_zero(ProdJ), CarryIn: Carry);
3291 ProdJ.insert(I: ProdJ.begin(), Elt: Sum);
3292 Carry = CarryOut;
3293 }
3294 }
3295 }
3296
3297 SmallVector<Value *> WordP;
3298 for (auto &P : Products) {
3299 assert(P.size() == 1 && "Should have been added together");
3300 WordP.push_back(Elt: P.front());
3301 }
3302
3303 return WordP;
3304}
3305
3306auto HvxIdioms::run() -> bool {
3307 bool Changed = false;
3308
3309 for (BasicBlock &B : HVC.F) {
3310 for (auto It = B.rbegin(); It != B.rend(); ++It) {
3311 if (auto Fxm = matchFxpMul(In&: *It)) {
3312 Value *New = processFxpMul(In&: *It, Op: *Fxm);
3313 // Always report "changed" for now.
3314 Changed = true;
3315 if (!New)
3316 continue;
3317 bool StartOver = !isa<Instruction>(Val: New);
3318 It->replaceAllUsesWith(V: New);
3319 RecursivelyDeleteTriviallyDeadInstructions(V: &*It, TLI: &HVC.TLI);
3320 It = StartOver ? B.rbegin()
3321 : cast<Instruction>(Val: New)->getReverseIterator();
3322 Changed = true;
3323 } else if (matchGather(In&: *It)) {
3324 Value *New = processVGather(In&: *It);
3325 if (!New)
3326 continue;
3327 LLVM_DEBUG(dbgs() << " Gather : " << *New << "\n");
3328 // We replace original intrinsic with a new pseudo call.
3329 It->eraseFromParent();
3330 It = cast<Instruction>(Val: New)->getReverseIterator();
3331 RecursivelyDeleteTriviallyDeadInstructions(V: &*It, TLI: &HVC.TLI);
3332 Changed = true;
3333 } else if (matchScatter(In&: *It)) {
3334 Value *New = processVScatter(In&: *It);
3335 if (!New)
3336 continue;
3337 LLVM_DEBUG(dbgs() << " Scatter : " << *New << "\n");
3338 // We replace original intrinsic with a new pseudo call.
3339 It->eraseFromParent();
3340 It = cast<Instruction>(Val: New)->getReverseIterator();
3341 RecursivelyDeleteTriviallyDeadInstructions(V: &*It, TLI: &HVC.TLI);
3342 Changed = true;
3343 } else if (matchMLoad(In&: *It)) {
3344 Value *New = processMLoad(In&: *It);
3345 if (!New)
3346 continue;
3347 LLVM_DEBUG(dbgs() << " MLoad : " << *New << "\n");
3348 Changed = true;
3349 } else if (matchMStore(In&: *It)) {
3350 Value *New = processMStore(In&: *It);
3351 if (!New)
3352 continue;
3353 LLVM_DEBUG(dbgs() << " MStore : " << *New << "\n");
3354 Changed = true;
3355 }
3356 }
3357 }
3358
3359 return Changed;
3360}
3361
3362// --- End HvxIdioms
3363
3364auto HexagonVectorCombine::run() -> bool {
3365 if (DumpModule)
3366 dbgs() << "Module before HexagonVectorCombine\n" << *F.getParent();
3367
3368 bool Changed = false;
3369 if (HST.useHVXOps()) {
3370 if (VAEnabled)
3371 Changed |= AlignVectors(*this).run();
3372 if (VIEnabled)
3373 Changed |= HvxIdioms(*this).run();
3374 }
3375
3376 if (DumpModule) {
3377 dbgs() << "Module " << (Changed ? "(modified)" : "(unchanged)")
3378 << " after HexagonVectorCombine\n"
3379 << *F.getParent();
3380 }
3381 return Changed;
3382}
3383
3384auto HexagonVectorCombine::getIntTy(unsigned Width) const -> IntegerType * {
3385 return IntegerType::get(C&: F.getContext(), NumBits: Width);
3386}
3387
3388auto HexagonVectorCombine::getByteTy(int ElemCount) const -> Type * {
3389 assert(ElemCount >= 0);
3390 IntegerType *ByteTy = Type::getInt8Ty(C&: F.getContext());
3391 if (ElemCount == 0)
3392 return ByteTy;
3393 return VectorType::get(ElementType: ByteTy, NumElements: ElemCount, /*Scalable=*/false);
3394}
3395
3396auto HexagonVectorCombine::getBoolTy(int ElemCount) const -> Type * {
3397 assert(ElemCount >= 0);
3398 IntegerType *BoolTy = Type::getInt1Ty(C&: F.getContext());
3399 if (ElemCount == 0)
3400 return BoolTy;
3401 return VectorType::get(ElementType: BoolTy, NumElements: ElemCount, /*Scalable=*/false);
3402}
3403
3404auto HexagonVectorCombine::getConstInt(int Val, unsigned Width) const
3405 -> ConstantInt * {
3406 return ConstantInt::getSigned(Ty: getIntTy(Width), V: Val);
3407}
3408
3409auto HexagonVectorCombine::isZero(const Value *Val) const -> bool {
3410 if (auto *C = dyn_cast<Constant>(Val))
3411 return C->isNullValue();
3412 return false;
3413}
3414
3415auto HexagonVectorCombine::getIntValue(const Value *Val) const
3416 -> std::optional<APInt> {
3417 if (auto *CI = dyn_cast<ConstantInt>(Val))
3418 return CI->getValue();
3419 return std::nullopt;
3420}
3421
3422auto HexagonVectorCombine::isUndef(const Value *Val) const -> bool {
3423 return isa<UndefValue>(Val);
3424}
3425
3426auto HexagonVectorCombine::isTrue(const Value *Val) const -> bool {
3427 return Val == ConstantInt::getTrue(Ty: Val->getType());
3428}
3429
3430auto HexagonVectorCombine::isFalse(const Value *Val) const -> bool {
3431 return isZero(Val);
3432}
3433
3434auto HexagonVectorCombine::getHvxTy(Type *ElemTy, bool Pair) const
3435 -> VectorType * {
3436 EVT ETy = EVT::getEVT(Ty: ElemTy, HandleUnknown: false);
3437 assert(ETy.isSimple() && "Invalid HVX element type");
3438 // Do not allow boolean types here: they don't have a fixed length.
3439 assert(HST.isHVXElementType(ETy.getSimpleVT(), /*IncludeBool=*/false) &&
3440 "Invalid HVX element type");
3441 unsigned HwLen = HST.getVectorLength();
3442 unsigned NumElems = (8 * HwLen) / ETy.getSizeInBits();
3443 return VectorType::get(ElementType: ElemTy, NumElements: Pair ? 2 * NumElems : NumElems,
3444 /*Scalable=*/false);
3445}
3446
3447auto HexagonVectorCombine::getSizeOf(const Value *Val, SizeKind Kind) const
3448 -> int {
3449 return getSizeOf(Ty: Val->getType(), Kind);
3450}
3451
3452auto HexagonVectorCombine::getSizeOf(const Type *Ty, SizeKind Kind) const
3453 -> int {
3454 auto *NcTy = const_cast<Type *>(Ty);
3455 switch (Kind) {
3456 case Store:
3457 return DL.getTypeStoreSize(Ty: NcTy).getFixedValue();
3458 case Alloc:
3459 return DL.getTypeAllocSize(Ty: NcTy).getFixedValue();
3460 }
3461 llvm_unreachable("Unhandled SizeKind enum");
3462}
3463
3464auto HexagonVectorCombine::getTypeAlignment(Type *Ty) const -> int {
3465 // The actual type may be shorter than the HVX vector, so determine
3466 // the alignment based on subtarget info.
3467 if (HST.isTypeForHVX(VecTy: Ty))
3468 return HST.getVectorLength();
3469 return DL.getABITypeAlign(Ty).value();
3470}
3471
3472auto HexagonVectorCombine::length(Value *Val) const -> size_t {
3473 return length(Ty: Val->getType());
3474}
3475
3476auto HexagonVectorCombine::length(Type *Ty) const -> size_t {
3477 auto *VecTy = dyn_cast<VectorType>(Val: Ty);
3478 assert(VecTy && "Must be a vector type");
3479 return VecTy->getElementCount().getFixedValue();
3480}
3481
3482auto HexagonVectorCombine::simplify(Value *V) const -> Value * {
3483 if (auto *In = dyn_cast<Instruction>(Val: V)) {
3484 SimplifyQuery Q(DL, &TLI, &DT, &AC, In);
3485 return simplifyInstruction(I: In, Q);
3486 }
3487 return nullptr;
3488}
3489
3490// Insert bytes [Start..Start+Length) of Src into Dst at byte Where.
3491auto HexagonVectorCombine::insertb(IRBuilderBase &Builder, Value *Dst,
3492 Value *Src, int Start, int Length,
3493 int Where) const -> Value * {
3494 assert(isByteVecTy(Dst->getType()) && isByteVecTy(Src->getType()));
3495 int SrcLen = getSizeOf(Val: Src);
3496 int DstLen = getSizeOf(Val: Dst);
3497 assert(0 <= Start && Start + Length <= SrcLen);
3498 assert(0 <= Where && Where + Length <= DstLen);
3499
3500 int P2Len = PowerOf2Ceil(A: SrcLen | DstLen);
3501 auto *Poison = PoisonValue::get(T: getByteTy());
3502 Value *P2Src = vresize(Builder, Val: Src, NewSize: P2Len, Pad: Poison);
3503 Value *P2Dst = vresize(Builder, Val: Dst, NewSize: P2Len, Pad: Poison);
3504
3505 SmallVector<int, 256> SMask(P2Len);
3506 for (int i = 0; i != P2Len; ++i) {
3507 // If i is in [Where, Where+Length), pick Src[Start+(i-Where)].
3508 // Otherwise, pick Dst[i];
3509 SMask[i] =
3510 (Where <= i && i < Where + Length) ? P2Len + Start + (i - Where) : i;
3511 }
3512
3513 Value *P2Insert = Builder.CreateShuffleVector(V1: P2Dst, V2: P2Src, Mask: SMask, Name: "shf");
3514 return vresize(Builder, Val: P2Insert, NewSize: DstLen, Pad: Poison);
3515}
3516
3517auto HexagonVectorCombine::vlalignb(IRBuilderBase &Builder, Value *Lo,
3518 Value *Hi, Value *Amt) const -> Value * {
3519 assert(Lo->getType() == Hi->getType() && "Argument type mismatch");
3520 if (isZero(Val: Amt))
3521 return Hi;
3522 int VecLen = getSizeOf(Val: Hi);
3523 if (auto IntAmt = getIntValue(Val: Amt))
3524 return getElementRange(Builder, Lo, Hi, Start: VecLen - IntAmt->getSExtValue(),
3525 Length: VecLen);
3526
3527 if (HST.isTypeForHVX(VecTy: Hi->getType())) {
3528 assert(static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
3529 "Expecting an exact HVX type");
3530 return createHvxIntrinsic(Builder, IntID: HST.getIntrinsicId(Opc: Hexagon::V6_vlalignb),
3531 RetTy: Hi->getType(), Args: {Hi, Lo, Amt});
3532 }
3533
3534 if (VecLen == 4) {
3535 Value *Pair = concat(Builder, Vecs: {Lo, Hi});
3536 Value *Shift =
3537 Builder.CreateLShr(LHS: Builder.CreateShl(LHS: Pair, RHS: Amt, Name: "shl"), RHS: 32, Name: "lsr");
3538 Value *Trunc =
3539 Builder.CreateTrunc(V: Shift, DestTy: Type::getInt32Ty(C&: F.getContext()), Name: "trn");
3540 return Builder.CreateBitCast(V: Trunc, DestTy: Hi->getType(), Name: "cst");
3541 }
3542 if (VecLen == 8) {
3543 Value *Sub = Builder.CreateSub(LHS: getConstInt(Val: VecLen), RHS: Amt, Name: "sub");
3544 return vralignb(Builder, Lo, Hi, Amt: Sub);
3545 }
3546 llvm_unreachable("Unexpected vector length");
3547}
3548
3549auto HexagonVectorCombine::vralignb(IRBuilderBase &Builder, Value *Lo,
3550 Value *Hi, Value *Amt) const -> Value * {
3551 assert(Lo->getType() == Hi->getType() && "Argument type mismatch");
3552 if (isZero(Val: Amt))
3553 return Lo;
3554 int VecLen = getSizeOf(Val: Lo);
3555 if (auto IntAmt = getIntValue(Val: Amt))
3556 return getElementRange(Builder, Lo, Hi, Start: IntAmt->getSExtValue(), Length: VecLen);
3557
3558 if (HST.isTypeForHVX(VecTy: Lo->getType())) {
3559 assert(static_cast<unsigned>(VecLen) == HST.getVectorLength() &&
3560 "Expecting an exact HVX type");
3561 return createHvxIntrinsic(Builder, IntID: HST.getIntrinsicId(Opc: Hexagon::V6_valignb),
3562 RetTy: Lo->getType(), Args: {Hi, Lo, Amt});
3563 }
3564
3565 if (VecLen == 4) {
3566 Value *Pair = concat(Builder, Vecs: {Lo, Hi});
3567 Value *Shift = Builder.CreateLShr(LHS: Pair, RHS: Amt, Name: "lsr");
3568 Value *Trunc =
3569 Builder.CreateTrunc(V: Shift, DestTy: Type::getInt32Ty(C&: F.getContext()), Name: "trn");
3570 return Builder.CreateBitCast(V: Trunc, DestTy: Lo->getType(), Name: "cst");
3571 }
3572 if (VecLen == 8) {
3573 Type *Int64Ty = Type::getInt64Ty(C&: F.getContext());
3574 Value *Lo64 = Builder.CreateBitCast(V: Lo, DestTy: Int64Ty, Name: "cst");
3575 Value *Hi64 = Builder.CreateBitCast(V: Hi, DestTy: Int64Ty, Name: "cst");
3576 Value *Call = Builder.CreateIntrinsic(ID: Intrinsic::hexagon_S2_valignrb,
3577 Args: {Hi64, Lo64, Amt},
3578 /*FMFSource=*/nullptr, Name: "cup");
3579 return Builder.CreateBitCast(V: Call, DestTy: Lo->getType(), Name: "cst");
3580 }
3581 llvm_unreachable("Unexpected vector length");
3582}
3583
3584// Concatenates a sequence of vectors of the same type.
3585auto HexagonVectorCombine::concat(IRBuilderBase &Builder,
3586 ArrayRef<Value *> Vecs) const -> Value * {
3587 assert(!Vecs.empty());
3588 SmallVector<int, 256> SMask;
3589 std::vector<Value *> Work[2];
3590 int ThisW = 0, OtherW = 1;
3591
3592 Work[ThisW].assign(first: Vecs.begin(), last: Vecs.end());
3593 while (Work[ThisW].size() > 1) {
3594 auto *Ty = cast<VectorType>(Val: Work[ThisW].front()->getType());
3595 SMask.resize(N: length(Ty) * 2);
3596 std::iota(first: SMask.begin(), last: SMask.end(), value: 0);
3597
3598 Work[OtherW].clear();
3599 if (Work[ThisW].size() % 2 != 0)
3600 Work[ThisW].push_back(x: UndefValue::get(T: Ty));
3601 for (int i = 0, e = Work[ThisW].size(); i < e; i += 2) {
3602 Value *Joined = Builder.CreateShuffleVector(
3603 V1: Work[ThisW][i], V2: Work[ThisW][i + 1], Mask: SMask, Name: "shf");
3604 Work[OtherW].push_back(x: Joined);
3605 }
3606 std::swap(a&: ThisW, b&: OtherW);
3607 }
3608
3609 // Since there may have been some undefs appended to make shuffle operands
3610 // have the same type, perform the last shuffle to only pick the original
3611 // elements.
3612 SMask.resize(N: Vecs.size() * length(Ty: Vecs.front()->getType()));
3613 std::iota(first: SMask.begin(), last: SMask.end(), value: 0);
3614 Value *Total = Work[ThisW].front();
3615 return Builder.CreateShuffleVector(V: Total, Mask: SMask, Name: "shf");
3616}
3617
3618auto HexagonVectorCombine::vresize(IRBuilderBase &Builder, Value *Val,
3619 int NewSize, Value *Pad) const -> Value * {
3620 assert(isa<VectorType>(Val->getType()));
3621 auto *ValTy = cast<VectorType>(Val: Val->getType());
3622 assert(ValTy->getElementType() == Pad->getType());
3623
3624 int CurSize = length(Ty: ValTy);
3625 if (CurSize == NewSize)
3626 return Val;
3627 // Truncate?
3628 if (CurSize > NewSize)
3629 return getElementRange(Builder, Lo: Val, /*Ignored*/ Hi: Val, Start: 0, Length: NewSize);
3630 // Extend.
3631 SmallVector<int, 128> SMask(NewSize);
3632 std::iota(first: SMask.begin(), last: SMask.begin() + CurSize, value: 0);
3633 std::fill(first: SMask.begin() + CurSize, last: SMask.end(), value: CurSize);
3634 Value *PadVec = Builder.CreateVectorSplat(NumElts: CurSize, V: Pad, Name: "spt");
3635 return Builder.CreateShuffleVector(V1: Val, V2: PadVec, Mask: SMask, Name: "shf");
3636}
3637
3638auto HexagonVectorCombine::rescale(IRBuilderBase &Builder, Value *Mask,
3639 Type *FromTy, Type *ToTy) const -> Value * {
3640 // Mask is a vector <N x i1>, where each element corresponds to an
3641 // element of FromTy. Remap it so that each element will correspond
3642 // to an element of ToTy.
3643 assert(isa<VectorType>(Mask->getType()));
3644
3645 Type *FromSTy = FromTy->getScalarType();
3646 Type *ToSTy = ToTy->getScalarType();
3647 if (FromSTy == ToSTy)
3648 return Mask;
3649
3650 int FromSize = getSizeOf(Ty: FromSTy);
3651 int ToSize = getSizeOf(Ty: ToSTy);
3652 assert(FromSize % ToSize == 0 || ToSize % FromSize == 0);
3653
3654 auto *MaskTy = cast<VectorType>(Val: Mask->getType());
3655 int FromCount = length(Ty: MaskTy);
3656 int ToCount = (FromCount * FromSize) / ToSize;
3657 assert((FromCount * FromSize) % ToSize == 0);
3658
3659 auto *FromITy = getIntTy(Width: FromSize * 8);
3660 auto *ToITy = getIntTy(Width: ToSize * 8);
3661
3662 // Mask <N x i1> -> sext to <N x FromTy> -> bitcast to <M x ToTy> ->
3663 // -> trunc to <M x i1>.
3664 Value *Ext = Builder.CreateSExt(
3665 V: Mask, DestTy: VectorType::get(ElementType: FromITy, NumElements: FromCount, /*Scalable=*/false), Name: "sxt");
3666 Value *Cast = Builder.CreateBitCast(
3667 V: Ext, DestTy: VectorType::get(ElementType: ToITy, NumElements: ToCount, /*Scalable=*/false), Name: "cst");
3668 return Builder.CreateTrunc(
3669 V: Cast, DestTy: VectorType::get(ElementType: getBoolTy(), NumElements: ToCount, /*Scalable=*/false), Name: "trn");
3670}
3671
3672// Bitcast to bytes, and return least significant bits.
3673auto HexagonVectorCombine::vlsb(IRBuilderBase &Builder, Value *Val) const
3674 -> Value * {
3675 Type *ScalarTy = Val->getType()->getScalarType();
3676 if (ScalarTy == getBoolTy())
3677 return Val;
3678
3679 Value *Bytes = vbytes(Builder, Val);
3680 if (auto *VecTy = dyn_cast<VectorType>(Val: Bytes->getType()))
3681 return Builder.CreateTrunc(V: Bytes, DestTy: getBoolTy(ElemCount: getSizeOf(Ty: VecTy)), Name: "trn");
3682 // If Bytes is a scalar (i.e. Val was a scalar byte), return i1, not
3683 // <1 x i1>.
3684 return Builder.CreateTrunc(V: Bytes, DestTy: getBoolTy(), Name: "trn");
3685}
3686
3687// Bitcast to bytes for non-bool. For bool, convert i1 -> i8.
3688auto HexagonVectorCombine::vbytes(IRBuilderBase &Builder, Value *Val) const
3689 -> Value * {
3690 Type *ScalarTy = Val->getType()->getScalarType();
3691 if (ScalarTy == getByteTy())
3692 return Val;
3693
3694 if (ScalarTy != getBoolTy())
3695 return Builder.CreateBitCast(V: Val, DestTy: getByteTy(ElemCount: getSizeOf(Val)), Name: "cst");
3696 // For bool, return a sext from i1 to i8.
3697 if (auto *VecTy = dyn_cast<VectorType>(Val: Val->getType()))
3698 return Builder.CreateSExt(V: Val, DestTy: VectorType::get(ElementType: getByteTy(), Other: VecTy), Name: "sxt");
3699 return Builder.CreateSExt(V: Val, DestTy: getByteTy(), Name: "sxt");
3700}
3701
3702auto HexagonVectorCombine::subvector(IRBuilderBase &Builder, Value *Val,
3703 unsigned Start, unsigned Length) const
3704 -> Value * {
3705 assert(Start + Length <= length(Val));
3706 return getElementRange(Builder, Lo: Val, /*Ignored*/ Hi: Val, Start, Length);
3707}
3708
3709auto HexagonVectorCombine::sublo(IRBuilderBase &Builder, Value *Val) const
3710 -> Value * {
3711 size_t Len = length(Val);
3712 assert(Len % 2 == 0 && "Length should be even");
3713 return subvector(Builder, Val, Start: 0, Length: Len / 2);
3714}
3715
3716auto HexagonVectorCombine::subhi(IRBuilderBase &Builder, Value *Val) const
3717 -> Value * {
3718 size_t Len = length(Val);
3719 assert(Len % 2 == 0 && "Length should be even");
3720 return subvector(Builder, Val, Start: Len / 2, Length: Len / 2);
3721}
3722
3723auto HexagonVectorCombine::vdeal(IRBuilderBase &Builder, Value *Val0,
3724 Value *Val1) const -> Value * {
3725 assert(Val0->getType() == Val1->getType());
3726 int Len = length(Val: Val0);
3727 SmallVector<int, 128> Mask(2 * Len);
3728
3729 for (int i = 0; i != Len; ++i) {
3730 Mask[i] = 2 * i; // Even
3731 Mask[i + Len] = 2 * i + 1; // Odd
3732 }
3733 return Builder.CreateShuffleVector(V1: Val0, V2: Val1, Mask, Name: "shf");
3734}
3735
3736auto HexagonVectorCombine::vshuff(IRBuilderBase &Builder, Value *Val0,
3737 Value *Val1) const -> Value * { //
3738 assert(Val0->getType() == Val1->getType());
3739 int Len = length(Val: Val0);
3740 SmallVector<int, 128> Mask(2 * Len);
3741
3742 for (int i = 0; i != Len; ++i) {
3743 Mask[2 * i + 0] = i; // Val0
3744 Mask[2 * i + 1] = i + Len; // Val1
3745 }
3746 return Builder.CreateShuffleVector(V1: Val0, V2: Val1, Mask, Name: "shf");
3747}
3748
3749auto HexagonVectorCombine::createHvxIntrinsic(IRBuilderBase &Builder,
3750 Intrinsic::ID IntID, Type *RetTy,
3751 ArrayRef<Value *> Args,
3752 ArrayRef<Type *> ArgTys,
3753 ArrayRef<Value *> MDSources) const
3754 -> Value * {
3755 auto getCast = [&](IRBuilderBase &Builder, Value *Val,
3756 Type *DestTy) -> Value * {
3757 Type *SrcTy = Val->getType();
3758 if (SrcTy == DestTy)
3759 return Val;
3760
3761 // Non-HVX type. It should be a scalar, and it should already have
3762 // a valid type.
3763 assert(HST.isTypeForHVX(SrcTy, /*IncludeBool=*/true));
3764
3765 Type *BoolTy = Type::getInt1Ty(C&: F.getContext());
3766 if (cast<VectorType>(Val: SrcTy)->getElementType() != BoolTy)
3767 return Builder.CreateBitCast(V: Val, DestTy, Name: "cst");
3768
3769 // Predicate HVX vector.
3770 unsigned HwLen = HST.getVectorLength();
3771 Intrinsic::ID TC = HwLen == 64 ? Intrinsic::hexagon_V6_pred_typecast
3772 : Intrinsic::hexagon_V6_pred_typecast_128B;
3773 return Builder.CreateIntrinsic(ID: TC, OverloadTypes: {DestTy, Val->getType()}, Args: {Val},
3774 /*FMFSource=*/nullptr, Name: "cup");
3775 };
3776
3777 Function *IntrFn =
3778 Intrinsic::getOrInsertDeclaration(M: F.getParent(), id: IntID, OverloadTys: ArgTys);
3779 FunctionType *IntrTy = IntrFn->getFunctionType();
3780
3781 SmallVector<Value *, 4> IntrArgs;
3782 for (int i = 0, e = Args.size(); i != e; ++i) {
3783 Value *A = Args[i];
3784 Type *T = IntrTy->getParamType(i);
3785 if (A->getType() != T) {
3786 IntrArgs.push_back(Elt: getCast(Builder, A, T));
3787 } else {
3788 IntrArgs.push_back(Elt: A);
3789 }
3790 }
3791 StringRef MaybeName = !IntrTy->getReturnType()->isVoidTy() ? "cup" : "";
3792 CallInst *Call = Builder.CreateCall(Callee: IntrFn, Args: IntrArgs, Name: MaybeName);
3793
3794 MemoryEffects ME = Call->getAttributes().getMemoryEffects();
3795 if (!ME.doesNotAccessMemory() && !ME.onlyAccessesInaccessibleMem())
3796 propagateMetadata(I: Call, VL: MDSources);
3797
3798 Type *CallTy = Call->getType();
3799 if (RetTy == nullptr || CallTy == RetTy)
3800 return Call;
3801 // Scalar types should have RetTy matching the call return type.
3802 assert(HST.isTypeForHVX(CallTy, /*IncludeBool=*/true));
3803 return getCast(Builder, Call, RetTy);
3804}
3805
3806auto HexagonVectorCombine::splitVectorElements(IRBuilderBase &Builder,
3807 Value *Vec,
3808 unsigned ToWidth) const
3809 -> SmallVector<Value *> {
3810 // Break a vector of wide elements into a series of vectors with narrow
3811 // elements:
3812 // (...c0:b0:a0, ...c1:b1:a1, ...c2:b2:a2, ...)
3813 // -->
3814 // (a0, a1, a2, ...) // lowest "ToWidth" bits
3815 // (b0, b1, b2, ...) // the next lowest...
3816 // (c0, c1, c2, ...) // ...
3817 // ...
3818 //
3819 // The number of elements in each resulting vector is the same as
3820 // in the original vector.
3821
3822 auto *VecTy = cast<VectorType>(Val: Vec->getType());
3823 assert(VecTy->getElementType()->isIntegerTy());
3824 unsigned FromWidth = VecTy->getScalarSizeInBits();
3825 assert(isPowerOf2_32(ToWidth) && isPowerOf2_32(FromWidth));
3826 assert(ToWidth <= FromWidth && "Breaking up into wider elements?");
3827 unsigned NumResults = FromWidth / ToWidth;
3828
3829 SmallVector<Value *> Results(NumResults);
3830 Results[0] = Vec;
3831 unsigned Length = length(Ty: VecTy);
3832
3833 // Do it by splitting in half, since those operations correspond to deal
3834 // instructions.
3835 auto splitInHalf = [&](unsigned Begin, unsigned End, auto splitFunc) -> void {
3836 // Take V = Results[Begin], split it in L, H.
3837 // Store Results[Begin] = L, Results[(Begin+End)/2] = H
3838 // Call itself recursively split(Begin, Half), split(Half+1, End)
3839 if (Begin + 1 == End)
3840 return;
3841
3842 Value *Val = Results[Begin];
3843 unsigned Width = Val->getType()->getScalarSizeInBits();
3844
3845 auto *VTy = VectorType::get(ElementType: getIntTy(Width: Width / 2), NumElements: 2 * Length, Scalable: false);
3846 Value *VVal = Builder.CreateBitCast(V: Val, DestTy: VTy, Name: "cst");
3847
3848 Value *Res = vdeal(Builder, Val0: sublo(Builder, Val: VVal), Val1: subhi(Builder, Val: VVal));
3849
3850 unsigned Half = (Begin + End) / 2;
3851 Results[Begin] = sublo(Builder, Val: Res);
3852 Results[Half] = subhi(Builder, Val: Res);
3853
3854 splitFunc(Begin, Half, splitFunc);
3855 splitFunc(Half, End, splitFunc);
3856 };
3857
3858 splitInHalf(0, NumResults, splitInHalf);
3859 return Results;
3860}
3861
3862auto HexagonVectorCombine::joinVectorElements(IRBuilderBase &Builder,
3863 ArrayRef<Value *> Values,
3864 VectorType *ToType) const
3865 -> Value * {
3866 assert(ToType->getElementType()->isIntegerTy());
3867
3868 // If the list of values does not have power-of-2 elements, append copies
3869 // of the sign bit to it, to make the size be 2^n.
3870 // The reason for this is that the values will be joined in pairs, because
3871 // otherwise the shuffles will result in convoluted code. With pairwise
3872 // joins, the shuffles will hopefully be folded into a perfect shuffle.
3873 // The output will need to be sign-extended to a type with element width
3874 // being a power-of-2 anyways.
3875 SmallVector<Value *> Inputs(Values);
3876
3877 unsigned ToWidth = ToType->getScalarSizeInBits();
3878 unsigned Width = Inputs.front()->getType()->getScalarSizeInBits();
3879 assert(Width <= ToWidth);
3880 assert(isPowerOf2_32(Width) && isPowerOf2_32(ToWidth));
3881 unsigned Length = length(Ty: Inputs.front()->getType());
3882
3883 unsigned NeedInputs = ToWidth / Width;
3884 if (Inputs.size() != NeedInputs) {
3885 // Having too many inputs is ok: drop the high bits (usual wrap-around).
3886 // If there are too few, fill them with the sign bit.
3887 Value *Last = Inputs.back();
3888 Value *Sign = Builder.CreateAShr(
3889 LHS: Last, RHS: ConstantInt::get(Ty: Last->getType(), V: Width - 1), Name: "asr");
3890 Inputs.resize(N: NeedInputs, NV: Sign);
3891 }
3892
3893 while (Inputs.size() > 1) {
3894 Width *= 2;
3895 auto *VTy = VectorType::get(ElementType: getIntTy(Width), NumElements: Length, Scalable: false);
3896 for (int i = 0, e = Inputs.size(); i < e; i += 2) {
3897 Value *Res = vshuff(Builder, Val0: Inputs[i], Val1: Inputs[i + 1]);
3898 Inputs[i / 2] = Builder.CreateBitCast(V: Res, DestTy: VTy, Name: "cst");
3899 }
3900 Inputs.resize(N: Inputs.size() / 2);
3901 }
3902
3903 assert(Inputs.front()->getType() == ToType);
3904 return Inputs.front();
3905}
3906
3907auto HexagonVectorCombine::calculatePointerDifference(Value *Ptr0,
3908 Value *Ptr1) const
3909 -> std::optional<int> {
3910 // Try SCEV first.
3911 const SCEV *Scev0 = SE.getSCEV(V: Ptr0);
3912 const SCEV *Scev1 = SE.getSCEV(V: Ptr1);
3913 const SCEV *ScevDiff = SE.getMinusSCEV(LHS: Scev0, RHS: Scev1);
3914 if (auto *Const = dyn_cast<SCEVConstant>(Val: ScevDiff)) {
3915 APInt V = Const->getAPInt();
3916 if (V.isSignedIntN(N: 8 * sizeof(int)))
3917 return static_cast<int>(V.getSExtValue());
3918 }
3919
3920 struct Builder : IRBuilder<> {
3921 Builder(BasicBlock *B) : IRBuilder<>(B->getTerminator()) {}
3922 ~Builder() {
3923 for (Instruction *I : llvm::reverse(C&: ToErase))
3924 I->eraseFromParent();
3925 }
3926 SmallVector<Instruction *, 8> ToErase;
3927 };
3928
3929#define CallBuilder(B, F) \
3930 [&](auto &B_) { \
3931 Value *V = B_.F; \
3932 if (auto *I = dyn_cast<Instruction>(V)) \
3933 B_.ToErase.push_back(I); \
3934 return V; \
3935 }(B)
3936
3937 auto Simplify = [this](Value *V) {
3938 if (Value *S = simplify(V))
3939 return S;
3940 return V;
3941 };
3942
3943 auto StripBitCast = [](Value *V) {
3944 while (auto *C = dyn_cast<BitCastInst>(Val: V))
3945 V = C->getOperand(i_nocapture: 0);
3946 return V;
3947 };
3948
3949 Ptr0 = StripBitCast(Ptr0);
3950 Ptr1 = StripBitCast(Ptr1);
3951 if (!isa<GetElementPtrInst>(Val: Ptr0) || !isa<GetElementPtrInst>(Val: Ptr1))
3952 return std::nullopt;
3953
3954 auto *Gep0 = cast<GetElementPtrInst>(Val: Ptr0);
3955 auto *Gep1 = cast<GetElementPtrInst>(Val: Ptr1);
3956 if (Gep0->getPointerOperand() != Gep1->getPointerOperand())
3957 return std::nullopt;
3958 if (Gep0->getSourceElementType() != Gep1->getSourceElementType())
3959 return std::nullopt;
3960
3961 Builder B(Gep0->getParent());
3962 int Scale = getSizeOf(Ty: Gep0->getSourceElementType(), Kind: Alloc);
3963
3964 // FIXME: for now only check GEPs with a single index.
3965 if (Gep0->getNumOperands() != 2 || Gep1->getNumOperands() != 2)
3966 return std::nullopt;
3967
3968 Value *Idx0 = Gep0->getOperand(i_nocapture: 1);
3969 Value *Idx1 = Gep1->getOperand(i_nocapture: 1);
3970
3971 // First, try to simplify the subtraction directly.
3972 if (auto *Diff = dyn_cast<ConstantInt>(
3973 Val: Simplify(CallBuilder(B, CreateSub(Idx0, Idx1)))))
3974 return Diff->getSExtValue() * Scale;
3975
3976 KnownBits Known0 = getKnownBits(V: Idx0, CtxI: Gep0);
3977 KnownBits Known1 = getKnownBits(V: Idx1, CtxI: Gep1);
3978 APInt Unknown = ~(Known0.Zero | Known0.One) | ~(Known1.Zero | Known1.One);
3979 if (Unknown.isAllOnes())
3980 return std::nullopt;
3981
3982 Value *MaskU = ConstantInt::get(Ty: Idx0->getType(), V: Unknown);
3983 Value *AndU0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskU)));
3984 Value *AndU1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskU)));
3985 Value *SubU = Simplify(CallBuilder(B, CreateSub(AndU0, AndU1)));
3986 int Diff0 = 0;
3987 if (auto *C = dyn_cast<ConstantInt>(Val: SubU)) {
3988 Diff0 = C->getSExtValue();
3989 } else {
3990 return std::nullopt;
3991 }
3992
3993 Value *MaskK = ConstantInt::get(Ty: MaskU->getType(), V: ~Unknown);
3994 Value *AndK0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskK)));
3995 Value *AndK1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskK)));
3996 Value *SubK = Simplify(CallBuilder(B, CreateSub(AndK0, AndK1)));
3997 int Diff1 = 0;
3998 if (auto *C = dyn_cast<ConstantInt>(Val: SubK)) {
3999 Diff1 = C->getSExtValue();
4000 } else {
4001 return std::nullopt;
4002 }
4003
4004 return (Diff0 + Diff1) * Scale;
4005
4006#undef CallBuilder
4007}
4008
4009auto HexagonVectorCombine::getNumSignificantBits(const Value *V,
4010 const Instruction *CtxI) const
4011 -> unsigned {
4012 return ComputeMaxSignificantBits(Op: V, DL, AC: &AC, CxtI: CtxI, DT: &DT);
4013}
4014
4015auto HexagonVectorCombine::getKnownBits(const Value *V,
4016 const Instruction *CtxI) const
4017 -> KnownBits {
4018 return computeKnownBits(V, DL, AC: &AC, CxtI: CtxI, DT: &DT);
4019}
4020
4021auto HexagonVectorCombine::isSafeToClone(const Instruction &In) const -> bool {
4022 if (In.mayHaveSideEffects() || In.isAtomic() || In.isVolatile() ||
4023 In.isFenceLike() || In.mayReadOrWriteMemory()) {
4024 return false;
4025 }
4026 if (isa<CallBase>(Val: In) || isa<AllocaInst>(Val: In))
4027 return false;
4028 return true;
4029}
4030
4031template <typename T>
4032auto HexagonVectorCombine::isSafeToMoveBeforeInBB(const Instruction &In,
4033 BasicBlock::const_iterator To,
4034 const T &IgnoreInsts) const
4035 -> bool {
4036 auto getLocOrNone =
4037 [this](const Instruction &I) -> std::optional<MemoryLocation> {
4038 if (const auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
4039 switch (II->getIntrinsicID()) {
4040 case Intrinsic::masked_load:
4041 return MemoryLocation::getForArgument(Call: II, ArgIdx: 0, TLI);
4042 case Intrinsic::masked_store:
4043 return MemoryLocation::getForArgument(Call: II, ArgIdx: 1, TLI);
4044 }
4045 }
4046 return MemoryLocation::getOrNone(Inst: &I);
4047 };
4048
4049 // The source and the destination must be in the same basic block.
4050 const BasicBlock &Block = *In.getParent();
4051 assert(Block.begin() == To || Block.end() == To || To->getParent() == &Block);
4052 // No PHIs.
4053 if (isa<PHINode>(Val: In) || (To != Block.end() && isa<PHINode>(Val: *To)))
4054 return false;
4055
4056 if (!mayHaveNonDefUseDependency(I: In))
4057 return true;
4058 bool MayWrite = In.mayWriteToMemory();
4059 auto MaybeLoc = getLocOrNone(In);
4060
4061 auto From = In.getIterator();
4062 if (From == To)
4063 return true;
4064 bool MoveUp = (To != Block.end() && To->comesBefore(Other: &In));
4065 auto Range =
4066 MoveUp ? std::make_pair(x&: To, y&: From) : std::make_pair(x: std::next(x: From), y&: To);
4067 for (auto It = Range.first; It != Range.second; ++It) {
4068 const Instruction &I = *It;
4069 if (llvm::is_contained(IgnoreInsts, &I))
4070 continue;
4071 // assume intrinsic can be ignored
4072 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
4073 if (II->getIntrinsicID() == Intrinsic::assume)
4074 continue;
4075 }
4076 // Parts based on isSafeToMoveBefore from CoveMoverUtils.cpp.
4077 if (I.mayThrow())
4078 return false;
4079 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
4080 if (!CB->hasFnAttr(Kind: Attribute::WillReturn))
4081 return false;
4082 if (!CB->hasFnAttr(Kind: Attribute::NoSync))
4083 return false;
4084 }
4085 if (I.mayReadOrWriteMemory()) {
4086 auto MaybeLocI = getLocOrNone(I);
4087 if (MayWrite || I.mayWriteToMemory()) {
4088 if (!MaybeLoc || !MaybeLocI)
4089 return false;
4090 if (!AA.isNoAlias(*MaybeLoc, *MaybeLocI))
4091 return false;
4092 }
4093 }
4094 }
4095 return true;
4096}
4097
4098auto HexagonVectorCombine::isByteVecTy(Type *Ty) const -> bool {
4099 if (auto *VecTy = dyn_cast<VectorType>(Val: Ty))
4100 return VecTy->getElementType() == getByteTy();
4101 return false;
4102}
4103
4104auto HexagonVectorCombine::getElementRange(IRBuilderBase &Builder, Value *Lo,
4105 Value *Hi, int Start,
4106 int Length) const -> Value * {
4107 assert(0 <= Start && size_t(Start + Length) < length(Lo) + length(Hi));
4108 SmallVector<int, 128> SMask(Length);
4109 std::iota(first: SMask.begin(), last: SMask.end(), value: Start);
4110 return Builder.CreateShuffleVector(V1: Lo, V2: Hi, Mask: SMask, Name: "shf");
4111}
4112
4113// Pass management.
4114
4115namespace {
4116class HexagonVectorCombineLegacy : public FunctionPass {
4117public:
4118 static char ID;
4119
4120 HexagonVectorCombineLegacy() : FunctionPass(ID) {}
4121
4122 StringRef getPassName() const override { return "Hexagon Vector Combine"; }
4123
4124 void getAnalysisUsage(AnalysisUsage &AU) const override {
4125 AU.setPreservesCFG();
4126 AU.addRequired<AAResultsWrapperPass>();
4127 AU.addRequired<AssumptionCacheTracker>();
4128 AU.addRequired<DominatorTreeWrapperPass>();
4129 AU.addRequired<ScalarEvolutionWrapperPass>();
4130 AU.addRequired<TargetLibraryInfoWrapperPass>();
4131 AU.addRequired<TargetPassConfig>();
4132 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4133 FunctionPass::getAnalysisUsage(AU);
4134 }
4135
4136 bool runOnFunction(Function &F) override {
4137 if (skipFunction(F))
4138 return false;
4139 AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
4140 AssumptionCache &AC =
4141 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4142 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4143 ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4144 TargetLibraryInfo &TLI =
4145 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
4146 auto &TM = getAnalysis<TargetPassConfig>().getTM<HexagonTargetMachine>();
4147 auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4148 HexagonVectorCombine HVC(F, AA, AC, DT, SE, TLI, TM, ORE);
4149 return HVC.run();
4150 }
4151};
4152} // namespace
4153
4154char HexagonVectorCombineLegacy::ID = 0;
4155
4156INITIALIZE_PASS_BEGIN(HexagonVectorCombineLegacy, DEBUG_TYPE,
4157 "Hexagon Vector Combine", false, false)
4158INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
4159INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
4160INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
4161INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
4162INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
4163INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
4164INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
4165INITIALIZE_PASS_END(HexagonVectorCombineLegacy, DEBUG_TYPE,
4166 "Hexagon Vector Combine", false, false)
4167
4168FunctionPass *llvm::createHexagonVectorCombineLegacyPass() {
4169 return new HexagonVectorCombineLegacy();
4170}
4171