1//===-- HexagonISelDAGToDAGHVX.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "HexagonISelDAGToDAG.h"
10#include "HexagonISelLowering.h"
11#include "llvm/ADT/BitVector.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/CodeGen/SelectionDAGISel.h"
14#include "llvm/IR/Intrinsics.h"
15#include "llvm/IR/IntrinsicsHexagon.h"
16#include "llvm/Support/Debug.h"
17#include "llvm/Support/MathExtras.h"
18
19#include <algorithm>
20#include <deque>
21#include <functional>
22#include <map>
23#include <optional>
24#include <set>
25#include <utility>
26#include <vector>
27
28#define DEBUG_TYPE "hexagon-isel"
29using namespace llvm;
30
31namespace {
32
33// --------------------------------------------------------------------
34// Implementation of permutation networks.
35
36// Implementation of the node routing through butterfly networks:
37// - Forward delta.
38// - Reverse delta.
39// - Benes.
40//
41//
42// Forward delta network consists of log(N) steps, where N is the number
43// of inputs. In each step, an input can stay in place, or it can get
44// routed to another position[1]. The step after that consists of two
45// networks, each half in size in terms of the number of nodes. In those
46// terms, in the given step, an input can go to either the upper or the
47// lower network in the next step.
48//
49// [1] Hexagon's vdelta/vrdelta allow an element to be routed to both
50// positions as long as there is no conflict.
51
52// Here's a delta network for 8 inputs, only the switching routes are
53// shown:
54//
55// Steps:
56// |- 1 ---------------|- 2 -----|- 3 -|
57//
58// Inp[0] *** *** *** *** Out[0]
59// \ / \ / \ /
60// \ / \ / X
61// \ / \ / / \
62// Inp[1] *** \ / *** X *** *** Out[1]
63// \ \ / / \ / \ /
64// \ \ / / X X
65// \ \ / / / \ / \
66// Inp[2] *** \ \ / / *** X *** *** Out[2]
67// \ \ X / / / \ \ /
68// \ \ / \ / / / \ X
69// \ X X / / \ / \
70// Inp[3] *** \ / \ / \ / *** *** *** Out[3]
71// \ X X X /
72// \ / \ / \ / \ /
73// X X X X
74// / \ / \ / \ / \
75// / X X X \
76// Inp[4] *** / \ / \ / \ *** *** *** Out[4]
77// / X X \ \ / \ /
78// / / \ / \ \ \ / X
79// / / X \ \ \ / / \
80// Inp[5] *** / / \ \ *** X *** *** Out[5]
81// / / \ \ \ / \ /
82// / / \ \ X X
83// / / \ \ / \ / \
84// Inp[6] *** / \ *** X *** *** Out[6]
85// / \ / \ \ /
86// / \ / \ X
87// / \ / \ / \
88// Inp[7] *** *** *** *** Out[7]
89//
90//
91// Reverse delta network is same as delta network, with the steps in
92// the opposite order.
93//
94//
95// Benes network is a forward delta network immediately followed by
96// a reverse delta network.
97
98enum class ColorKind { None, Red, Black };
99
100// Graph coloring utility used to partition nodes into two groups:
101// they will correspond to nodes routed to the upper and lower networks.
102struct Coloring {
103 using Node = int;
104 using MapType = std::map<Node, ColorKind>;
105 static constexpr Node Ignore = Node(-1);
106
107 Coloring(ArrayRef<Node> Ord) : Order(Ord) {
108 build();
109 if (!color())
110 Colors.clear();
111 }
112
113 const MapType &colors() const {
114 return Colors;
115 }
116
117 ColorKind other(ColorKind Color) {
118 if (Color == ColorKind::None)
119 return ColorKind::Red;
120 return Color == ColorKind::Red ? ColorKind::Black : ColorKind::Red;
121 }
122
123 LLVM_DUMP_METHOD void dump() const;
124
125private:
126 ArrayRef<Node> Order;
127 MapType Colors;
128 std::set<Node> Needed;
129
130 using NodeSet = std::set<Node>;
131 std::map<Node,NodeSet> Edges;
132
133 Node conj(Node Pos) {
134 Node Num = Order.size();
135 return (Pos < Num/2) ? Pos + Num/2 : Pos - Num/2;
136 }
137
138 ColorKind getColor(Node N) {
139 auto F = Colors.find(x: N);
140 return F != Colors.end() ? F->second : ColorKind::None;
141 }
142
143 std::pair<bool, ColorKind> getUniqueColor(const NodeSet &Nodes);
144
145 void build();
146 bool color();
147};
148} // namespace
149
150std::pair<bool, ColorKind> Coloring::getUniqueColor(const NodeSet &Nodes) {
151 auto Color = ColorKind::None;
152 for (Node N : Nodes) {
153 ColorKind ColorN = getColor(N);
154 if (ColorN == ColorKind::None)
155 continue;
156 if (Color == ColorKind::None)
157 Color = ColorN;
158 else if (Color != ColorKind::None && Color != ColorN)
159 return { false, ColorKind::None };
160 }
161 return { true, Color };
162}
163
164void Coloring::build() {
165 // Add Order[P] and Order[conj(P)] to Edges.
166 for (unsigned P = 0; P != Order.size(); ++P) {
167 Node I = Order[P];
168 if (I != Ignore) {
169 Needed.insert(x: I);
170 Node PC = Order[conj(Pos: P)];
171 if (PC != Ignore && PC != I)
172 Edges[I].insert(x: PC);
173 }
174 }
175 // Add I and conj(I) to Edges.
176 for (unsigned I = 0; I != Order.size(); ++I) {
177 if (!Needed.count(x: I))
178 continue;
179 Node C = conj(Pos: I);
180 // This will create an entry in the edge table, even if I is not
181 // connected to any other node. This is necessary, because it still
182 // needs to be colored.
183 NodeSet &Is = Edges[I];
184 if (Needed.count(x: C))
185 Is.insert(x: C);
186 }
187}
188
189bool Coloring::color() {
190 SetVector<Node> FirstQ;
191 auto Enqueue = [this,&FirstQ] (Node N) {
192 SetVector<Node> Q;
193 Q.insert(X: N);
194 for (unsigned I = 0; I != Q.size(); ++I) {
195 NodeSet &Ns = Edges[Q[I]];
196 Q.insert_range(R&: Ns);
197 }
198 FirstQ.insert_range(R&: Q);
199 };
200 for (Node N : Needed)
201 Enqueue(N);
202
203 for (Node N : FirstQ) {
204 if (Colors.count(x: N))
205 continue;
206 NodeSet &Ns = Edges[N];
207 auto P = getUniqueColor(Nodes: Ns);
208 if (!P.first)
209 return false;
210 Colors[N] = other(Color: P.second);
211 }
212
213 // First, color nodes that don't have any dups.
214 for (auto E : Edges) {
215 Node N = E.first;
216 if (!Needed.count(x: conj(Pos: N)) || Colors.count(x: N))
217 continue;
218 auto P = getUniqueColor(Nodes: E.second);
219 if (!P.first)
220 return false;
221 Colors[N] = other(Color: P.second);
222 }
223
224 // Now, nodes that are still uncolored. Since the graph can be modified
225 // in this step, create a work queue.
226 std::vector<Node> WorkQ;
227 for (auto E : Edges) {
228 Node N = E.first;
229 if (!Colors.count(x: N))
230 WorkQ.push_back(x: N);
231 }
232
233 for (Node N : WorkQ) {
234 NodeSet &Ns = Edges[N];
235 auto P = getUniqueColor(Nodes: Ns);
236 if (P.first) {
237 Colors[N] = other(Color: P.second);
238 continue;
239 }
240
241 // Coloring failed. Split this node.
242 Node C = conj(Pos: N);
243 ColorKind ColorN = other(Color: ColorKind::None);
244 ColorKind ColorC = other(Color: ColorN);
245 NodeSet &Cs = Edges[C];
246 NodeSet CopyNs = Ns;
247 for (Node M : CopyNs) {
248 ColorKind ColorM = getColor(N: M);
249 if (ColorM == ColorC) {
250 // Connect M with C, disconnect M from N.
251 Cs.insert(x: M);
252 Edges[M].insert(x: C);
253 Ns.erase(x: M);
254 Edges[M].erase(x: N);
255 }
256 }
257 Colors[N] = ColorN;
258 Colors[C] = ColorC;
259 }
260
261 // Explicitly assign "None" to all uncolored nodes.
262 for (unsigned I = 0; I != Order.size(); ++I)
263 Colors.try_emplace(k: I, args: ColorKind::None);
264
265 return true;
266}
267
268#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
269void Coloring::dump() const {
270 dbgs() << "{ Order: {";
271 for (Node P : Order) {
272 if (P != Ignore)
273 dbgs() << ' ' << P;
274 else
275 dbgs() << " -";
276 }
277 dbgs() << " }\n";
278 dbgs() << " Needed: {";
279 for (Node N : Needed)
280 dbgs() << ' ' << N;
281 dbgs() << " }\n";
282
283 dbgs() << " Edges: {\n";
284 for (auto E : Edges) {
285 dbgs() << " " << E.first << " -> {";
286 for (auto N : E.second)
287 dbgs() << ' ' << N;
288 dbgs() << " }\n";
289 }
290 dbgs() << " }\n";
291
292 auto ColorKindToName = [](ColorKind C) {
293 switch (C) {
294 case ColorKind::None:
295 return "None";
296 case ColorKind::Red:
297 return "Red";
298 case ColorKind::Black:
299 return "Black";
300 }
301 llvm_unreachable("all ColorKinds should be handled by the switch above");
302 };
303
304 dbgs() << " Colors: {\n";
305 for (auto C : Colors)
306 dbgs() << " " << C.first << " -> " << ColorKindToName(C.second) << "\n";
307 dbgs() << " }\n}\n";
308}
309#endif
310
311namespace {
312// Base class of for reordering networks. They don't strictly need to be
313// permutations, as outputs with repeated occurrences of an input element
314// are allowed.
315struct PermNetwork {
316 using Controls = std::vector<uint8_t>;
317 using ElemType = int;
318 static constexpr ElemType Ignore = ElemType(-1);
319
320 enum : uint8_t {
321 None,
322 Pass,
323 Switch
324 };
325 enum : uint8_t {
326 Forward,
327 Reverse
328 };
329
330 PermNetwork(ArrayRef<ElemType> Ord, unsigned Mult = 1) {
331 Order.assign(first: Ord.data(), last: Ord.data()+Ord.size());
332 Log = 0;
333
334 unsigned S = Order.size();
335 while (S >>= 1)
336 ++Log;
337
338 Table.resize(new_size: Order.size());
339 for (RowType &Row : Table)
340 Row.resize(new_size: Mult*Log, x: None);
341 }
342
343 void getControls(Controls &V, unsigned StartAt, uint8_t Dir) const {
344 unsigned Size = Order.size();
345 V.resize(new_size: Size);
346 for (unsigned I = 0; I != Size; ++I) {
347 unsigned W = 0;
348 for (unsigned L = 0; L != Log; ++L) {
349 unsigned C = ctl(Pos: I, Step: StartAt+L) == Switch;
350 if (Dir == Forward)
351 W |= C << (Log-1-L);
352 else
353 W |= C << L;
354 }
355 assert(isUInt<8>(W));
356 V[I] = uint8_t(W);
357 }
358 }
359
360 uint8_t ctl(ElemType Pos, unsigned Step) const {
361 return Table[Pos][Step];
362 }
363 unsigned size() const {
364 return Order.size();
365 }
366 unsigned steps() const {
367 return Log;
368 }
369
370protected:
371 unsigned Log;
372 std::vector<ElemType> Order;
373 using RowType = std::vector<uint8_t>;
374 std::vector<RowType> Table;
375};
376
377struct ForwardDeltaNetwork : public PermNetwork {
378 ForwardDeltaNetwork(ArrayRef<ElemType> Ord) : PermNetwork(Ord) {}
379
380 bool run(Controls &V) {
381 if (!route(P: Order.data(), T: Table.data(), Size: size(), Step: 0))
382 return false;
383 getControls(V, StartAt: 0, Dir: Forward);
384 return true;
385 }
386
387private:
388 bool route(ElemType *P, RowType *T, unsigned Size, unsigned Step);
389};
390
391struct ReverseDeltaNetwork : public PermNetwork {
392 ReverseDeltaNetwork(ArrayRef<ElemType> Ord) : PermNetwork(Ord) {}
393
394 bool run(Controls &V) {
395 if (!route(P: Order.data(), T: Table.data(), Size: size(), Step: 0))
396 return false;
397 getControls(V, StartAt: 0, Dir: Reverse);
398 return true;
399 }
400
401private:
402 bool route(ElemType *P, RowType *T, unsigned Size, unsigned Step);
403};
404
405struct BenesNetwork : public PermNetwork {
406 BenesNetwork(ArrayRef<ElemType> Ord) : PermNetwork(Ord, 2) {}
407
408 bool run(Controls &F, Controls &R) {
409 if (!route(P: Order.data(), T: Table.data(), Size: size(), Step: 0))
410 return false;
411
412 getControls(V&: F, StartAt: 0, Dir: Forward);
413 getControls(V&: R, StartAt: Log, Dir: Reverse);
414 return true;
415 }
416
417private:
418 bool route(ElemType *P, RowType *T, unsigned Size, unsigned Step);
419};
420} // namespace
421
422bool ForwardDeltaNetwork::route(ElemType *P, RowType *T, unsigned Size,
423 unsigned Step) {
424 bool UseUp = false, UseDown = false;
425 ElemType Num = Size;
426
427 // Cannot use coloring here, because coloring is used to determine
428 // the "big" switch, i.e. the one that changes halves, and in a forward
429 // network, a color can be simultaneously routed to both halves in the
430 // step we're working on.
431 for (ElemType J = 0; J != Num; ++J) {
432 ElemType I = P[J];
433 // I is the position in the input,
434 // J is the position in the output.
435 if (I == Ignore)
436 continue;
437 uint8_t S;
438 if (I < Num/2)
439 S = (J < Num/2) ? Pass : Switch;
440 else
441 S = (J < Num/2) ? Switch : Pass;
442
443 // U is the element in the table that needs to be updated.
444 ElemType U = (S == Pass) ? I : (I < Num/2 ? I+Num/2 : I-Num/2);
445 if (U < Num/2)
446 UseUp = true;
447 else
448 UseDown = true;
449 if (T[U][Step] != S && T[U][Step] != None)
450 return false;
451 T[U][Step] = S;
452 }
453
454 for (ElemType J = 0; J != Num; ++J)
455 if (P[J] != Ignore && P[J] >= Num/2)
456 P[J] -= Num/2;
457
458 if (Step+1 < Log) {
459 if (UseUp && !route(P, T, Size: Size/2, Step: Step+1))
460 return false;
461 if (UseDown && !route(P: P+Size/2, T: T+Size/2, Size: Size/2, Step: Step+1))
462 return false;
463 }
464 return true;
465}
466
467bool ReverseDeltaNetwork::route(ElemType *P, RowType *T, unsigned Size,
468 unsigned Step) {
469 unsigned Pets = Log-1 - Step;
470 bool UseUp = false, UseDown = false;
471 ElemType Num = Size;
472
473 // In this step half-switching occurs, so coloring can be used.
474 Coloring G({P,Size});
475 const Coloring::MapType &M = G.colors();
476 if (M.empty())
477 return false;
478
479 ColorKind ColorUp = ColorKind::None;
480 for (ElemType J = 0; J != Num; ++J) {
481 ElemType I = P[J];
482 // I is the position in the input,
483 // J is the position in the output.
484 if (I == Ignore)
485 continue;
486 ColorKind C = M.at(k: I);
487 if (C == ColorKind::None)
488 continue;
489 // During "Step", inputs cannot switch halves, so if the "up" color
490 // is still unknown, make sure that it is selected in such a way that
491 // "I" will stay in the same half.
492 bool InpUp = I < Num/2;
493 if (ColorUp == ColorKind::None)
494 ColorUp = InpUp ? C : G.other(Color: C);
495 if ((C == ColorUp) != InpUp) {
496 // If I should go to a different half than where is it now, give up.
497 return false;
498 }
499
500 uint8_t S;
501 if (InpUp) {
502 S = (J < Num/2) ? Pass : Switch;
503 UseUp = true;
504 } else {
505 S = (J < Num/2) ? Switch : Pass;
506 UseDown = true;
507 }
508 T[J][Pets] = S;
509 }
510
511 // Reorder the working permutation according to the computed switch table
512 // for the last step (i.e. Pets).
513 for (ElemType J = 0, E = Size / 2; J != E; ++J) {
514 ElemType PJ = P[J]; // Current values of P[J]
515 ElemType PC = P[J+Size/2]; // and P[conj(J)]
516 ElemType QJ = PJ; // New values of P[J]
517 ElemType QC = PC; // and P[conj(J)]
518 if (T[J][Pets] == Switch)
519 QC = PJ;
520 if (T[J+Size/2][Pets] == Switch)
521 QJ = PC;
522 P[J] = QJ;
523 P[J+Size/2] = QC;
524 }
525
526 for (ElemType J = 0; J != Num; ++J)
527 if (P[J] != Ignore && P[J] >= Num/2)
528 P[J] -= Num/2;
529
530 if (Step+1 < Log) {
531 if (UseUp && !route(P, T, Size: Size/2, Step: Step+1))
532 return false;
533 if (UseDown && !route(P: P+Size/2, T: T+Size/2, Size: Size/2, Step: Step+1))
534 return false;
535 }
536 return true;
537}
538
539bool BenesNetwork::route(ElemType *P, RowType *T, unsigned Size,
540 unsigned Step) {
541 Coloring G({P,Size});
542 const Coloring::MapType &M = G.colors();
543 if (M.empty())
544 return false;
545 ElemType Num = Size;
546
547 unsigned Pets = 2*Log-1 - Step;
548 bool UseUp = false, UseDown = false;
549
550 // Both assignments, i.e. Red->Up and Red->Down are valid, but they will
551 // result in different controls. Let's pick the one where the first
552 // control will be "Pass".
553 ColorKind ColorUp = ColorKind::None;
554 for (ElemType J = 0; J != Num; ++J) {
555 ElemType I = P[J];
556 if (I == Ignore)
557 continue;
558 ColorKind C = M.at(k: I);
559 if (C == ColorKind::None)
560 continue;
561 if (ColorUp == ColorKind::None) {
562 ColorUp = (I < Num / 2) ? ColorKind::Red : ColorKind::Black;
563 }
564 unsigned CI = (I < Num/2) ? I+Num/2 : I-Num/2;
565 if (C == ColorUp) {
566 if (I < Num/2)
567 T[I][Step] = Pass;
568 else
569 T[CI][Step] = Switch;
570 T[J][Pets] = (J < Num/2) ? Pass : Switch;
571 UseUp = true;
572 } else { // Down
573 if (I < Num/2)
574 T[CI][Step] = Switch;
575 else
576 T[I][Step] = Pass;
577 T[J][Pets] = (J < Num/2) ? Switch : Pass;
578 UseDown = true;
579 }
580 }
581
582 // Reorder the working permutation according to the computed switch table
583 // for the last step (i.e. Pets).
584 for (ElemType J = 0; J != Num/2; ++J) {
585 ElemType PJ = P[J]; // Current values of P[J]
586 ElemType PC = P[J+Num/2]; // and P[conj(J)]
587 ElemType QJ = PJ; // New values of P[J]
588 ElemType QC = PC; // and P[conj(J)]
589 if (T[J][Pets] == Switch)
590 QC = PJ;
591 if (T[J+Num/2][Pets] == Switch)
592 QJ = PC;
593 P[J] = QJ;
594 P[J+Num/2] = QC;
595 }
596
597 for (ElemType J = 0; J != Num; ++J)
598 if (P[J] != Ignore && P[J] >= Num/2)
599 P[J] -= Num/2;
600
601 if (Step+1 < Log) {
602 if (UseUp && !route(P, T, Size: Size/2, Step: Step+1))
603 return false;
604 if (UseDown && !route(P: P+Size/2, T: T+Size/2, Size: Size/2, Step: Step+1))
605 return false;
606 }
607 return true;
608}
609
610// --------------------------------------------------------------------
611// Support for building selection results (output instructions that are
612// parts of the final selection).
613
614namespace {
615struct OpRef {
616 OpRef(SDValue V) : OpV(V) {}
617 bool isValue() const { return OpV.getNode() != nullptr; }
618 bool isValid() const { return isValue() || !(OpN & Invalid); }
619 bool isUndef() const { return OpN & Undef; }
620 static OpRef res(int N) { return OpRef(Whole | (N & Index)); }
621 static OpRef fail() { return OpRef(Invalid); }
622
623 static OpRef lo(const OpRef &R) {
624 assert(!R.isValue());
625 return OpRef(R.OpN & (Undef | Index | LoHalf));
626 }
627 static OpRef hi(const OpRef &R) {
628 assert(!R.isValue());
629 return OpRef(R.OpN & (Undef | Index | HiHalf));
630 }
631 static OpRef undef(MVT Ty) { return OpRef(Undef | Ty.SimpleTy); }
632
633 // Direct value.
634 SDValue OpV = SDValue();
635
636 // Reference to the operand of the input node:
637 // If the 31st bit is 1, it's undef, otherwise, bits 28..0 are the
638 // operand index:
639 // If bit 30 is set, it's the high half of the operand.
640 // If bit 29 is set, it's the low half of the operand.
641 unsigned OpN = 0;
642
643 enum : unsigned {
644 Invalid = 0x10000000,
645 LoHalf = 0x20000000,
646 HiHalf = 0x40000000,
647 Whole = LoHalf | HiHalf,
648 Undef = 0x80000000,
649 Index = 0x0FFFFFFF, // Mask of the index value.
650 IndexBits = 28,
651 };
652
653 LLVM_DUMP_METHOD
654 void print(raw_ostream &OS, const SelectionDAG &G) const;
655
656private:
657 OpRef(unsigned N) : OpN(N) {}
658};
659
660struct NodeTemplate {
661 NodeTemplate() = default;
662 unsigned Opc = 0;
663 MVT Ty = MVT::Other;
664 std::vector<OpRef> Ops;
665
666 LLVM_DUMP_METHOD void print(raw_ostream &OS, const SelectionDAG &G) const;
667};
668
669struct ResultStack {
670 ResultStack(SDNode *Inp)
671 : InpNode(Inp), InpTy(Inp->getValueType(ResNo: 0).getSimpleVT()) {}
672 SDNode *InpNode;
673 MVT InpTy;
674 unsigned push(const NodeTemplate &Res) {
675 List.push_back(x: Res);
676 return List.size()-1;
677 }
678 unsigned push(unsigned Opc, MVT Ty, std::vector<OpRef> &&Ops) {
679 NodeTemplate Res;
680 Res.Opc = Opc;
681 Res.Ty = Ty;
682 Res.Ops = Ops;
683 return push(Res);
684 }
685 bool empty() const { return List.empty(); }
686 unsigned size() const { return List.size(); }
687 unsigned top() const { return size()-1; }
688 const NodeTemplate &operator[](unsigned I) const { return List[I]; }
689 unsigned reset(unsigned NewTop) {
690 List.resize(new_size: NewTop+1);
691 return NewTop;
692 }
693
694 using BaseType = std::vector<NodeTemplate>;
695 BaseType::iterator begin() { return List.begin(); }
696 BaseType::iterator end() { return List.end(); }
697 BaseType::const_iterator begin() const { return List.begin(); }
698 BaseType::const_iterator end() const { return List.end(); }
699
700 BaseType List;
701
702 LLVM_DUMP_METHOD
703 void print(raw_ostream &OS, const SelectionDAG &G) const;
704};
705} // namespace
706
707#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
708void OpRef::print(raw_ostream &OS, const SelectionDAG &G) const {
709 if (isValue()) {
710 OpV.getNode()->print(OS, &G);
711 return;
712 }
713 if (OpN & Invalid) {
714 OS << "invalid";
715 return;
716 }
717 if (OpN & Undef) {
718 OS << "undef";
719 return;
720 }
721 if ((OpN & Whole) != Whole) {
722 assert((OpN & Whole) == LoHalf || (OpN & Whole) == HiHalf);
723 if (OpN & LoHalf)
724 OS << "lo ";
725 else
726 OS << "hi ";
727 }
728 OS << '#' << SignExtend32(OpN & Index, IndexBits);
729}
730
731void NodeTemplate::print(raw_ostream &OS, const SelectionDAG &G) const {
732 const TargetInstrInfo &TII = *G.getSubtarget().getInstrInfo();
733 OS << format("%8s", EVT(Ty).getEVTString().c_str()) << " "
734 << TII.getName(Opc);
735 bool Comma = false;
736 for (const auto &R : Ops) {
737 if (Comma)
738 OS << ',';
739 Comma = true;
740 OS << ' ';
741 R.print(OS, G);
742 }
743}
744
745void ResultStack::print(raw_ostream &OS, const SelectionDAG &G) const {
746 OS << "Input node:\n";
747#ifndef NDEBUG
748 InpNode->dumpr(&G);
749#endif
750 OS << "Result templates:\n";
751 for (unsigned I = 0, E = List.size(); I != E; ++I) {
752 OS << '[' << I << "] ";
753 List[I].print(OS, G);
754 OS << '\n';
755 }
756}
757#endif
758
759namespace {
760struct ShuffleMask {
761 ShuffleMask(ArrayRef<int> M) : Mask(M) {
762 for (int M : Mask) {
763 if (M == -1)
764 continue;
765 MinSrc = (MinSrc == -1) ? M : std::min(a: MinSrc, b: M);
766 MaxSrc = (MaxSrc == -1) ? M : std::max(a: MaxSrc, b: M);
767 }
768 }
769
770 ArrayRef<int> Mask;
771 int MinSrc = -1, MaxSrc = -1;
772
773 ShuffleMask lo() const {
774 size_t H = Mask.size()/2;
775 return ShuffleMask(Mask.take_front(N: H));
776 }
777 ShuffleMask hi() const {
778 size_t H = Mask.size()/2;
779 return ShuffleMask(Mask.take_back(N: H));
780 }
781
782 void print(raw_ostream &OS) const {
783 OS << "MinSrc:" << MinSrc << ", MaxSrc:" << MaxSrc << " {";
784 for (int M : Mask)
785 OS << ' ' << M;
786 OS << " }";
787 }
788};
789
790[[maybe_unused]]
791raw_ostream &operator<<(raw_ostream &OS, const ShuffleMask &SM) {
792 SM.print(OS);
793 return OS;
794}
795} // namespace
796
797namespace shuffles {
798using MaskT = SmallVector<int, 128>;
799// Vdd = vshuffvdd(Vu, Vv, Rt)
800// Vdd = vdealvdd(Vu, Vv, Rt)
801// Vd = vpack(Vu, Vv, Size, TakeOdd)
802// Vd = vshuff(Vu, Vv, Size, TakeOdd)
803// Vd = vdeal(Vu, Vv, Size, TakeOdd)
804// Vd = vdealb4w(Vu, Vv)
805
806ArrayRef<int> lo(ArrayRef<int> Vuu) { return Vuu.take_front(N: Vuu.size() / 2); }
807ArrayRef<int> hi(ArrayRef<int> Vuu) { return Vuu.take_back(N: Vuu.size() / 2); }
808
809MaskT vshuffvdd(ArrayRef<int> Vu, ArrayRef<int> Vv, unsigned Rt) {
810 int Len = Vu.size();
811 MaskT Vdd(2 * Len);
812 llvm::copy(Range&: Vv, Out: Vdd.begin());
813 llvm::copy(Range&: Vu, Out: Vdd.begin() + Len);
814
815 auto Vd0 = MutableArrayRef<int>(Vdd).take_front(N: Len);
816 auto Vd1 = MutableArrayRef<int>(Vdd).take_back(N: Len);
817
818 for (int Offset = 1; Offset < Len; Offset *= 2) {
819 if ((Rt & Offset) == 0)
820 continue;
821 for (int i = 0; i != Len; ++i) {
822 if ((i & Offset) == 0)
823 std::swap(a&: Vd1[i], b&: Vd0[i + Offset]);
824 }
825 }
826 return Vdd;
827}
828
829MaskT vdealvdd(ArrayRef<int> Vu, ArrayRef<int> Vv, unsigned Rt) {
830 int Len = Vu.size();
831 MaskT Vdd(2 * Len);
832 llvm::copy(Range&: Vv, Out: Vdd.begin());
833 llvm::copy(Range&: Vu, Out: Vdd.begin() + Len);
834
835 auto Vd0 = MutableArrayRef<int>(Vdd).take_front(N: Len);
836 auto Vd1 = MutableArrayRef<int>(Vdd).take_back(N: Len);
837
838 for (int Offset = Len / 2; Offset > 0; Offset /= 2) {
839 if ((Rt & Offset) == 0)
840 continue;
841 for (int i = 0; i != Len; ++i) {
842 if ((i & Offset) == 0)
843 std::swap(a&: Vd1[i], b&: Vd0[i + Offset]);
844 }
845 }
846 return Vdd;
847}
848
849MaskT vpack(ArrayRef<int> Vu, ArrayRef<int> Vv, unsigned Size, bool TakeOdd) {
850 int Len = Vu.size();
851 MaskT Vd(Len);
852 auto Odd = static_cast<int>(TakeOdd);
853 for (int i = 0, e = Len / (2 * Size); i != e; ++i) {
854 for (int b = 0; b != static_cast<int>(Size); ++b) {
855 // clang-format off
856 Vd[i * Size + b] = Vv[(2 * i + Odd) * Size + b];
857 Vd[i * Size + b + Len / 2] = Vu[(2 * i + Odd) * Size + b];
858 // clang-format on
859 }
860 }
861 return Vd;
862}
863
864MaskT vshuff(ArrayRef<int> Vu, ArrayRef<int> Vv, unsigned Size, bool TakeOdd) {
865 int Len = Vu.size();
866 MaskT Vd(Len);
867 auto Odd = static_cast<int>(TakeOdd);
868 for (int i = 0, e = Len / (2 * Size); i != e; ++i) {
869 for (int b = 0; b != static_cast<int>(Size); ++b) {
870 Vd[(2 * i + 0) * Size + b] = Vv[(2 * i + Odd) * Size + b];
871 Vd[(2 * i + 1) * Size + b] = Vu[(2 * i + Odd) * Size + b];
872 }
873 }
874 return Vd;
875}
876
877MaskT vdeal(ArrayRef<int> Vu, ArrayRef<int> Vv, unsigned Size, bool TakeOdd) {
878 int Len = Vu.size();
879 MaskT T = vdealvdd(Vu, Vv, Rt: Len - 2 * Size);
880 return vpack(Vu: hi(Vuu: T), Vv: lo(Vuu: T), Size, TakeOdd);
881}
882
883MaskT vdealb4w(ArrayRef<int> Vu, ArrayRef<int> Vv) {
884 int Len = Vu.size();
885 MaskT Vd(Len);
886 for (int i = 0, e = Len / 4; i != e; ++i) {
887 Vd[0 * (Len / 4) + i] = Vv[4 * i + 0];
888 Vd[1 * (Len / 4) + i] = Vv[4 * i + 2];
889 Vd[2 * (Len / 4) + i] = Vu[4 * i + 0];
890 Vd[3 * (Len / 4) + i] = Vu[4 * i + 2];
891 }
892 return Vd;
893}
894
895template <typename ShuffFunc, typename... OptArgs>
896auto mask(ShuffFunc S, unsigned Length, OptArgs... args) -> MaskT {
897 MaskT Vu(Length), Vv(Length);
898 std::iota(first: Vu.begin(), last: Vu.end(), value: Length); // High
899 std::iota(first: Vv.begin(), last: Vv.end(), value: 0); // Low
900 return S(Vu, Vv, args...);
901}
902
903} // namespace shuffles
904
905// --------------------------------------------------------------------
906// The HvxSelector class.
907
908static const HexagonTargetLowering &getHexagonLowering(SelectionDAG &G) {
909 return static_cast<const HexagonTargetLowering&>(G.getTargetLoweringInfo());
910}
911static const HexagonSubtarget &getHexagonSubtarget(SelectionDAG &G) {
912 return G.getSubtarget<HexagonSubtarget>();
913}
914
915namespace llvm {
916 struct HvxSelector {
917 const HexagonTargetLowering &Lower;
918 HexagonDAGToDAGISel &ISel;
919 SelectionDAG &DAG;
920 const HexagonSubtarget &HST;
921 const unsigned HwLen;
922
923 HvxSelector(HexagonDAGToDAGISel &HS, SelectionDAG &G)
924 : Lower(getHexagonLowering(G)), ISel(HS), DAG(G),
925 HST(getHexagonSubtarget(G)), HwLen(HST.getVectorLength()) {}
926
927 MVT getSingleVT(MVT ElemTy) const {
928 assert(ElemTy != MVT::i1 && "Use getBoolVT for predicates");
929 unsigned NumElems = HwLen / (ElemTy.getSizeInBits() / 8);
930 return MVT::getVectorVT(VT: ElemTy, NumElements: NumElems);
931 }
932
933 MVT getPairVT(MVT ElemTy) const {
934 assert(ElemTy != MVT::i1); // Suspicious: there are no predicate pairs.
935 unsigned NumElems = (2 * HwLen) / (ElemTy.getSizeInBits() / 8);
936 return MVT::getVectorVT(VT: ElemTy, NumElements: NumElems);
937 }
938
939 MVT getBoolVT() const {
940 // Return HwLen x i1.
941 return MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
942 }
943
944 void selectExtractSubvector(SDNode *N);
945 void selectShuffle(SDNode *N);
946 void selectRor(SDNode *N);
947 void selectVAlign(SDNode *N);
948
949 static SmallVector<uint32_t, 8> getPerfectCompletions(ShuffleMask SM,
950 unsigned Width);
951 static SmallVector<uint32_t, 8> completeToPerfect(
952 ArrayRef<uint32_t> Completions, unsigned Width);
953 static std::optional<int> rotationDistance(ShuffleMask SM, unsigned WrapAt);
954
955 private:
956 void select(SDNode *ISelN);
957 void materialize(const ResultStack &Results);
958
959 SDValue getConst32(unsigned Val, const SDLoc &dl);
960 SDValue getSignedConst32(int Val, const SDLoc &dl);
961 SDValue getVectorConstant(ArrayRef<uint8_t> Data, const SDLoc &dl);
962
963 enum : unsigned {
964 None,
965 PackMux,
966 };
967 OpRef concats(OpRef Va, OpRef Vb, ResultStack &Results);
968 OpRef funnels(OpRef Va, OpRef Vb, int Amount, ResultStack &Results);
969
970 OpRef packs(ShuffleMask SM, OpRef Va, OpRef Vb, ResultStack &Results,
971 MutableArrayRef<int> NewMask, unsigned Options = None);
972 OpRef packp(ShuffleMask SM, OpRef Va, OpRef Vb, ResultStack &Results,
973 MutableArrayRef<int> NewMask);
974 OpRef vmuxs(ArrayRef<uint8_t> Bytes, OpRef Va, OpRef Vb,
975 ResultStack &Results);
976 OpRef vmuxp(ArrayRef<uint8_t> Bytes, OpRef Va, OpRef Vb,
977 ResultStack &Results);
978
979 OpRef shuffs1(ShuffleMask SM, OpRef Va, ResultStack &Results);
980 OpRef shuffs2(ShuffleMask SM, OpRef Va, OpRef Vb, ResultStack &Results);
981 OpRef shuffp1(ShuffleMask SM, OpRef Va, ResultStack &Results);
982 OpRef shuffp2(ShuffleMask SM, OpRef Va, OpRef Vb, ResultStack &Results);
983
984 OpRef butterfly(ShuffleMask SM, OpRef Va, ResultStack &Results);
985 OpRef contracting(ShuffleMask SM, OpRef Va, OpRef Vb, ResultStack &Results);
986 OpRef expanding(ShuffleMask SM, OpRef Va, ResultStack &Results);
987 OpRef perfect(ShuffleMask SM, OpRef Va, ResultStack &Results);
988
989 bool selectVectorConstants(SDNode *N);
990 bool scalarizeShuffle(ArrayRef<int> Mask, const SDLoc &dl, MVT ResTy,
991 SDValue Va, SDValue Vb, SDNode *N);
992 };
993} // namespace llvm
994
995static void splitMask(ArrayRef<int> Mask, MutableArrayRef<int> MaskL,
996 MutableArrayRef<int> MaskR) {
997 unsigned VecLen = Mask.size();
998 assert(MaskL.size() == VecLen && MaskR.size() == VecLen);
999 for (unsigned I = 0; I != VecLen; ++I) {
1000 int M = Mask[I];
1001 if (M < 0) {
1002 MaskL[I] = MaskR[I] = -1;
1003 } else if (unsigned(M) < VecLen) {
1004 MaskL[I] = M;
1005 MaskR[I] = -1;
1006 } else {
1007 MaskL[I] = -1;
1008 MaskR[I] = M-VecLen;
1009 }
1010 }
1011}
1012
1013static std::pair<int,unsigned> findStrip(ArrayRef<int> A, int Inc,
1014 unsigned MaxLen) {
1015 assert(A.size() > 0 && A.size() >= MaxLen);
1016 int F = A[0];
1017 int E = F;
1018 for (unsigned I = 1; I != MaxLen; ++I) {
1019 if (A[I] - E != Inc)
1020 return { F, I };
1021 E = A[I];
1022 }
1023 return { F, MaxLen };
1024}
1025
1026static bool isUndef(ArrayRef<int> Mask) {
1027 for (int Idx : Mask)
1028 if (Idx != -1)
1029 return false;
1030 return true;
1031}
1032
1033static bool isIdentity(ArrayRef<int> Mask) {
1034 for (int I = 0, E = Mask.size(); I != E; ++I) {
1035 int M = Mask[I];
1036 if (M >= 0 && M != I)
1037 return false;
1038 }
1039 return true;
1040}
1041
1042static bool isLowHalfOnly(ArrayRef<int> Mask) {
1043 int L = Mask.size();
1044 assert(L % 2 == 0);
1045 // Check if the second half of the mask is all-undef.
1046 return llvm::all_of(Range: Mask.drop_front(N: L / 2), P: [](int M) { return M < 0; });
1047}
1048
1049static SmallVector<unsigned, 4> getInputSegmentList(ShuffleMask SM,
1050 unsigned SegLen) {
1051 assert(isPowerOf2_32(SegLen));
1052 SmallVector<unsigned, 4> SegList;
1053 if (SM.MaxSrc == -1)
1054 return SegList;
1055
1056 unsigned Shift = Log2_32(Value: SegLen);
1057 BitVector Segs(alignTo(Value: SM.MaxSrc + 1, Align: SegLen) >> Shift);
1058
1059 for (int M : SM.Mask) {
1060 if (M >= 0)
1061 Segs.set(M >> Shift);
1062 }
1063
1064 llvm::append_range(C&: SegList, R: Segs.set_bits());
1065 return SegList;
1066}
1067
1068static SmallVector<unsigned, 4> getOutputSegmentMap(ShuffleMask SM,
1069 unsigned SegLen) {
1070 // Calculate the layout of the output segments in terms of the input
1071 // segments.
1072 // For example [1,3,1,0] means that the output consists of 4 output
1073 // segments, where the first output segment has only elements of the
1074 // input segment at index 1. The next output segment only has elements
1075 // of the input segment 3, etc.
1076 // If an output segment only has undef elements, the value will be ~0u.
1077 // If an output segment has elements from more than one input segment,
1078 // the corresponding value will be ~1u.
1079 unsigned MaskLen = SM.Mask.size();
1080 assert(MaskLen % SegLen == 0);
1081 SmallVector<unsigned, 4> Map(MaskLen / SegLen);
1082
1083 for (int S = 0, E = Map.size(); S != E; ++S) {
1084 unsigned Idx = ~0u;
1085 for (int I = 0; I != static_cast<int>(SegLen); ++I) {
1086 int M = SM.Mask[S*SegLen + I];
1087 if (M < 0)
1088 continue;
1089 unsigned G = M / SegLen; // Input segment of this element.
1090 if (Idx == ~0u) {
1091 Idx = G;
1092 } else if (Idx != G) {
1093 Idx = ~1u;
1094 break;
1095 }
1096 }
1097 Map[S] = Idx;
1098 }
1099
1100 return Map;
1101}
1102
1103static void packSegmentMask(ArrayRef<int> Mask, ArrayRef<unsigned> OutSegMap,
1104 unsigned SegLen, MutableArrayRef<int> PackedMask) {
1105 SmallVector<unsigned, 4> InvMap;
1106 for (int I = OutSegMap.size() - 1; I >= 0; --I) {
1107 unsigned S = OutSegMap[I];
1108 assert(S != ~0u && "Unexpected undef");
1109 assert(S != ~1u && "Unexpected multi");
1110 if (InvMap.size() <= S)
1111 InvMap.resize(N: S+1);
1112 InvMap[S] = I;
1113 }
1114
1115 unsigned Shift = Log2_32(Value: SegLen);
1116 for (int I = 0, E = Mask.size(); I != E; ++I) {
1117 int M = Mask[I];
1118 if (M >= 0) {
1119 int OutIdx = InvMap[M >> Shift];
1120 M = (M & (SegLen-1)) + SegLen*OutIdx;
1121 }
1122 PackedMask[I] = M;
1123 }
1124}
1125
1126bool HvxSelector::selectVectorConstants(SDNode *N) {
1127 // Constant vectors are generated as loads from constant pools or as
1128 // splats of a constant value. Since they are generated during the
1129 // selection process, the main selection algorithm is not aware of them.
1130 // Select them directly here.
1131 SmallVector<SDNode*,4> Nodes;
1132 SetVector<SDNode*> WorkQ;
1133
1134 // The DAG can change (due to CSE) during selection, so cache all the
1135 // unselected nodes first to avoid traversing a mutating DAG.
1136 WorkQ.insert(X: N);
1137 for (unsigned i = 0; i != WorkQ.size(); ++i) {
1138 SDNode *W = WorkQ[i];
1139 if (!W->isMachineOpcode() && W->getOpcode() == HexagonISD::ISEL)
1140 Nodes.push_back(Elt: W);
1141 for (unsigned j = 0, f = W->getNumOperands(); j != f; ++j)
1142 WorkQ.insert(X: W->getOperand(Num: j).getNode());
1143 }
1144
1145 for (SDNode *L : Nodes)
1146 select(ISelN: L);
1147
1148 return !Nodes.empty();
1149}
1150
1151void HvxSelector::materialize(const ResultStack &Results) {
1152 DEBUG_WITH_TYPE("isel", {
1153 dbgs() << "Materializing\n";
1154 Results.print(dbgs(), DAG);
1155 });
1156 if (Results.empty())
1157 return;
1158 const SDLoc &dl(Results.InpNode);
1159 std::vector<SDValue> Output;
1160
1161 for (unsigned I = 0, E = Results.size(); I != E; ++I) {
1162 const NodeTemplate &Node = Results[I];
1163 std::vector<SDValue> Ops;
1164 for (const OpRef &R : Node.Ops) {
1165 assert(R.isValid());
1166 if (R.isValue()) {
1167 Ops.push_back(x: R.OpV);
1168 continue;
1169 }
1170 if (R.OpN & OpRef::Undef) {
1171 MVT::SimpleValueType SVT = MVT::SimpleValueType(R.OpN & OpRef::Index);
1172 Ops.push_back(x: ISel.selectUndef(dl, ResTy: MVT(SVT)));
1173 continue;
1174 }
1175 // R is an index of a result.
1176 unsigned Part = R.OpN & OpRef::Whole;
1177 int Idx = SignExtend32(X: R.OpN & OpRef::Index, B: OpRef::IndexBits);
1178 if (Idx < 0)
1179 Idx += I;
1180 assert(Idx >= 0 && unsigned(Idx) < Output.size());
1181 SDValue Op = Output[Idx];
1182 MVT OpTy = Op.getValueType().getSimpleVT();
1183 if (Part != OpRef::Whole) {
1184 assert(Part == OpRef::LoHalf || Part == OpRef::HiHalf);
1185 MVT HalfTy = MVT::getVectorVT(VT: OpTy.getVectorElementType(),
1186 NumElements: OpTy.getVectorNumElements()/2);
1187 unsigned Sub = (Part == OpRef::LoHalf) ? Hexagon::vsub_lo
1188 : Hexagon::vsub_hi;
1189 Op = DAG.getTargetExtractSubreg(SRIdx: Sub, DL: dl, VT: HalfTy, Operand: Op);
1190 }
1191 Ops.push_back(x: Op);
1192 } // for (Node : Results)
1193
1194 assert(Node.Ty != MVT::Other);
1195 SDNode *ResN = (Node.Opc == TargetOpcode::COPY)
1196 ? Ops.front().getNode()
1197 : DAG.getMachineNode(Opcode: Node.Opc, dl, VT: Node.Ty, Ops);
1198 Output.push_back(x: SDValue(ResN, 0));
1199 }
1200
1201 SDNode *OutN = Output.back().getNode();
1202 SDNode *InpN = Results.InpNode;
1203 DEBUG_WITH_TYPE("isel", {
1204 dbgs() << "Generated node:\n";
1205 OutN->dumpr(&DAG);
1206 });
1207
1208 ISel.ReplaceNode(F: InpN, T: OutN);
1209 selectVectorConstants(N: OutN);
1210 DAG.RemoveDeadNodes();
1211}
1212
1213OpRef HvxSelector::concats(OpRef Lo, OpRef Hi, ResultStack &Results) {
1214 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1215 const SDLoc &dl(Results.InpNode);
1216 Results.push(Opc: TargetOpcode::REG_SEQUENCE, Ty: getPairVT(ElemTy: MVT::i8), Ops: {
1217 getConst32(Val: Hexagon::HvxWRRegClassID, dl),
1218 Lo, getConst32(Val: Hexagon::vsub_lo, dl),
1219 Hi, getConst32(Val: Hexagon::vsub_hi, dl),
1220 });
1221 return OpRef::res(N: Results.top());
1222}
1223
1224OpRef HvxSelector::funnels(OpRef Va, OpRef Vb, int Amount,
1225 ResultStack &Results) {
1226 // Do a funnel shift towards the low end (shift right) by Amount bytes.
1227 // If Amount < 0, treat it as shift left, i.e. do a shift right by
1228 // Amount + HwLen.
1229 auto VecLen = static_cast<int>(HwLen);
1230
1231 if (Amount == 0)
1232 return Va;
1233 if (Amount == VecLen)
1234 return Vb;
1235
1236 MVT Ty = getSingleVT(ElemTy: MVT::i8);
1237 const SDLoc &dl(Results.InpNode);
1238
1239 if (Amount < 0)
1240 Amount += VecLen;
1241 if (Amount > VecLen) {
1242 Amount -= VecLen;
1243 std::swap(a&: Va, b&: Vb);
1244 }
1245
1246 if (isUInt<3>(x: Amount)) {
1247 SDValue A = getConst32(Val: Amount, dl);
1248 Results.push(Opc: Hexagon::V6_valignbi, Ty, Ops: {Vb, Va, A});
1249 } else if (isUInt<3>(x: VecLen - Amount)) {
1250 SDValue A = getConst32(Val: VecLen - Amount, dl);
1251 Results.push(Opc: Hexagon::V6_vlalignbi, Ty, Ops: {Vb, Va, A});
1252 } else {
1253 SDValue A = getConst32(Val: Amount, dl);
1254 Results.push(Opc: Hexagon::A2_tfrsi, Ty, Ops: {A});
1255 Results.push(Opc: Hexagon::V6_valignb, Ty, Ops: {Vb, Va, OpRef::res(N: -1)});
1256 }
1257 return OpRef::res(N: Results.top());
1258}
1259
1260// Va, Vb are single vectors. If SM only uses two vector halves from Va/Vb,
1261// pack these halves into a single vector, and remap SM into NewMask to use
1262// the new vector instead.
1263OpRef HvxSelector::packs(ShuffleMask SM, OpRef Va, OpRef Vb,
1264 ResultStack &Results, MutableArrayRef<int> NewMask,
1265 unsigned Options) {
1266 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1267 if (!Va.isValid() || !Vb.isValid())
1268 return OpRef::fail();
1269
1270 if (Vb.isUndef()) {
1271 llvm::copy(Range&: SM.Mask, Out: NewMask.begin());
1272 return Va;
1273 }
1274 if (Va.isUndef()) {
1275 llvm::copy(Range&: SM.Mask, Out: NewMask.begin());
1276 ShuffleVectorSDNode::commuteMask(Mask: NewMask);
1277 return Vb;
1278 }
1279
1280 MVT Ty = getSingleVT(ElemTy: MVT::i8);
1281 MVT PairTy = getPairVT(ElemTy: MVT::i8);
1282 OpRef Inp[2] = {Va, Vb};
1283 unsigned VecLen = SM.Mask.size();
1284
1285 auto valign = [this](OpRef Lo, OpRef Hi, unsigned Amt, MVT Ty,
1286 ResultStack &Results) {
1287 if (Amt == 0)
1288 return Lo;
1289 const SDLoc &dl(Results.InpNode);
1290 if (isUInt<3>(x: Amt) || isUInt<3>(x: HwLen - Amt)) {
1291 bool IsRight = isUInt<3>(x: Amt); // Right align.
1292 SDValue S = getConst32(Val: IsRight ? Amt : HwLen - Amt, dl);
1293 unsigned Opc = IsRight ? Hexagon::V6_valignbi : Hexagon::V6_vlalignbi;
1294 Results.push(Opc, Ty, Ops: {Hi, Lo, S});
1295 return OpRef::res(N: Results.top());
1296 }
1297 Results.push(Opc: Hexagon::A2_tfrsi, Ty: MVT::i32, Ops: {getConst32(Val: Amt, dl)});
1298 OpRef A = OpRef::res(N: Results.top());
1299 Results.push(Opc: Hexagon::V6_valignb, Ty, Ops: {Hi, Lo, A});
1300 return OpRef::res(N: Results.top());
1301 };
1302
1303 // Segment is a vector half.
1304 unsigned SegLen = HwLen / 2;
1305
1306 // Check if we can shuffle vector halves around to get the used elements
1307 // into a single vector.
1308 shuffles::MaskT MaskH(SM.Mask);
1309 SmallVector<unsigned, 4> SegList = getInputSegmentList(SM: SM.Mask, SegLen);
1310 unsigned SegCount = SegList.size();
1311 SmallVector<unsigned, 4> SegMap = getOutputSegmentMap(SM: SM.Mask, SegLen);
1312
1313 if (SegList.empty())
1314 return OpRef::undef(Ty);
1315
1316 // NOTE:
1317 // In the following part of the function, where the segments are rearranged,
1318 // the shuffle mask SM can be of any length that is a multiple of a vector
1319 // (i.e. a multiple of 2*SegLen), and non-zero.
1320 // The output segment map is computed, and it may have any even number of
1321 // entries, but the rearrangement of input segments will be done based only
1322 // on the first two (non-undef) entries in the segment map.
1323 // For example, if the output map is 3, 1, 1, 3 (it can have at most two
1324 // distinct entries!), the segments 1 and 3 of Va/Vb will be packaged into
1325 // a single vector V = 3:1. The output mask will then be updated to use
1326 // seg(0,V), seg(1,V), seg(1,V), seg(0,V).
1327 //
1328 // Picking the segments based on the output map is an optimization. For
1329 // correctness it is only necessary that Seg0 and Seg1 are the two input
1330 // segments that are used in the output.
1331
1332 unsigned Seg0 = ~0u, Seg1 = ~0u;
1333 for (unsigned X : SegMap) {
1334 if (X == ~0u)
1335 continue;
1336 if (Seg0 == ~0u)
1337 Seg0 = X;
1338 else if (Seg1 != ~0u)
1339 break;
1340 if (X == ~1u || X != Seg0)
1341 Seg1 = X;
1342 }
1343
1344 if (SegCount == 1) {
1345 unsigned SrcOp = SegList[0] / 2;
1346 for (int I = 0; I != static_cast<int>(VecLen); ++I) {
1347 int M = SM.Mask[I];
1348 if (M >= 0) {
1349 M -= SrcOp * HwLen;
1350 assert(M >= 0);
1351 }
1352 NewMask[I] = M;
1353 }
1354 return Inp[SrcOp];
1355 }
1356
1357 if (SegCount == 2) {
1358 // Seg0 should not be undef here: this would imply a SegList
1359 // with <= 1 elements, which was checked earlier.
1360 assert(Seg0 != ~0u);
1361
1362 // If Seg0 or Seg1 are "multi-defined", pick them from the input
1363 // segment list instead.
1364 if (Seg0 == ~1u || Seg1 == ~1u) {
1365 if (Seg0 == Seg1) {
1366 Seg0 = SegList[0];
1367 Seg1 = SegList[1];
1368 } else if (Seg0 == ~1u) {
1369 Seg0 = SegList[0] != Seg1 ? SegList[0] : SegList[1];
1370 } else {
1371 assert(Seg1 == ~1u);
1372 Seg1 = SegList[0] != Seg0 ? SegList[0] : SegList[1];
1373 }
1374 }
1375 assert(Seg0 != ~1u && Seg1 != ~1u);
1376
1377 assert(Seg0 != Seg1 && "Expecting different segments");
1378 const SDLoc &dl(Results.InpNode);
1379 Results.push(Opc: Hexagon::A2_tfrsi, Ty: MVT::i32, Ops: {getConst32(Val: SegLen, dl)});
1380 OpRef HL = OpRef::res(N: Results.top());
1381
1382 // Va = AB, Vb = CD
1383
1384 if (Seg0 / 2 == Seg1 / 2) {
1385 // Same input vector.
1386 Va = Inp[Seg0 / 2];
1387 if (Seg0 > Seg1) {
1388 // Swap halves.
1389 Results.push(Opc: Hexagon::V6_vror, Ty, Ops: {Inp[Seg0 / 2], HL});
1390 Va = OpRef::res(N: Results.top());
1391 }
1392 packSegmentMask(Mask: SM.Mask, OutSegMap: {Seg0, Seg1}, SegLen, PackedMask: MaskH);
1393 } else if (Seg0 % 2 == Seg1 % 2) {
1394 // Picking AC, BD, CA, or DB.
1395 // vshuff(CD,AB,HL) -> BD:AC
1396 // vshuff(AB,CD,HL) -> DB:CA
1397 auto Vs = (Seg0 == 0 || Seg0 == 1) ? std::make_pair(x&: Vb, y&: Va) // AC or BD
1398 : std::make_pair(x&: Va, y&: Vb); // CA or DB
1399 Results.push(Opc: Hexagon::V6_vshuffvdd, Ty: PairTy, Ops: {Vs.first, Vs.second, HL});
1400 OpRef P = OpRef::res(N: Results.top());
1401 Va = (Seg0 == 0 || Seg0 == 2) ? OpRef::lo(R: P) : OpRef::hi(R: P);
1402 packSegmentMask(Mask: SM.Mask, OutSegMap: {Seg0, Seg1}, SegLen, PackedMask: MaskH);
1403 } else {
1404 // Picking AD, BC, CB, or DA.
1405 if ((Seg0 == 0 && Seg1 == 3) || (Seg0 == 2 && Seg1 == 1)) {
1406 // AD or BC: this can be done using vmux.
1407 // Q = V6_pred_scalar2 SegLen
1408 // V = V6_vmux Q, (Va, Vb) or (Vb, Va)
1409 Results.push(Opc: Hexagon::V6_pred_scalar2, Ty: getBoolVT(), Ops: {HL});
1410 OpRef Qt = OpRef::res(N: Results.top());
1411 auto Vs = (Seg0 == 0) ? std::make_pair(x&: Va, y&: Vb) // AD
1412 : std::make_pair(x&: Vb, y&: Va); // CB
1413 Results.push(Opc: Hexagon::V6_vmux, Ty, Ops: {Qt, Vs.first, Vs.second});
1414 Va = OpRef::res(N: Results.top());
1415 packSegmentMask(Mask: SM.Mask, OutSegMap: {Seg0, Seg1}, SegLen, PackedMask: MaskH);
1416 } else {
1417 // BC or DA: this could be done via valign by SegLen.
1418 // Do nothing here, because valign (if possible) will be generated
1419 // later on (make sure the Seg0 values are as expected).
1420 assert(Seg0 == 1 || Seg0 == 3);
1421 }
1422 }
1423 }
1424
1425 // Check if the arguments can be packed by valign(Va,Vb) or valign(Vb,Va).
1426 // FIXME: maybe remove this?
1427 ShuffleMask SMH(MaskH);
1428 assert(SMH.Mask.size() == VecLen);
1429 shuffles::MaskT MaskA(SMH.Mask);
1430
1431 if (SMH.MaxSrc - SMH.MinSrc >= static_cast<int>(HwLen)) {
1432 // valign(Lo=Va,Hi=Vb) won't work. Try swapping Va/Vb.
1433 shuffles::MaskT Swapped(SMH.Mask);
1434 ShuffleVectorSDNode::commuteMask(Mask: Swapped);
1435 ShuffleMask SW(Swapped);
1436 if (SW.MaxSrc - SW.MinSrc < static_cast<int>(HwLen)) {
1437 MaskA.assign(in_start: SW.Mask.begin(), in_end: SW.Mask.end());
1438 std::swap(a&: Va, b&: Vb);
1439 }
1440 }
1441 ShuffleMask SMA(MaskA);
1442 assert(SMA.Mask.size() == VecLen);
1443
1444 if (SMA.MaxSrc - SMA.MinSrc < static_cast<int>(HwLen)) {
1445 int ShiftR = SMA.MinSrc;
1446 if (ShiftR >= static_cast<int>(HwLen)) {
1447 Va = Vb;
1448 Vb = OpRef::undef(Ty);
1449 ShiftR -= HwLen;
1450 }
1451 OpRef RetVal = valign(Va, Vb, ShiftR, Ty, Results);
1452
1453 for (int I = 0; I != static_cast<int>(VecLen); ++I) {
1454 int M = SMA.Mask[I];
1455 if (M != -1)
1456 M -= SMA.MinSrc;
1457 NewMask[I] = M;
1458 }
1459 return RetVal;
1460 }
1461
1462 // By here, packing by segment (half-vector) shuffling, and vector alignment
1463 // failed. Try vmux.
1464 // Note: since this is using the original mask, Va and Vb must not have been
1465 // modified.
1466
1467 if (Options & PackMux) {
1468 // If elements picked from Va and Vb have all different (source) indexes
1469 // (relative to the start of the argument), do a mux, and update the mask.
1470 BitVector Picked(HwLen);
1471 SmallVector<uint8_t,128> MuxBytes(HwLen);
1472 bool CanMux = true;
1473 for (int I = 0; I != static_cast<int>(VecLen); ++I) {
1474 int M = SM.Mask[I];
1475 if (M == -1)
1476 continue;
1477 if (M >= static_cast<int>(HwLen))
1478 M -= HwLen;
1479 else
1480 MuxBytes[M] = 0xFF;
1481 if (Picked[M]) {
1482 CanMux = false;
1483 break;
1484 }
1485 NewMask[I] = M;
1486 }
1487 if (CanMux)
1488 return vmuxs(Bytes: MuxBytes, Va, Vb, Results);
1489 }
1490 return OpRef::fail();
1491}
1492
1493// Va, Vb are vector pairs. If SM only uses two single vectors from Va/Vb,
1494// pack these vectors into a pair, and remap SM into NewMask to use the
1495// new pair instead.
1496OpRef HvxSelector::packp(ShuffleMask SM, OpRef Va, OpRef Vb,
1497 ResultStack &Results, MutableArrayRef<int> NewMask) {
1498 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1499 SmallVector<unsigned, 4> SegList = getInputSegmentList(SM: SM.Mask, SegLen: HwLen);
1500 if (SegList.empty())
1501 return OpRef::undef(Ty: getPairVT(ElemTy: MVT::i8));
1502
1503 // If more than two halves are used, bail.
1504 // TODO: be more aggressive here?
1505 unsigned SegCount = SegList.size();
1506 if (SegCount > 2)
1507 return OpRef::fail();
1508
1509 MVT HalfTy = getSingleVT(ElemTy: MVT::i8);
1510
1511 OpRef Inp[2] = { Va, Vb };
1512 OpRef Out[2] = { OpRef::undef(Ty: HalfTy), OpRef::undef(Ty: HalfTy) };
1513
1514 // Really make sure we have at most 2 vectors used in the mask.
1515 assert(SegCount <= 2);
1516
1517 for (int I = 0, E = SegList.size(); I != E; ++I) {
1518 unsigned S = SegList[I];
1519 OpRef Op = Inp[S / 2];
1520 Out[I] = (S & 1) ? OpRef::hi(R: Op) : OpRef::lo(R: Op);
1521 }
1522
1523 // NOTE: Using SegList as the packing map here (not SegMap). This works,
1524 // because we're not concerned here about the order of the segments (i.e.
1525 // single vectors) in the output pair. Changing the order of vectors is
1526 // free (as opposed to changing the order of vector halves as in packs),
1527 // and so there is no extra cost added in case the order needs to be
1528 // changed later.
1529 packSegmentMask(Mask: SM.Mask, OutSegMap: SegList, SegLen: HwLen, PackedMask: NewMask);
1530 return concats(Lo: Out[0], Hi: Out[1], Results);
1531}
1532
1533OpRef HvxSelector::vmuxs(ArrayRef<uint8_t> Bytes, OpRef Va, OpRef Vb,
1534 ResultStack &Results) {
1535 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1536 MVT ByteTy = getSingleVT(ElemTy: MVT::i8);
1537 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1538 const SDLoc &dl(Results.InpNode);
1539 SDValue B = getVectorConstant(Data: Bytes, dl);
1540 Results.push(Opc: Hexagon::V6_vd0, Ty: ByteTy, Ops: {});
1541 Results.push(Opc: Hexagon::V6_veqb, Ty: BoolTy, Ops: {OpRef(B), OpRef::res(N: -1)});
1542 Results.push(Opc: Hexagon::V6_vmux, Ty: ByteTy, Ops: {OpRef::res(N: -1), Vb, Va});
1543 return OpRef::res(N: Results.top());
1544}
1545
1546OpRef HvxSelector::vmuxp(ArrayRef<uint8_t> Bytes, OpRef Va, OpRef Vb,
1547 ResultStack &Results) {
1548 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1549 size_t S = Bytes.size() / 2;
1550 OpRef L = vmuxs(Bytes: Bytes.take_front(N: S), Va: OpRef::lo(R: Va), Vb: OpRef::lo(R: Vb), Results);
1551 OpRef H = vmuxs(Bytes: Bytes.drop_front(N: S), Va: OpRef::hi(R: Va), Vb: OpRef::hi(R: Vb), Results);
1552 return concats(Lo: L, Hi: H, Results);
1553}
1554
1555OpRef HvxSelector::shuffs1(ShuffleMask SM, OpRef Va, ResultStack &Results) {
1556 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1557 unsigned VecLen = SM.Mask.size();
1558 assert(HwLen == VecLen);
1559 (void)VecLen;
1560 assert(all_of(SM.Mask, [this](int M) { return M == -1 || M < int(HwLen); }));
1561
1562 if (isIdentity(Mask: SM.Mask))
1563 return Va;
1564 if (isUndef(Mask: SM.Mask))
1565 return OpRef::undef(Ty: getSingleVT(ElemTy: MVT::i8));
1566
1567 // First, check for rotations.
1568 if (auto Dist = rotationDistance(SM, WrapAt: VecLen)) {
1569 OpRef Rotate = funnels(Va, Vb: Va, Amount: *Dist, Results);
1570 if (Rotate.isValid())
1571 return Rotate;
1572 }
1573 unsigned HalfLen = HwLen / 2;
1574 assert(isPowerOf2_32(HalfLen));
1575
1576 // Handle special case where the output is the same half of the input
1577 // repeated twice, i.e. if Va = AB, then handle the output of AA or BB.
1578 std::pair<int, unsigned> Strip1 = findStrip(A: SM.Mask, Inc: 1, MaxLen: HalfLen);
1579 if ((Strip1.first & ~HalfLen) == 0 && Strip1.second == HalfLen) {
1580 std::pair<int, unsigned> Strip2 =
1581 findStrip(A: SM.Mask.drop_front(N: HalfLen), Inc: 1, MaxLen: HalfLen);
1582 if (Strip1 == Strip2) {
1583 const SDLoc &dl(Results.InpNode);
1584 Results.push(Opc: Hexagon::A2_tfrsi, Ty: MVT::i32, Ops: {getConst32(Val: HalfLen, dl)});
1585 Results.push(Opc: Hexagon::V6_vshuffvdd, Ty: getPairVT(ElemTy: MVT::i8),
1586 Ops: {Va, Va, OpRef::res(N: Results.top())});
1587 OpRef S = OpRef::res(N: Results.top());
1588 return (Strip1.first == 0) ? OpRef::lo(R: S) : OpRef::hi(R: S);
1589 }
1590 }
1591
1592 OpRef P = perfect(SM, Va, Results);
1593 if (P.isValid())
1594 return P;
1595 return butterfly(SM, Va, Results);
1596}
1597
1598OpRef HvxSelector::shuffs2(ShuffleMask SM, OpRef Va, OpRef Vb,
1599 ResultStack &Results) {
1600 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1601 if (isUndef(Mask: SM.Mask))
1602 return OpRef::undef(Ty: getSingleVT(ElemTy: MVT::i8));
1603
1604 OpRef C = contracting(SM, Va, Vb, Results);
1605 if (C.isValid())
1606 return C;
1607
1608 int VecLen = SM.Mask.size();
1609 shuffles::MaskT PackedMask(VecLen);
1610 OpRef P = packs(SM, Va, Vb, Results, NewMask: PackedMask);
1611 if (P.isValid())
1612 return shuffs1(SM: ShuffleMask(PackedMask), Va: P, Results);
1613
1614 // TODO: Before we split the mask, try perfect shuffle on concatenated
1615 // operands.
1616
1617 shuffles::MaskT MaskL(VecLen), MaskR(VecLen);
1618 splitMask(Mask: SM.Mask, MaskL, MaskR);
1619
1620 OpRef L = shuffs1(SM: ShuffleMask(MaskL), Va, Results);
1621 OpRef R = shuffs1(SM: ShuffleMask(MaskR), Va: Vb, Results);
1622 if (!L.isValid() || !R.isValid())
1623 return OpRef::fail();
1624
1625 SmallVector<uint8_t, 128> Bytes(VecLen);
1626 for (int I = 0; I != VecLen; ++I) {
1627 if (MaskL[I] != -1)
1628 Bytes[I] = 0xFF;
1629 }
1630 return vmuxs(Bytes, Va: L, Vb: R, Results);
1631}
1632
1633OpRef HvxSelector::shuffp1(ShuffleMask SM, OpRef Va, ResultStack &Results) {
1634 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1635 int VecLen = SM.Mask.size();
1636
1637 if (isIdentity(Mask: SM.Mask))
1638 return Va;
1639 if (isUndef(Mask: SM.Mask))
1640 return OpRef::undef(Ty: getPairVT(ElemTy: MVT::i8));
1641
1642 shuffles::MaskT PackedMask(VecLen);
1643 OpRef P = packs(SM, Va: OpRef::lo(R: Va), Vb: OpRef::hi(R: Va), Results, NewMask: PackedMask);
1644 if (P.isValid()) {
1645 ShuffleMask PM(PackedMask);
1646 OpRef E = expanding(SM: PM, Va: P, Results);
1647 if (E.isValid())
1648 return E;
1649
1650 OpRef L = shuffs1(SM: PM.lo(), Va: P, Results);
1651 OpRef H = shuffs1(SM: PM.hi(), Va: P, Results);
1652 if (L.isValid() && H.isValid())
1653 return concats(Lo: L, Hi: H, Results);
1654 }
1655
1656 if (!isLowHalfOnly(Mask: SM.Mask)) {
1657 // Doing a perfect shuffle on a low-half mask (i.e. where the upper half
1658 // is all-undef) may produce a perfect shuffle that generates legitimate
1659 // upper half. This isn't wrong, but if the perfect shuffle was possible,
1660 // then there is a good chance that a shorter (contracting) code may be
1661 // used as well (e.g. V6_vshuffeb, etc).
1662 OpRef R = perfect(SM, Va, Results);
1663 if (R.isValid())
1664 return R;
1665 // TODO commute the mask and try the opposite order of the halves.
1666 }
1667
1668 OpRef L = shuffs2(SM: SM.lo(), Va: OpRef::lo(R: Va), Vb: OpRef::hi(R: Va), Results);
1669 OpRef H = shuffs2(SM: SM.hi(), Va: OpRef::lo(R: Va), Vb: OpRef::hi(R: Va), Results);
1670 if (L.isValid() && H.isValid())
1671 return concats(Lo: L, Hi: H, Results);
1672
1673 return OpRef::fail();
1674}
1675
1676OpRef HvxSelector::shuffp2(ShuffleMask SM, OpRef Va, OpRef Vb,
1677 ResultStack &Results) {
1678 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1679 if (isUndef(Mask: SM.Mask))
1680 return OpRef::undef(Ty: getPairVT(ElemTy: MVT::i8));
1681
1682 int VecLen = SM.Mask.size();
1683 SmallVector<int,256> PackedMask(VecLen);
1684 OpRef P = packp(SM, Va, Vb, Results, NewMask: PackedMask);
1685 if (P.isValid())
1686 return shuffp1(SM: ShuffleMask(PackedMask), Va: P, Results);
1687
1688 SmallVector<int,256> MaskL(VecLen), MaskR(VecLen);
1689 splitMask(Mask: SM.Mask, MaskL, MaskR);
1690
1691 OpRef L = shuffp1(SM: ShuffleMask(MaskL), Va, Results);
1692 OpRef R = shuffp1(SM: ShuffleMask(MaskR), Va: Vb, Results);
1693 if (!L.isValid() || !R.isValid())
1694 return OpRef::fail();
1695
1696 // Mux the results.
1697 SmallVector<uint8_t,256> Bytes(VecLen);
1698 for (int I = 0; I != VecLen; ++I) {
1699 if (MaskL[I] != -1)
1700 Bytes[I] = 0xFF;
1701 }
1702 return vmuxp(Bytes, Va: L, Vb: R, Results);
1703}
1704
1705namespace {
1706 struct Deleter : public SelectionDAG::DAGNodeDeletedListener {
1707 template <typename T>
1708 Deleter(SelectionDAG &D, T &C)
1709 : SelectionDAG::DAGNodeDeletedListener(D, [&C] (SDNode *N, SDNode *E) {
1710 C.erase(N);
1711 }) {}
1712 };
1713
1714 template <typename T>
1715 struct NullifyingVector : public T {
1716 DenseMap<SDNode*, SDNode**> Refs;
1717 NullifyingVector(T &&V) : T(V) {
1718 for (unsigned i = 0, e = T::size(); i != e; ++i) {
1719 SDNode *&N = T::operator[](i);
1720 Refs[N] = &N;
1721 }
1722 }
1723 void erase(SDNode *N) {
1724 auto F = Refs.find(Val: N);
1725 if (F != Refs.end())
1726 *F->second = nullptr;
1727 }
1728 };
1729}
1730
1731void HvxSelector::select(SDNode *ISelN) {
1732 // What's important here is to select the right set of nodes. The main
1733 // selection algorithm loops over nodes in a topological order, i.e. users
1734 // are visited before their operands.
1735 //
1736 // It is an error to have an unselected node with a selected operand, and
1737 // there is an assertion in the main selector code to enforce that.
1738 //
1739 // Such a situation could occur if we selected a node, which is both a
1740 // subnode of ISelN, and a subnode of an unrelated (and yet unselected)
1741 // node in the DAG.
1742 assert(ISelN->getOpcode() == HexagonISD::ISEL);
1743 SDNode *N0 = ISelN->getOperand(Num: 0).getNode();
1744
1745 // There could have been nodes created (i.e. inserted into the DAG)
1746 // that are now dead. Remove them, in case they use any of the nodes
1747 // to select (and make them look shared).
1748 DAG.RemoveDeadNodes();
1749
1750 SetVector<SDNode *> SubNodes;
1751
1752 if (!N0->isMachineOpcode()) {
1753 // Don't want to select N0 if it's shared with another node, except if
1754 // it's shared with other ISELs.
1755 auto IsISelN = [](SDNode *T) { return T->getOpcode() == HexagonISD::ISEL; };
1756 if (llvm::all_of(Range: N0->users(), P: IsISelN))
1757 SubNodes.insert(X: N0);
1758 }
1759 if (SubNodes.empty()) {
1760 ISel.ReplaceNode(F: ISelN, T: N0);
1761 return;
1762 }
1763
1764 // Need to manually select the nodes that are dominated by the ISEL. Other
1765 // nodes are reachable from the rest of the DAG, and so will be selected
1766 // by the DAG selection routine.
1767 SetVector<SDNode*> Dom, NonDom;
1768 Dom.insert(X: N0);
1769
1770 auto IsDomRec = [&Dom, &NonDom] (SDNode *T, auto Rec) -> bool {
1771 if (Dom.count(key: T))
1772 return true;
1773 if (T->use_empty() || NonDom.count(key: T))
1774 return false;
1775 for (SDNode *U : T->users()) {
1776 // If T is reachable from a known non-dominated node, then T itself
1777 // is non-dominated.
1778 if (!Rec(U, Rec)) {
1779 NonDom.insert(X: T);
1780 return false;
1781 }
1782 }
1783 Dom.insert(X: T);
1784 return true;
1785 };
1786
1787 auto IsDom = [&IsDomRec] (SDNode *T) { return IsDomRec(T, IsDomRec); };
1788
1789 // Add the rest of nodes dominated by ISEL to SubNodes.
1790 for (unsigned I = 0; I != SubNodes.size(); ++I) {
1791 for (SDValue Op : SubNodes[I]->ops()) {
1792 SDNode *O = Op.getNode();
1793 if (IsDom(O))
1794 SubNodes.insert(X: O);
1795 }
1796 }
1797
1798 // Do a topological sort of nodes from Dom.
1799 SetVector<SDNode*> TmpQ;
1800
1801 std::map<SDNode *, unsigned> OpCount;
1802 for (SDNode *T : Dom) {
1803 unsigned NumDomOps = llvm::count_if(Range: T->ops(), P: [&Dom](const SDUse &U) {
1804 return Dom.count(key: U.getNode());
1805 });
1806
1807 OpCount.insert(x: {T, NumDomOps});
1808 if (NumDomOps == 0)
1809 TmpQ.insert(X: T);
1810 }
1811
1812 for (unsigned I = 0; I != TmpQ.size(); ++I) {
1813 SDNode *S = TmpQ[I];
1814 for (SDNode *U : S->users()) {
1815 if (U == ISelN)
1816 continue;
1817 auto F = OpCount.find(x: U);
1818 assert(F != OpCount.end());
1819 if (F->second > 0 && !--F->second)
1820 TmpQ.insert(X: F->first);
1821 }
1822 }
1823
1824 // Remove the marker.
1825 ISel.ReplaceNode(F: ISelN, T: N0);
1826
1827 assert(SubNodes.size() == TmpQ.size());
1828 NullifyingVector<decltype(TmpQ)::vector_type> Queue(TmpQ.takeVector());
1829
1830 Deleter DUQ(DAG, Queue);
1831 for (SDNode *S : reverse(C&: Queue)) {
1832 if (S == nullptr)
1833 continue;
1834 DEBUG_WITH_TYPE("isel", {dbgs() << "HVX selecting: "; S->dump(&DAG);});
1835 ISel.Select(N: S);
1836 }
1837}
1838
1839bool HvxSelector::scalarizeShuffle(ArrayRef<int> Mask, const SDLoc &dl,
1840 MVT ResTy, SDValue Va, SDValue Vb,
1841 SDNode *N) {
1842 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
1843 MVT ElemTy = ResTy.getVectorElementType();
1844 assert(ElemTy == MVT::i8);
1845 unsigned VecLen = Mask.size();
1846 bool HavePairs = (2*HwLen == VecLen);
1847 MVT SingleTy = getSingleVT(ElemTy: MVT::i8);
1848
1849 // The prior attempts to handle this shuffle may have left a bunch of
1850 // dead nodes in the DAG (such as constants). These nodes will be added
1851 // at the end of DAG's node list, which at that point had already been
1852 // sorted topologically. In the main selection loop, the node list is
1853 // traversed backwards from the root node, which means that any new
1854 // nodes (from the end of the list) will not be visited.
1855 // Scalarization will replace the shuffle node with the scalarized
1856 // expression, and if that expression reused any if the leftoever (dead)
1857 // nodes, these nodes would not be selected (since the "local" selection
1858 // only visits nodes that are not in AllNodes).
1859 // To avoid this issue, remove all dead nodes from the DAG now.
1860// DAG.RemoveDeadNodes();
1861
1862 SmallVector<SDValue,128> Ops;
1863 LLVMContext &Ctx = *DAG.getContext();
1864 MVT LegalTy = Lower.getTypeToTransformTo(Context&: Ctx, VT: ElemTy).getSimpleVT();
1865 for (int I : Mask) {
1866 if (I < 0) {
1867 Ops.push_back(Elt: ISel.selectUndef(dl, ResTy: LegalTy));
1868 continue;
1869 }
1870 SDValue Vec;
1871 unsigned M = I;
1872 if (M < VecLen) {
1873 Vec = Va;
1874 } else {
1875 Vec = Vb;
1876 M -= VecLen;
1877 }
1878 if (HavePairs) {
1879 if (M < HwLen) {
1880 Vec = DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_lo, DL: dl, VT: SingleTy, Operand: Vec);
1881 } else {
1882 Vec = DAG.getTargetExtractSubreg(SRIdx: Hexagon::vsub_hi, DL: dl, VT: SingleTy, Operand: Vec);
1883 M -= HwLen;
1884 }
1885 }
1886 SDValue Idx = DAG.getConstant(Val: M, DL: dl, VT: MVT::i32);
1887 SDValue Ex = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: LegalTy, Ops: {Vec, Idx});
1888 SDValue L = Lower.LowerOperation(Op: Ex, DAG);
1889 assert(L.getNode());
1890 Ops.push_back(Elt: L);
1891 }
1892
1893 SDValue LV;
1894 if (2*HwLen == VecLen) {
1895 SDValue B0 = DAG.getBuildVector(VT: SingleTy, DL: dl, Ops: {Ops.data(), HwLen});
1896 SDValue L0 = Lower.LowerOperation(Op: B0, DAG);
1897 SDValue B1 = DAG.getBuildVector(VT: SingleTy, DL: dl, Ops: {Ops.data()+HwLen, HwLen});
1898 SDValue L1 = Lower.LowerOperation(Op: B1, DAG);
1899 // XXX CONCAT_VECTORS is legal for HVX vectors. Legalizing (lowering)
1900 // functions may expect to be called only for illegal operations, so
1901 // make sure that they are not called for legal ones. Develop a better
1902 // mechanism for dealing with this.
1903 LV = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: {L0, L1});
1904 } else {
1905 SDValue BV = DAG.getBuildVector(VT: ResTy, DL: dl, Ops);
1906 LV = Lower.LowerOperation(Op: BV, DAG);
1907 }
1908
1909 assert(!N->use_empty());
1910 SDValue IS = DAG.getNode(Opcode: HexagonISD::ISEL, DL: dl, VT: ResTy, Operand: LV);
1911 ISel.ReplaceNode(F: N, T: IS.getNode());
1912 select(ISelN: IS.getNode());
1913 DAG.RemoveDeadNodes();
1914 return true;
1915}
1916
1917SmallVector<uint32_t, 8> HvxSelector::getPerfectCompletions(ShuffleMask SM,
1918 unsigned Width) {
1919 auto possibilities = [](ArrayRef<uint8_t> Bs, unsigned Width) -> uint32_t {
1920 unsigned Impossible = ~(1u << Width) + 1;
1921 for (unsigned I = 0, E = Bs.size(); I != E; ++I) {
1922 uint8_t B = Bs[I];
1923 if (B == 0xff)
1924 continue;
1925 if (~Impossible == 0)
1926 break;
1927 for (unsigned Log = 0; Log != Width; ++Log) {
1928 if (Impossible & (1u << Log))
1929 continue;
1930 unsigned Expected = (I >> Log) % 2;
1931 if (B != Expected)
1932 Impossible |= (1u << Log);
1933 }
1934 }
1935 return ~Impossible;
1936 };
1937
1938 SmallVector<uint32_t, 8> Worklist(Width);
1939
1940 for (unsigned BitIdx = 0; BitIdx != Width; ++BitIdx) {
1941 SmallVector<uint8_t> BitValues(SM.Mask.size());
1942 for (int i = 0, e = SM.Mask.size(); i != e; ++i) {
1943 int M = SM.Mask[i];
1944 if (M < 0)
1945 BitValues[i] = 0xff;
1946 else
1947 BitValues[i] = (M & (1u << BitIdx)) != 0;
1948 }
1949 Worklist[BitIdx] = possibilities(BitValues, Width);
1950 }
1951
1952 // If there is a word P in Worklist that matches multiple possibilities,
1953 // then if any other word Q matches any of the possibilities matched by P,
1954 // then Q matches all the possibilities matched by P. In fact, P == Q.
1955 // In other words, for each words P, Q, the sets of possibilities matched
1956 // by P and Q are either equal or disjoint (no partial overlap).
1957 //
1958 // Illustration: For 4-bit values there are 4 complete sequences:
1959 // a: 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1
1960 // b: 0 0 1 1 0 0 1 1 0 0 1 1 0 0 1 1
1961 // c: 0 0 0 0 1 1 1 1 0 0 0 0 1 1 1 1
1962 // d: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1
1963 //
1964 // Words containing unknown bits that match two of the complete
1965 // sequences:
1966 // ab: 0 u u 1 0 u u 1 0 u u 1 0 u u 1
1967 // ac: 0 u 0 u u 1 u 1 0 u 0 u u 1 u 1
1968 // ad: 0 u 0 u 0 u 0 u u 1 u 1 u 1 u 1
1969 // bc: 0 0 u u u u 1 1 0 0 u u u u 1 1
1970 // bd: 0 0 u u 0 0 u u u u 1 1 u u 1 1
1971 // cd: 0 0 0 0 u u u u u u u u 1 1 1 1
1972 //
1973 // Proof of the claim above:
1974 // Let P be a word that matches s0 and s1. For that to happen, all known
1975 // bits in P must match s0 and s1 exactly.
1976 // Assume there is Q that matches s1. Note that since P and Q came from
1977 // the same shuffle mask, the positions of unknown bits in P and Q match
1978 // exactly, which makes the indices of known bits be exactly the same
1979 // between P and Q. Since P matches s0 and s1, the known bits of P much
1980 // match both s0 and s1. Also, since Q matches s1, the known bits in Q
1981 // are exactly the same as in s1, which means that they are exactly the
1982 // same as in P. This implies that P == Q.
1983
1984 // There can be a situation where there are more entries with the same
1985 // bits set than there are set bits (e.g. value 9 occurring more than 2
1986 // times). In such cases it will be impossible to complete this to a
1987 // perfect shuffle.
1988 SmallVector<uint32_t, 8> Sorted(Worklist);
1989 llvm::sort(C&: Sorted);
1990
1991 for (unsigned I = 0, E = Sorted.size(); I != E;) {
1992 unsigned P = Sorted[I], Count = 1;
1993 while (++I != E && P == Sorted[I])
1994 ++Count;
1995 if ((unsigned)llvm::popcount(Value: P) < Count) {
1996 // Reset all occurrences of P, if there are more occurrences of P
1997 // than there are bits in P.
1998 llvm::replace(Range&: Worklist, OldValue: P, NewValue: 0U);
1999 }
2000 }
2001
2002 return Worklist;
2003}
2004
2005SmallVector<uint32_t, 8>
2006HvxSelector::completeToPerfect(ArrayRef<uint32_t> Completions, unsigned Width) {
2007 // Pick a completion if there are multiple possibilities. For now just
2008 // select any valid completion.
2009 SmallVector<uint32_t, 8> Comps(Completions);
2010
2011 for (unsigned I = 0; I != Width; ++I) {
2012 uint32_t P = Comps[I];
2013 assert(P != 0);
2014 if (isPowerOf2_32(Value: P))
2015 continue;
2016 // T = least significant bit of P.
2017 uint32_t T = P ^ ((P - 1) & P);
2018 // Clear T in all remaining words matching P.
2019 for (unsigned J = I + 1; J != Width; ++J) {
2020 if (Comps[J] == P)
2021 Comps[J] ^= T;
2022 }
2023 Comps[I] = T;
2024 }
2025
2026#ifndef NDEBUG
2027 // Check that we have generated a valid completion.
2028 uint32_t OrAll = 0;
2029 for (uint32_t C : Comps) {
2030 assert(isPowerOf2_32(C));
2031 OrAll |= C;
2032 }
2033 assert(OrAll == (1u << Width) -1);
2034#endif
2035
2036 return Comps;
2037}
2038
2039std::optional<int> HvxSelector::rotationDistance(ShuffleMask SM,
2040 unsigned WrapAt) {
2041 std::optional<int> Dist;
2042 for (int I = 0, E = SM.Mask.size(); I != E; ++I) {
2043 int M = SM.Mask[I];
2044 if (M < 0)
2045 continue;
2046 if (Dist) {
2047 if ((I + *Dist) % static_cast<int>(WrapAt) != M)
2048 return std::nullopt;
2049 } else {
2050 // Integer a%b operator assumes rounding towards zero by /, so it
2051 // "misbehaves" when a crosses 0 (the remainder also changes sign).
2052 // Add WrapAt in an attempt to keep I+Dist non-negative.
2053 Dist = M - I;
2054 if (Dist < 0)
2055 Dist = *Dist + WrapAt;
2056 }
2057 }
2058 return Dist;
2059}
2060
2061OpRef HvxSelector::contracting(ShuffleMask SM, OpRef Va, OpRef Vb,
2062 ResultStack &Results) {
2063 DEBUG_WITH_TYPE("isel", { dbgs() << __func__ << '\n'; });
2064 if (!Va.isValid() || !Vb.isValid())
2065 return OpRef::fail();
2066
2067 // Contracting shuffles, i.e. instructions that always discard some bytes
2068 // from the operand vectors.
2069 //
2070 // Funnel shifts
2071 // V6_vshuff{e,o}b
2072 // V6_vshuf{e,o}h
2073 // V6_vdealb4w
2074 // V6_vpack{e,o}{b,h}
2075
2076 int VecLen = SM.Mask.size();
2077
2078 // First, check for funnel shifts.
2079 if (auto Dist = rotationDistance(SM, WrapAt: 2 * VecLen)) {
2080 OpRef Funnel = funnels(Va, Vb, Amount: *Dist, Results);
2081 if (Funnel.isValid())
2082 return Funnel;
2083 }
2084
2085 MVT SingleTy = getSingleVT(ElemTy: MVT::i8);
2086 MVT PairTy = getPairVT(ElemTy: MVT::i8);
2087
2088 auto same = [](ArrayRef<int> Mask1, ArrayRef<int> Mask2) -> bool {
2089 return Mask1 == Mask2;
2090 };
2091
2092 using PackConfig = std::pair<unsigned, bool>;
2093 PackConfig Packs[] = {
2094 {1, false}, // byte, even
2095 {1, true}, // byte, odd
2096 {2, false}, // half, even
2097 {2, true}, // half, odd
2098 };
2099
2100 { // Check vpack
2101 unsigned Opcodes[] = {
2102 Hexagon::V6_vpackeb,
2103 Hexagon::V6_vpackob,
2104 Hexagon::V6_vpackeh,
2105 Hexagon::V6_vpackoh,
2106 };
2107 for (int i = 0, e = std::size(Opcodes); i != e; ++i) {
2108 auto [Size, Odd] = Packs[i];
2109 if (same(SM.Mask, shuffles::mask(S: shuffles::vpack, Length: HwLen, args: Size, args: Odd))) {
2110 Results.push(Opc: Opcodes[i], Ty: SingleTy, Ops: {Vb, Va});
2111 return OpRef::res(N: Results.top());
2112 }
2113 }
2114 }
2115
2116 { // Check vshuff
2117 unsigned Opcodes[] = {
2118 Hexagon::V6_vshuffeb,
2119 Hexagon::V6_vshuffob,
2120 Hexagon::V6_vshufeh,
2121 Hexagon::V6_vshufoh,
2122 };
2123 for (int i = 0, e = std::size(Opcodes); i != e; ++i) {
2124 auto [Size, Odd] = Packs[i];
2125 if (same(SM.Mask, shuffles::mask(S: shuffles::vshuff, Length: HwLen, args: Size, args: Odd))) {
2126 Results.push(Opc: Opcodes[i], Ty: SingleTy, Ops: {Vb, Va});
2127 return OpRef::res(N: Results.top());
2128 }
2129 }
2130 }
2131
2132 { // Check vdeal
2133 // There is no "V6_vdealeb", etc, but the supposed behavior of vdealeb
2134 // is equivalent to "(V6_vpackeb (V6_vdealvdd Vu, Vv, -2))". Other such
2135 // variants of "deal" can be done similarly.
2136 unsigned Opcodes[] = {
2137 Hexagon::V6_vpackeb,
2138 Hexagon::V6_vpackob,
2139 Hexagon::V6_vpackeh,
2140 Hexagon::V6_vpackoh,
2141 };
2142 const SDLoc &dl(Results.InpNode);
2143
2144 for (int i = 0, e = std::size(Opcodes); i != e; ++i) {
2145 auto [Size, Odd] = Packs[i];
2146 if (same(SM.Mask, shuffles::mask(S: shuffles::vdeal, Length: HwLen, args: Size, args: Odd))) {
2147 Results.push(Opc: Hexagon::A2_tfrsi, Ty: MVT::i32,
2148 Ops: {getSignedConst32(Val: -2 * Size, dl)});
2149 Results.push(Opc: Hexagon::V6_vdealvdd, Ty: PairTy, Ops: {Vb, Va, OpRef::res(N: -1)});
2150 auto vdeal = OpRef::res(N: Results.top());
2151 Results.push(Opc: Opcodes[i], Ty: SingleTy,
2152 Ops: {OpRef::hi(R: vdeal), OpRef::lo(R: vdeal)});
2153 return OpRef::res(N: Results.top());
2154 }
2155 }
2156 }
2157
2158 if (same(SM.Mask, shuffles::mask(S: shuffles::vdealb4w, Length: HwLen))) {
2159 Results.push(Opc: Hexagon::V6_vdealb4w, Ty: SingleTy, Ops: {Vb, Va});
2160 return OpRef::res(N: Results.top());
2161 }
2162
2163 return OpRef::fail();
2164}
2165
2166OpRef HvxSelector::expanding(ShuffleMask SM, OpRef Va, ResultStack &Results) {
2167 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
2168 // Expanding shuffles (using all elements and inserting into larger vector):
2169 //
2170 // V6_vunpacku{b,h} [*]
2171 //
2172 // [*] Only if the upper elements (filled with 0s) are "don't care" in Mask.
2173 //
2174 // Note: V6_vunpacko{b,h} are or-ing the high byte/half in the result, so
2175 // they are not shuffles.
2176 //
2177 // The argument is a single vector.
2178
2179 int VecLen = SM.Mask.size();
2180 assert(2*HwLen == unsigned(VecLen) && "Expecting vector-pair type");
2181
2182 std::pair<int,unsigned> Strip = findStrip(A: SM.Mask, Inc: 1, MaxLen: VecLen);
2183
2184 // The patterns for the unpacks, in terms of the starting offsets of the
2185 // consecutive strips (L = length of the strip, N = VecLen):
2186 //
2187 // vunpacku: 0, -1, L, -1, 2L, -1 ...
2188
2189 if (Strip.first != 0)
2190 return OpRef::fail();
2191
2192 // The vunpackus only handle byte and half-word.
2193 if (Strip.second != 1 && Strip.second != 2)
2194 return OpRef::fail();
2195
2196 int N = VecLen;
2197 int L = Strip.second;
2198
2199 // First, check the non-ignored strips.
2200 for (int I = 2*L; I < N; I += 2*L) {
2201 auto S = findStrip(A: SM.Mask.drop_front(N: I), Inc: 1, MaxLen: N-I);
2202 if (S.second != unsigned(L))
2203 return OpRef::fail();
2204 if (2*S.first != I)
2205 return OpRef::fail();
2206 }
2207 // Check the -1s.
2208 for (int I = L; I < N; I += 2*L) {
2209 auto S = findStrip(A: SM.Mask.drop_front(N: I), Inc: 0, MaxLen: N-I);
2210 if (S.first != -1 || S.second != unsigned(L))
2211 return OpRef::fail();
2212 }
2213
2214 unsigned Opc = Strip.second == 1 ? Hexagon::V6_vunpackub
2215 : Hexagon::V6_vunpackuh;
2216 Results.push(Opc, Ty: getPairVT(ElemTy: MVT::i8), Ops: {Va});
2217 return OpRef::res(N: Results.top());
2218}
2219
2220OpRef HvxSelector::perfect(ShuffleMask SM, OpRef Va, ResultStack &Results) {
2221 DEBUG_WITH_TYPE("isel", { dbgs() << __func__ << '\n'; });
2222 // V6_vdeal{b,h}
2223 // V6_vshuff{b,h}
2224
2225 // V6_vshufoe{b,h} those are equivalent to vshuffvdd(..,{1,2})
2226 // V6_vshuffvdd (V6_vshuff)
2227 // V6_dealvdd (V6_vdeal)
2228
2229 int VecLen = SM.Mask.size();
2230 assert(isPowerOf2_32(VecLen) && Log2_32(VecLen) <= 8);
2231 unsigned LogLen = Log2_32(Value: VecLen);
2232 unsigned HwLog = Log2_32(Value: HwLen);
2233 // The result length must be the same as the length of a single vector,
2234 // or a vector pair.
2235 assert(LogLen == HwLog || LogLen == HwLog + 1);
2236 bool HavePairs = LogLen == HwLog + 1;
2237
2238 SmallVector<unsigned, 8> Perm(LogLen);
2239
2240 // Check if this could be a perfect shuffle, or a combination of perfect
2241 // shuffles.
2242 //
2243 // Consider this permutation (using hex digits to make the ASCII diagrams
2244 // easier to read):
2245 // { 0, 8, 1, 9, 2, A, 3, B, 4, C, 5, D, 6, E, 7, F }.
2246 // This is a "deal" operation: divide the input into two halves, and
2247 // create the output by picking elements by alternating between these two
2248 // halves:
2249 // 0 1 2 3 4 5 6 7 --> 0 8 1 9 2 A 3 B 4 C 5 D 6 E 7 F [*]
2250 // 8 9 A B C D E F
2251 //
2252 // Aside from a few special explicit cases (V6_vdealb, etc.), HVX provides
2253 // a somwehat different mechanism that could be used to perform shuffle/
2254 // deal operations: a 2x2 transpose.
2255 // Consider the halves of inputs again, they can be interpreted as a 2x8
2256 // matrix. A 2x8 matrix can be looked at four 2x2 matrices concatenated
2257 // together. Now, when considering 2 elements at a time, it will be a 2x4
2258 // matrix (with elements 01, 23, 45, etc.), or two 2x2 matrices:
2259 // 01 23 45 67
2260 // 89 AB CD EF
2261 // With groups of 4, this will become a single 2x2 matrix, and so on.
2262 //
2263 // The 2x2 transpose instruction works by transposing each of the 2x2
2264 // matrices (or "sub-matrices"), given a specific group size. For example,
2265 // if the group size is 1 (i.e. each element is its own group), there
2266 // will be four transposes of the four 2x2 matrices that form the 2x8.
2267 // For example, with the inputs as above, the result will be:
2268 // 0 8 2 A 4 C 6 E
2269 // 1 9 3 B 5 D 7 F
2270 // Now, this result can be transposed again, but with the group size of 2:
2271 // 08 19 4C 5D
2272 // 2A 3B 6E 7F
2273 // If we then transpose that result, but with the group size of 4, we get:
2274 // 0819 2A3B
2275 // 4C5D 6E7F
2276 // If we concatenate these two rows, it will be
2277 // 0 8 1 9 2 A 3 B 4 C 5 D 6 E 7 F
2278 // which is the same as the "deal" [*] above.
2279 //
2280 // In general, a "deal" of individual elements is a series of 2x2 transposes,
2281 // with changing group size. HVX has two instructions:
2282 // Vdd = V6_vdealvdd Vu, Vv, Rt
2283 // Vdd = V6_shufvdd Vu, Vv, Rt
2284 // that perform exactly that. The register Rt controls which transposes are
2285 // going to happen: a bit at position n (counting from 0) indicates that a
2286 // transpose with a group size of 2^n will take place. If multiple bits are
2287 // set, multiple transposes will happen: vdealvdd will perform them starting
2288 // with the largest group size, vshuffvdd will do them in the reverse order.
2289 //
2290 // The main observation is that each 2x2 transpose corresponds to swapping
2291 // columns of bits in the binary representation of the values.
2292 //
2293 // The numbers {3,2,1,0} and the log2 of the number of contiguous 1 bits
2294 // in a given column. The * denote the columns that will be swapped.
2295 // The transpose with the group size 2^n corresponds to swapping columns
2296 // 3 (the highest log) and log2(n):
2297 //
2298 // 3 2 1 0 0 2 1 3 0 2 3 1
2299 // * * * * * *
2300 // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
2301 // 1 0 0 0 1 8 1 0 0 0 8 1 0 0 0 8 1 0 0 0
2302 // 2 0 0 1 0 2 0 0 1 0 1 0 0 0 1 1 0 0 0 1
2303 // 3 0 0 1 1 A 1 0 1 0 9 1 0 0 1 9 1 0 0 1
2304 // 4 0 1 0 0 4 0 1 0 0 4 0 1 0 0 2 0 0 1 0
2305 // 5 0 1 0 1 C 1 1 0 0 C 1 1 0 0 A 1 0 1 0
2306 // 6 0 1 1 0 6 0 1 1 0 5 0 1 0 1 3 0 0 1 1
2307 // 7 0 1 1 1 E 1 1 1 0 D 1 1 0 1 B 1 0 1 1
2308 // 8 1 0 0 0 1 0 0 0 1 2 0 0 1 0 4 0 1 0 0
2309 // 9 1 0 0 1 9 1 0 0 1 A 1 0 1 0 C 1 1 0 0
2310 // A 1 0 1 0 3 0 0 1 1 3 0 0 1 1 5 0 1 0 1
2311 // B 1 0 1 1 B 1 0 1 1 B 1 0 1 1 D 1 1 0 1
2312 // C 1 1 0 0 5 0 1 0 1 6 0 1 1 0 6 0 1 1 0
2313 // D 1 1 0 1 D 1 1 0 1 E 1 1 1 0 E 1 1 1 0
2314 // E 1 1 1 0 7 0 1 1 1 7 0 1 1 1 7 0 1 1 1
2315 // F 1 1 1 1 F 1 1 1 1 F 1 1 1 1 F 1 1 1 1
2316
2317 // There is one special case that is not a perfect shuffle, but can be
2318 // turned into one easily: when the shuffle operates on a vector pair,
2319 // but the two vectors in the pair are swapped. The code that identifies
2320 // perfect shuffles will reject it, unless the order is reversed.
2321 shuffles::MaskT MaskStorage(SM.Mask);
2322 bool InvertedPair = false;
2323 if (HavePairs && SM.Mask[0] >= int(HwLen)) {
2324 for (int i = 0, e = SM.Mask.size(); i != e; ++i) {
2325 int M = SM.Mask[i];
2326 MaskStorage[i] = M >= int(HwLen) ? M - HwLen : M + HwLen;
2327 }
2328 InvertedPair = true;
2329 SM = ShuffleMask(MaskStorage);
2330 }
2331
2332 auto Comps = getPerfectCompletions(SM, Width: LogLen);
2333 if (llvm::is_contained(Range&: Comps, Element: 0))
2334 return OpRef::fail();
2335
2336 auto Pick = completeToPerfect(Completions: Comps, Width: LogLen);
2337 for (unsigned I = 0; I != LogLen; ++I)
2338 Perm[I] = Log2_32(Value: Pick[I]);
2339
2340 // Once we have Perm, represent it as cycles. Denote the maximum log2
2341 // (equal to log2(VecLen)-1) as M. The cycle containing M can then be
2342 // written as (M a1 a2 a3 ... an). That cycle can be broken up into
2343 // simple swaps as (M a1)(M a2)(M a3)...(M an), with the composition
2344 // order being from left to right. Any (contiguous) segment where the
2345 // values ai, ai+1...aj are either all increasing or all decreasing,
2346 // can be implemented via a single vshuffvdd/vdealvdd respectively.
2347 //
2348 // If there is a cycle (a1 a2 ... an) that does not involve M, it can
2349 // be written as (M an)(a1 a2 ... an)(M a1). The first two cycles can
2350 // then be folded to get (M a1 a2 ... an)(M a1), and the above procedure
2351 // can be used to generate a sequence of vshuffvdd/vdealvdd.
2352 //
2353 // Example:
2354 // Assume M = 4 and consider a permutation (0 1)(2 3). It can be written
2355 // as (4 0 1)(4 0) composed with (4 2 3)(4 2), or simply
2356 // (4 0 1)(4 0)(4 2 3)(4 2).
2357 // It can then be expanded into swaps as
2358 // (4 0)(4 1)(4 0)(4 2)(4 3)(4 2),
2359 // and broken up into "increasing" segments as
2360 // [(4 0)(4 1)] [(4 0)(4 2)(4 3)] [(4 2)].
2361 // This is equivalent to
2362 // (4 0 1)(4 0 2 3)(4 2),
2363 // which can be implemented as 3 vshufvdd instructions.
2364
2365 using CycleType = SmallVector<unsigned, 8>;
2366 std::set<CycleType> Cycles;
2367 std::set<unsigned> All;
2368
2369 for (unsigned I : Perm)
2370 All.insert(x: I);
2371
2372 // If the cycle contains LogLen-1, move it to the front of the cycle.
2373 // Otherwise, return the cycle unchanged.
2374 auto canonicalize = [LogLen](const CycleType &C) -> CycleType {
2375 unsigned LogPos, N = C.size();
2376 for (LogPos = 0; LogPos != N; ++LogPos)
2377 if (C[LogPos] == LogLen - 1)
2378 break;
2379 if (LogPos == N)
2380 return C;
2381
2382 CycleType NewC(C.begin() + LogPos, C.end());
2383 NewC.append(in_start: C.begin(), in_end: C.begin() + LogPos);
2384 return NewC;
2385 };
2386
2387 auto pfs = [](const std::set<CycleType> &Cs, unsigned Len) {
2388 // Ordering: shuff: 5 0 1 2 3 4, deal: 5 4 3 2 1 0 (for Log=6),
2389 // for bytes zero is included, for halfwords is not.
2390 if (Cs.size() != 1)
2391 return 0u;
2392 const CycleType &C = *Cs.begin();
2393 if (C[0] != Len - 1)
2394 return 0u;
2395 int D = Len - C.size();
2396 if (D != 0 && D != 1)
2397 return 0u;
2398
2399 bool IsDeal = true, IsShuff = true;
2400 for (unsigned I = 1; I != Len - D; ++I) {
2401 if (C[I] != Len - 1 - I)
2402 IsDeal = false;
2403 if (C[I] != I - (1 - D)) // I-1, I
2404 IsShuff = false;
2405 }
2406 // At most one, IsDeal or IsShuff, can be non-zero.
2407 assert(!(IsDeal || IsShuff) || IsDeal != IsShuff);
2408 static unsigned Deals[] = {Hexagon::V6_vdealb, Hexagon::V6_vdealh};
2409 static unsigned Shufs[] = {Hexagon::V6_vshuffb, Hexagon::V6_vshuffh};
2410 return IsDeal ? Deals[D] : (IsShuff ? Shufs[D] : 0);
2411 };
2412
2413 while (!All.empty()) {
2414 unsigned A = *All.begin();
2415 All.erase(x: A);
2416 CycleType C;
2417 C.push_back(Elt: A);
2418 for (unsigned B = Perm[A]; B != A; B = Perm[B]) {
2419 C.push_back(Elt: B);
2420 All.erase(x: B);
2421 }
2422 if (C.size() <= 1)
2423 continue;
2424 Cycles.insert(x: canonicalize(C));
2425 }
2426
2427 MVT SingleTy = getSingleVT(ElemTy: MVT::i8);
2428 MVT PairTy = getPairVT(ElemTy: MVT::i8);
2429
2430 // Recognize patterns for V6_vdeal{b,h} and V6_vshuff{b,h}.
2431 if (unsigned(VecLen) == HwLen) {
2432 if (unsigned SingleOpc = pfs(Cycles, LogLen)) {
2433 Results.push(Opc: SingleOpc, Ty: SingleTy, Ops: {Va});
2434 return OpRef::res(N: Results.top());
2435 }
2436 }
2437
2438 // From the cycles, construct the sequence of values that will
2439 // then form the control values for vdealvdd/vshuffvdd, i.e.
2440 // (M a1 a2)(M a3 a4 a5)... -> a1 a2 a3 a4 a5
2441 // This essentially strips the M value from the cycles where
2442 // it's present, and performs the insertion of M (then stripping)
2443 // for cycles without M (as described in an earlier comment).
2444 SmallVector<unsigned, 8> SwapElems;
2445 // When the input is extended (i.e. single vector becomes a pair),
2446 // this is done by using an "undef" vector as the second input.
2447 // However, then we get
2448 // input 1: GOODBITS
2449 // input 2: ........
2450 // but we need
2451 // input 1: ....BITS
2452 // input 2: ....GOOD
2453 // Then at the end, this needs to be undone. To accomplish this,
2454 // artificially add "LogLen-1" at both ends of the sequence.
2455 if (!HavePairs)
2456 SwapElems.push_back(Elt: LogLen - 1);
2457 for (const CycleType &C : Cycles) {
2458 // Do the transformation: (a1..an) -> (M a1..an)(M a1).
2459 unsigned First = (C[0] == LogLen - 1) ? 1 : 0;
2460 SwapElems.append(in_start: C.begin() + First, in_end: C.end());
2461 if (First == 0)
2462 SwapElems.push_back(Elt: C[0]);
2463 }
2464 if (!HavePairs)
2465 SwapElems.push_back(Elt: LogLen - 1);
2466
2467 const SDLoc &dl(Results.InpNode);
2468 OpRef Arg = HavePairs ? Va : concats(Lo: Va, Hi: OpRef::undef(Ty: SingleTy), Results);
2469 if (InvertedPair)
2470 Arg = concats(Lo: OpRef::hi(R: Arg), Hi: OpRef::lo(R: Arg), Results);
2471
2472 for (unsigned I = 0, E = SwapElems.size(); I != E;) {
2473 bool IsInc = I == E - 1 || SwapElems[I] < SwapElems[I + 1];
2474 unsigned S = (1u << SwapElems[I]);
2475 if (I < E - 1) {
2476 while (++I < E - 1 && IsInc == (SwapElems[I] < SwapElems[I + 1]))
2477 S |= 1u << SwapElems[I];
2478 // The above loop will not add a bit for the final SwapElems[I+1],
2479 // so add it here.
2480 S |= 1u << SwapElems[I];
2481 }
2482 ++I;
2483
2484 // Upper bits of the vdeal/vshuff parameter that do not cover any byte in
2485 // the vector are ignored. Technically, A2_tfrsi takes a signed value, which
2486 // is sign-extended to 32 bit if there is no extender. The practical
2487 // advantages are that signed values are smaller in common use cases and are
2488 // not sensitive to the vector size.
2489 int SS = SignExtend32(X: S, B: HwLog);
2490
2491 NodeTemplate Res;
2492 Results.push(Opc: Hexagon::A2_tfrsi, Ty: MVT::i32, Ops: {getSignedConst32(Val: SS, dl)});
2493 Res.Opc = IsInc ? Hexagon::V6_vshuffvdd : Hexagon::V6_vdealvdd;
2494 Res.Ty = PairTy;
2495 Res.Ops = {OpRef::hi(R: Arg), OpRef::lo(R: Arg), OpRef::res(N: -1)};
2496 Results.push(Res);
2497 Arg = OpRef::res(N: Results.top());
2498 }
2499
2500 return HavePairs ? Arg : OpRef::lo(R: Arg);
2501}
2502
2503OpRef HvxSelector::butterfly(ShuffleMask SM, OpRef Va, ResultStack &Results) {
2504 DEBUG_WITH_TYPE("isel", {dbgs() << __func__ << '\n';});
2505 // Butterfly shuffles.
2506 //
2507 // V6_vdelta
2508 // V6_vrdelta
2509 // V6_vror
2510
2511 // The assumption here is that all elements picked by Mask are in the
2512 // first operand to the vector_shuffle. This assumption is enforced
2513 // by the caller.
2514
2515 MVT ResTy = getSingleVT(ElemTy: MVT::i8);
2516 PermNetwork::Controls FC, RC;
2517 const SDLoc &dl(Results.InpNode);
2518 int VecLen = SM.Mask.size();
2519
2520 for (int M : SM.Mask) {
2521 if (M != -1 && M >= VecLen)
2522 return OpRef::fail();
2523 }
2524
2525 // Try the deltas/benes for both single vectors and vector pairs.
2526 ForwardDeltaNetwork FN(SM.Mask);
2527 if (FN.run(V&: FC)) {
2528 SDValue Ctl = getVectorConstant(Data: FC, dl);
2529 Results.push(Opc: Hexagon::V6_vdelta, Ty: ResTy, Ops: {Va, OpRef(Ctl)});
2530 return OpRef::res(N: Results.top());
2531 }
2532
2533 // Try reverse delta.
2534 ReverseDeltaNetwork RN(SM.Mask);
2535 if (RN.run(V&: RC)) {
2536 SDValue Ctl = getVectorConstant(Data: RC, dl);
2537 Results.push(Opc: Hexagon::V6_vrdelta, Ty: ResTy, Ops: {Va, OpRef(Ctl)});
2538 return OpRef::res(N: Results.top());
2539 }
2540
2541 // Do Benes.
2542 BenesNetwork BN(SM.Mask);
2543 if (BN.run(F&: FC, R&: RC)) {
2544 SDValue CtlF = getVectorConstant(Data: FC, dl);
2545 SDValue CtlR = getVectorConstant(Data: RC, dl);
2546 Results.push(Opc: Hexagon::V6_vdelta, Ty: ResTy, Ops: {Va, OpRef(CtlF)});
2547 Results.push(Opc: Hexagon::V6_vrdelta, Ty: ResTy,
2548 Ops: {OpRef::res(N: -1), OpRef(CtlR)});
2549 return OpRef::res(N: Results.top());
2550 }
2551
2552 return OpRef::fail();
2553}
2554
2555SDValue HvxSelector::getConst32(unsigned Val, const SDLoc &dl) {
2556 return DAG.getTargetConstant(Val, DL: dl, VT: MVT::i32);
2557}
2558
2559SDValue HvxSelector::getSignedConst32(int Val, const SDLoc &dl) {
2560 return DAG.getSignedTargetConstant(Val, DL: dl, VT: MVT::i32);
2561}
2562
2563SDValue HvxSelector::getVectorConstant(ArrayRef<uint8_t> Data,
2564 const SDLoc &dl) {
2565 SmallVector<SDValue, 128> Elems;
2566 for (uint8_t C : Data)
2567 Elems.push_back(Elt: DAG.getConstant(Val: C, DL: dl, VT: MVT::i8));
2568 MVT VecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: Data.size());
2569 SDValue BV = DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Elems);
2570 SDValue LV = Lower.LowerOperation(Op: BV, DAG);
2571 DAG.RemoveDeadNode(N: BV.getNode());
2572 return DAG.getNode(Opcode: HexagonISD::ISEL, DL: dl, VT: VecTy, Operand: LV);
2573}
2574
2575void HvxSelector::selectExtractSubvector(SDNode *N) {
2576 SDValue Inp = N->getOperand(Num: 0);
2577 MVT ResTy = N->getValueType(ResNo: 0).getSimpleVT();
2578 unsigned Idx = N->getConstantOperandVal(Num: 1);
2579
2580 [[maybe_unused]] MVT InpTy = Inp.getValueType().getSimpleVT();
2581 [[maybe_unused]] unsigned ResLen = ResTy.getVectorNumElements();
2582 assert(InpTy.getVectorElementType() == ResTy.getVectorElementType());
2583 assert(2 * ResLen == InpTy.getVectorNumElements());
2584 assert(Idx == 0 || Idx == ResLen);
2585
2586 unsigned SubReg = Idx == 0 ? Hexagon::vsub_lo : Hexagon::vsub_hi;
2587 SDValue Ext = DAG.getTargetExtractSubreg(SRIdx: SubReg, DL: SDLoc(N), VT: ResTy, Operand: Inp);
2588
2589 ISel.ReplaceNode(F: N, T: Ext.getNode());
2590}
2591
2592void HvxSelector::selectShuffle(SDNode *N) {
2593 DEBUG_WITH_TYPE("isel", {
2594 dbgs() << "Starting " << __func__ << " on node:\n";
2595 N->dump(&DAG);
2596 });
2597 MVT ResTy = N->getValueType(ResNo: 0).getSimpleVT();
2598 // Assume that vector shuffles operate on vectors of bytes.
2599 assert(ResTy.isVectorOf(MVT::i8));
2600
2601 auto *SN = cast<ShuffleVectorSDNode>(Val: N);
2602 std::vector<int> Mask(SN->getMask().begin(), SN->getMask().end());
2603 // This shouldn't really be necessary. Is it?
2604 for (int &Idx : Mask)
2605 if (Idx != -1 && Idx < 0)
2606 Idx = -1;
2607
2608 unsigned VecLen = Mask.size();
2609 bool HavePairs = (2*HwLen == VecLen);
2610 assert(ResTy.getSizeInBits() / 8 == VecLen);
2611
2612 // Vd = vector_shuffle Va, Vb, Mask
2613 //
2614
2615 bool UseLeft = false, UseRight = false;
2616 for (unsigned I = 0; I != VecLen; ++I) {
2617 if (Mask[I] == -1)
2618 continue;
2619 unsigned Idx = Mask[I];
2620 assert(Idx < 2*VecLen);
2621 if (Idx < VecLen)
2622 UseLeft = true;
2623 else
2624 UseRight = true;
2625 }
2626
2627 DEBUG_WITH_TYPE("isel", {
2628 dbgs() << "VecLen=" << VecLen << " HwLen=" << HwLen << " UseLeft="
2629 << UseLeft << " UseRight=" << UseRight << " HavePairs="
2630 << HavePairs << '\n';
2631 });
2632 // If the mask is all -1's, generate "undef".
2633 if (!UseLeft && !UseRight) {
2634 ISel.ReplaceNode(F: N, T: ISel.selectUndef(dl: SDLoc(SN), ResTy).getNode());
2635 return;
2636 }
2637
2638 SDValue Vec0 = N->getOperand(Num: 0);
2639 SDValue Vec1 = N->getOperand(Num: 1);
2640 assert(Vec0.getValueType() == ResTy && Vec1.getValueType() == ResTy);
2641
2642 ResultStack Results(SN);
2643 OpRef Va = OpRef::undef(Ty: ResTy);
2644 OpRef Vb = OpRef::undef(Ty: ResTy);
2645
2646 if (!Vec0.isUndef()) {
2647 Results.push(Opc: TargetOpcode::COPY, Ty: ResTy, Ops: {Vec0});
2648 Va = OpRef::OpRef::res(N: Results.top());
2649 }
2650 if (!Vec1.isUndef()) {
2651 Results.push(Opc: TargetOpcode::COPY, Ty: ResTy, Ops: {Vec1});
2652 Vb = OpRef::res(N: Results.top());
2653 }
2654
2655 OpRef Res = !HavePairs ? shuffs2(SM: ShuffleMask(Mask), Va, Vb, Results)
2656 : shuffp2(SM: ShuffleMask(Mask), Va, Vb, Results);
2657
2658 bool Done = Res.isValid();
2659 if (Done) {
2660 // Make sure that Res is on the stack before materializing.
2661 Results.push(Opc: TargetOpcode::COPY, Ty: ResTy, Ops: {Res});
2662 materialize(Results);
2663 } else {
2664 Done = scalarizeShuffle(Mask, dl: SDLoc(N), ResTy, Va: Vec0, Vb: Vec1, N);
2665 }
2666
2667 if (!Done) {
2668#ifndef NDEBUG
2669 dbgs() << "Unhandled shuffle:\n";
2670 SN->dumpr(&DAG);
2671#endif
2672 llvm_unreachable("Failed to select vector shuffle");
2673 }
2674}
2675
2676void HvxSelector::selectRor(SDNode *N) {
2677 // If this is a rotation by less than 8, use V6_valignbi.
2678 MVT Ty = N->getValueType(ResNo: 0).getSimpleVT();
2679 const SDLoc &dl(N);
2680 SDValue VecV = N->getOperand(Num: 0);
2681 SDValue RotV = N->getOperand(Num: 1);
2682 SDNode *NewN = nullptr;
2683
2684 if (auto *CN = dyn_cast<ConstantSDNode>(Val: RotV.getNode())) {
2685 unsigned S = CN->getZExtValue() % HST.getVectorLength();
2686 if (S == 0) {
2687 NewN = VecV.getNode();
2688 } else if (isUInt<3>(x: S)) {
2689 NewN = DAG.getMachineNode(Opcode: Hexagon::V6_valignbi, dl, VT: Ty,
2690 Ops: {VecV, VecV, getConst32(Val: S, dl)});
2691 }
2692 }
2693
2694 if (!NewN)
2695 NewN = DAG.getMachineNode(Opcode: Hexagon::V6_vror, dl, VT: Ty, Ops: {VecV, RotV});
2696
2697 ISel.ReplaceNode(F: N, T: NewN);
2698}
2699
2700void HvxSelector::selectVAlign(SDNode *N) {
2701 SDValue Vv = N->getOperand(Num: 0);
2702 SDValue Vu = N->getOperand(Num: 1);
2703 SDValue Rt = N->getOperand(Num: 2);
2704 SDNode *NewN = DAG.getMachineNode(Opcode: Hexagon::V6_valignb, dl: SDLoc(N),
2705 VT: N->getValueType(ResNo: 0), Ops: {Vv, Vu, Rt});
2706 ISel.ReplaceNode(F: N, T: NewN);
2707 DAG.RemoveDeadNode(N);
2708}
2709
2710void HexagonDAGToDAGISel::PreprocessHvxISelDAG() {
2711 auto getNodes = [this]() -> std::vector<SDNode *> {
2712 std::vector<SDNode *> T;
2713 T.reserve(n: CurDAG->allnodes_size());
2714 for (SDNode &N : CurDAG->allnodes())
2715 T.push_back(x: &N);
2716 return T;
2717 };
2718
2719 ppHvxShuffleOfShuffle(Nodes: getNodes());
2720}
2721
2722template <> struct std::hash<SDValue> {
2723 std::size_t operator()(SDValue V) const {
2724 return std::hash<const void *>()(V.getNode()) +
2725 std::hash<unsigned>()(V.getResNo());
2726 };
2727};
2728
2729void HexagonDAGToDAGISel::ppHvxShuffleOfShuffle(std::vector<SDNode *> &&Nodes) {
2730 // Motivating case:
2731 // t10: v64i32 = ...
2732 // t46: v128i8 = vector_shuffle<...> t44, t45
2733 // t48: v128i8 = vector_shuffle<...> t44, t45
2734 // t42: v128i8 = vector_shuffle<...> t46, t48
2735 // t12: v32i32 = extract_subvector t10, Constant:i32<0>
2736 // t44: v128i8 = bitcast t12
2737 // t15: v32i32 = extract_subvector t10, Constant:i32<32>
2738 // t45: v128i8 = bitcast t15
2739 SelectionDAG &DAG = *CurDAG;
2740 unsigned HwLen = HST->getVectorLength();
2741
2742 struct SubVectorInfo {
2743 SubVectorInfo(SDValue S, unsigned H) : Src(S), HalfIdx(H) {}
2744 SDValue Src;
2745 unsigned HalfIdx;
2746 };
2747
2748 using MapType = DenseMap<SDValue, unsigned>;
2749
2750 auto getMaskElt = [&](unsigned Idx, ShuffleVectorSDNode *Shuff0,
2751 ShuffleVectorSDNode *Shuff1,
2752 const MapType &OpMap) -> int {
2753 // Treat Shuff0 and Shuff1 as operands to another vector shuffle, and
2754 // Idx as a (non-undef) element of the top level shuffle's mask, that
2755 // is, index into concat(Shuff0, Shuff1).
2756 // Assuming that Shuff0 and Shuff1 both operate on subvectors of the
2757 // same source vector (as described by OpMap), return the index of
2758 // that source vector corresponding to Idx.
2759 ShuffleVectorSDNode *OpShuff = Idx < HwLen ? Shuff0 : Shuff1;
2760 if (Idx >= HwLen)
2761 Idx -= HwLen;
2762
2763 // Get the mask index that M points at in the corresponding operand.
2764 int MaybeN = OpShuff->getMaskElt(Idx);
2765 if (MaybeN < 0)
2766 return -1;
2767
2768 auto N = static_cast<unsigned>(MaybeN);
2769 unsigned SrcBase = N < HwLen ? OpMap.at(Val: OpShuff->getOperand(Num: 0))
2770 : OpMap.at(Val: OpShuff->getOperand(Num: 1));
2771 if (N >= HwLen)
2772 N -= HwLen;
2773
2774 return N + SrcBase;
2775 };
2776
2777 auto fold3 = [&](SDValue TopShuff, SDValue Inp, MapType &&OpMap) -> SDValue {
2778 // Fold all 3 shuffles into a single one.
2779 auto *This = cast<ShuffleVectorSDNode>(Val&: TopShuff);
2780 auto *S0 = cast<ShuffleVectorSDNode>(Val: TopShuff.getOperand(i: 0));
2781 auto *S1 = cast<ShuffleVectorSDNode>(Val: TopShuff.getOperand(i: 1));
2782 ArrayRef<int> TopMask = This->getMask();
2783 // This should be guaranteed by type checks in the caller, and the fact
2784 // that all shuffles should have been promoted to operate on MVT::i8.
2785 assert(TopMask.size() == S0->getMask().size() &&
2786 TopMask.size() == S1->getMask().size());
2787 assert(TopMask.size() == HwLen);
2788
2789 SmallVector<int, 256> FoldedMask(2 * HwLen);
2790 for (unsigned I = 0; I != HwLen; ++I) {
2791 int MaybeM = TopMask[I];
2792 if (MaybeM >= 0) {
2793 FoldedMask[I] =
2794 getMaskElt(static_cast<unsigned>(MaybeM), S0, S1, OpMap);
2795 } else {
2796 FoldedMask[I] = -1;
2797 }
2798 }
2799 // The second half of the result will be all-undef.
2800 std::fill(first: FoldedMask.begin() + HwLen, last: FoldedMask.end(), value: -1);
2801
2802 // Return
2803 // FoldedShuffle = (Shuffle Inp, undef, FoldedMask)
2804 // (LoHalf FoldedShuffle)
2805 const SDLoc &dl(TopShuff);
2806 MVT SingleTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2807 MVT PairTy = MVT::getVectorVT(VT: MVT::i8, NumElements: 2 * HwLen);
2808 SDValue FoldedShuff =
2809 DAG.getVectorShuffle(VT: PairTy, dl, N1: DAG.getBitcast(VT: PairTy, V: Inp),
2810 N2: DAG.getUNDEF(VT: PairTy), Mask: FoldedMask);
2811 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SingleTy, N1: FoldedShuff,
2812 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
2813 };
2814
2815 auto getSourceInfo = [](SDValue V) -> std::optional<SubVectorInfo> {
2816 while (V.getOpcode() == ISD::BITCAST)
2817 V = V.getOperand(i: 0);
2818 if (V.getOpcode() != ISD::EXTRACT_SUBVECTOR)
2819 return std::nullopt;
2820 return SubVectorInfo(V.getOperand(i: 0),
2821 !cast<ConstantSDNode>(Val: V.getOperand(i: 1))->isZero());
2822 };
2823
2824 for (SDNode *N : Nodes) {
2825 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
2826 continue;
2827 EVT ResTy = N->getValueType(ResNo: 0);
2828 if (ResTy.getVectorElementType() != MVT::i8)
2829 continue;
2830 if (ResTy.getVectorNumElements() != HwLen)
2831 continue;
2832
2833 SDValue V0 = N->getOperand(Num: 0);
2834 SDValue V1 = N->getOperand(Num: 1);
2835 if (V0.getOpcode() != ISD::VECTOR_SHUFFLE)
2836 continue;
2837 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE)
2838 continue;
2839 if (V0.getValueType() != ResTy || V1.getValueType() != ResTy)
2840 continue;
2841
2842 // Check if all operands of the two operand shuffles are extract_subvectors
2843 // from the same vector pair.
2844 auto V0A = getSourceInfo(V0.getOperand(i: 0));
2845 if (!V0A.has_value())
2846 continue;
2847 auto V0B = getSourceInfo(V0.getOperand(i: 1));
2848 if (!V0B.has_value() || V0B->Src != V0A->Src)
2849 continue;
2850 auto V1A = getSourceInfo(V1.getOperand(i: 0));
2851 if (!V1A.has_value() || V1A->Src != V0A->Src)
2852 continue;
2853 auto V1B = getSourceInfo(V1.getOperand(i: 1));
2854 if (!V1B.has_value() || V1B->Src != V0A->Src)
2855 continue;
2856
2857 // The source must be a pair. This should be guaranteed here,
2858 // but check just in case.
2859 assert(V0A->Src.getValueType().getSizeInBits() == 16 * HwLen);
2860
2861 MapType OpMap = {
2862 {V0.getOperand(i: 0), V0A->HalfIdx * HwLen},
2863 {V0.getOperand(i: 1), V0B->HalfIdx * HwLen},
2864 {V1.getOperand(i: 0), V1A->HalfIdx * HwLen},
2865 {V1.getOperand(i: 1), V1B->HalfIdx * HwLen},
2866 };
2867 SDValue NewS = fold3(SDValue(N, 0), V0A->Src, std::move(OpMap));
2868 ReplaceNode(F: N, T: NewS.getNode());
2869 }
2870}
2871
2872void HexagonDAGToDAGISel::SelectHvxExtractSubvector(SDNode *N) {
2873 HvxSelector(*this, *CurDAG).selectExtractSubvector(N);
2874}
2875
2876void HexagonDAGToDAGISel::SelectHvxShuffle(SDNode *N) {
2877 HvxSelector(*this, *CurDAG).selectShuffle(N);
2878}
2879
2880void HexagonDAGToDAGISel::SelectHvxRor(SDNode *N) {
2881 HvxSelector(*this, *CurDAG).selectRor(N);
2882}
2883
2884void HexagonDAGToDAGISel::SelectHvxVAlign(SDNode *N) {
2885 HvxSelector(*this, *CurDAG).selectVAlign(N);
2886}
2887
2888void HexagonDAGToDAGISel::SelectV65GatherPred(SDNode *N) {
2889 const SDLoc &dl(N);
2890 SDValue Chain = N->getOperand(Num: 0);
2891 SDValue Address = N->getOperand(Num: 2);
2892 SDValue Predicate = N->getOperand(Num: 3);
2893 SDValue Base = N->getOperand(Num: 4);
2894 SDValue Modifier = N->getOperand(Num: 5);
2895 SDValue Offset = N->getOperand(Num: 6);
2896 SDValue ImmOperand = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
2897
2898 unsigned Opcode;
2899 unsigned IntNo = N->getConstantOperandVal(Num: 1);
2900 switch (IntNo) {
2901 default:
2902 llvm_unreachable("Unexpected HVX gather intrinsic.");
2903 case Intrinsic::hexagon_V6_vgathermhq:
2904 case Intrinsic::hexagon_V6_vgathermhq_128B:
2905 Opcode = Hexagon::V6_vgathermhq_pseudo;
2906 break;
2907 case Intrinsic::hexagon_V6_vgathermwq:
2908 case Intrinsic::hexagon_V6_vgathermwq_128B:
2909 Opcode = Hexagon::V6_vgathermwq_pseudo;
2910 break;
2911 case Intrinsic::hexagon_V6_vgathermhwq:
2912 case Intrinsic::hexagon_V6_vgathermhwq_128B:
2913 Opcode = Hexagon::V6_vgathermhwq_pseudo;
2914 break;
2915 }
2916
2917 SDVTList VTs = CurDAG->getVTList(VT: MVT::Other);
2918 SDValue Ops[] = { Address, ImmOperand,
2919 Predicate, Base, Modifier, Offset, Chain };
2920 SDNode *Result = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
2921
2922 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2923 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Result), NewMemRefs: {MemOp});
2924
2925 ReplaceNode(F: N, T: Result);
2926}
2927
2928void HexagonDAGToDAGISel::SelectV65Gather(SDNode *N) {
2929 const SDLoc &dl(N);
2930 SDValue Chain = N->getOperand(Num: 0);
2931 SDValue Address = N->getOperand(Num: 2);
2932 SDValue Base = N->getOperand(Num: 3);
2933 SDValue Modifier = N->getOperand(Num: 4);
2934 SDValue Offset = N->getOperand(Num: 5);
2935 SDValue ImmOperand = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
2936
2937 unsigned Opcode;
2938 unsigned IntNo = N->getConstantOperandVal(Num: 1);
2939 switch (IntNo) {
2940 default:
2941 llvm_unreachable("Unexpected HVX gather intrinsic.");
2942 case Intrinsic::hexagon_V6_vgathermh:
2943 case Intrinsic::hexagon_V6_vgathermh_128B:
2944 Opcode = Hexagon::V6_vgathermh_pseudo;
2945 break;
2946 case Intrinsic::hexagon_V6_vgathermw:
2947 case Intrinsic::hexagon_V6_vgathermw_128B:
2948 Opcode = Hexagon::V6_vgathermw_pseudo;
2949 break;
2950 case Intrinsic::hexagon_V6_vgathermhw:
2951 case Intrinsic::hexagon_V6_vgathermhw_128B:
2952 Opcode = Hexagon::V6_vgathermhw_pseudo;
2953 break;
2954 case Intrinsic::hexagon_V6_vgather_vscattermh:
2955 case Intrinsic::hexagon_V6_vgather_vscattermh_128B:
2956 Opcode = Hexagon::V6_vgather_vscatter_mh_pseudo;
2957 break;
2958 }
2959
2960 SDVTList VTs = CurDAG->getVTList(VT: MVT::Other);
2961 SDValue Ops[] = { Address, ImmOperand, Base, Modifier, Offset, Chain };
2962 SDNode *Result = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
2963
2964 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2965 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Result), NewMemRefs: {MemOp});
2966
2967 ReplaceNode(F: N, T: Result);
2968}
2969
2970void HexagonDAGToDAGISel::SelectHVXDualOutput(SDNode *N) {
2971 unsigned IID = N->getConstantOperandVal(Num: 0);
2972 SDNode *Result;
2973 switch (IID) {
2974 case Intrinsic::hexagon_V6_vaddcarry: {
2975 std::array<SDValue, 3> Ops = {
2976 ._M_elems: {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3)}};
2977 SDVTList VTs = CurDAG->getVTList(VT1: MVT::v16i32, VT2: MVT::v64i1);
2978 Result = CurDAG->getMachineNode(Opcode: Hexagon::V6_vaddcarry, dl: SDLoc(N), VTs, Ops);
2979 break;
2980 }
2981 case Intrinsic::hexagon_V6_vaddcarry_128B: {
2982 std::array<SDValue, 3> Ops = {
2983 ._M_elems: {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3)}};
2984 SDVTList VTs = CurDAG->getVTList(VT1: MVT::v32i32, VT2: MVT::v128i1);
2985 Result = CurDAG->getMachineNode(Opcode: Hexagon::V6_vaddcarry, dl: SDLoc(N), VTs, Ops);
2986 break;
2987 }
2988 case Intrinsic::hexagon_V6_vsubcarry: {
2989 std::array<SDValue, 3> Ops = {
2990 ._M_elems: {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3)}};
2991 SDVTList VTs = CurDAG->getVTList(VT1: MVT::v16i32, VT2: MVT::v64i1);
2992 Result = CurDAG->getMachineNode(Opcode: Hexagon::V6_vsubcarry, dl: SDLoc(N), VTs, Ops);
2993 break;
2994 }
2995 case Intrinsic::hexagon_V6_vsubcarry_128B: {
2996 std::array<SDValue, 3> Ops = {
2997 ._M_elems: {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3)}};
2998 SDVTList VTs = CurDAG->getVTList(VT1: MVT::v32i32, VT2: MVT::v128i1);
2999 Result = CurDAG->getMachineNode(Opcode: Hexagon::V6_vsubcarry, dl: SDLoc(N), VTs, Ops);
3000 break;
3001 }
3002 default:
3003 llvm_unreachable("Unexpected HVX dual output intrinsic.");
3004 }
3005 ReplaceUses(F: N, T: Result);
3006 ReplaceUses(F: SDValue(N, 0), T: SDValue(Result, 0));
3007 ReplaceUses(F: SDValue(N, 1), T: SDValue(Result, 1));
3008 CurDAG->RemoveDeadNode(N);
3009}
3010
3011// Check if the intrinsic corresponds to IEEE HVX instruction.
3012bool HexagonDAGToDAGISel::isIEEEHVXIntrinsic(unsigned Opcode) {
3013 return Opcode == Intrinsic::hexagon_V6_vabs_hf_128B ||
3014 Opcode == Intrinsic::hexagon_V6_vabs_sf_128B ||
3015 Opcode == Intrinsic::hexagon_V6_vsub_hf_hf_128B ||
3016 Opcode == Intrinsic::hexagon_V6_vadd_hf_hf_128B ||
3017 Opcode == Intrinsic::hexagon_V6_vadd_sf_hf_128B ||
3018 Opcode == Intrinsic::hexagon_V6_vsub_sf_hf_128B ||
3019 Opcode == Intrinsic::hexagon_V6_vadd_sf_sf_128B ||
3020 Opcode == Intrinsic::hexagon_V6_vsub_sf_sf_128B ||
3021 Opcode == Intrinsic::hexagon_V6_vassign_fp_128B ||
3022 Opcode == Intrinsic::hexagon_V6_vfmin_hf_128B ||
3023 Opcode == Intrinsic::hexagon_V6_vfmin_sf_128B ||
3024 Opcode == Intrinsic::hexagon_V6_vfmax_hf_128B ||
3025 Opcode == Intrinsic::hexagon_V6_vfmax_sf_128B ||
3026 Opcode == Intrinsic::hexagon_V6_vfneg_hf_128B ||
3027 Opcode == Intrinsic::hexagon_V6_vfneg_sf_128B ||
3028 Opcode == Intrinsic::hexagon_V6_vmpy_sf_hf_acc_128B ||
3029 Opcode == Intrinsic::hexagon_V6_vmpy_hf_hf_acc_128B ||
3030 Opcode == Intrinsic::hexagon_V6_vmpy_sf_hf_128B ||
3031 Opcode == Intrinsic::hexagon_V6_vmpy_hf_hf_128B ||
3032 Opcode == Intrinsic::hexagon_V6_vmpy_sf_sf_128B ||
3033 Opcode == Intrinsic::hexagon_V6_vcvt_sf_hf_128B ||
3034 Opcode == Intrinsic::hexagon_V6_vcvt_hf_h_128B;
3035}
3036
3037// Translate IEEE HVX intrinsics to QFloat instructions.
3038void HexagonDAGToDAGISel::translateIEEEIntrinsicToQFloat(SDNode *N,
3039 unsigned &Opcode) {
3040 SDLoc DL(N);
3041 MVT ResTy = N->getValueType(ResNo: 0).getSimpleVT();
3042 switch (Opcode) {
3043 // v0.sf = vadd(v0.sf,v1.sf) is translated to
3044 // v2.qf32 = vadd(v0.sf,v1.sf); v0.sf = v2.qf32.
3045 case Intrinsic::hexagon_V6_vadd_sf_sf_128B: {
3046 SDNode *AddNode = CurDAG->getMachineNode(
3047 Opcode: Hexagon::V6_vadd_sf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3048 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL,
3049 VT: ResTy, Op1: SDValue(AddNode, 0));
3050 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3051 CurDAG->RemoveDeadNode(N);
3052 return;
3053 }
3054 // v0.sf = vsub(v0.sf,v1.sf) is translated to
3055 // v2.qf32 = vsub(v0.sf,v1.sf); v0.sf = v2.qf32.
3056 case Intrinsic::hexagon_V6_vsub_sf_sf_128B: {
3057 SDNode *SubNode = CurDAG->getMachineNode(
3058 Opcode: Hexagon::V6_vsub_sf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3059 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL,
3060 VT: ResTy, Op1: SDValue(SubNode, 0));
3061 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3062 CurDAG->RemoveDeadNode(N);
3063 return;
3064 }
3065 // v0.hf = vadd(v0.hf,v1.hf) is translated to
3066 // v2.qf16 = vadd(v0.hf,v1.hf); v0.hf = v2.qf16.
3067 case Intrinsic::hexagon_V6_vadd_hf_hf_128B: {
3068 SDNode *AddNode = CurDAG->getMachineNode(
3069 Opcode: Hexagon::V6_vadd_hf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3070 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_hf_qf16, dl: DL,
3071 VT: ResTy, Op1: SDValue(AddNode, 0));
3072 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3073 CurDAG->RemoveDeadNode(N);
3074 return;
3075 }
3076 // v0.hf = vsub(v0.hf,v1.hf) is translated to
3077 // v2.qf16 = vsub(v0.hf,v1.hf); v0.hf = v2.qf16.
3078 case Intrinsic::hexagon_V6_vsub_hf_hf_128B: {
3079 SDNode *SubNode = CurDAG->getMachineNode(
3080 Opcode: Hexagon::V6_vsub_hf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3081 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_hf_qf16, dl: DL,
3082 VT: ResTy, Op1: SDValue(SubNode, 0));
3083 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3084 CurDAG->RemoveDeadNode(N);
3085 return;
3086 }
3087 // v1:0.sf = vadd(v0.hf,v1.hf) is translated to
3088 // r2 = #15360; v2.h = vsplat(r2);
3089 // v5:4.qf32 = vmpy(v1.hf,v2.hf); v31:30.qf32 = vmpy(v0.hf,v2.hf)
3090 // v4.qf32 = vadd(v30.qf32,v4.qf32); v3.qf32 = vadd(v31.qf32,v5.qf32)
3091 // v0.sf = v4.qf32; v1.sf = v3.qf32
3092 // Widen the hf operands to sf by doing a widening multiply with 1.0f
3093 // and perform the add.
3094 case Intrinsic::hexagon_V6_vadd_sf_hf_128B: {
3095 SDValue Const = CurDAG->getTargetConstant(Val: 0x3C00U, DL, VT: MVT::i32);
3096 SDNode *SplatPseudoNode =
3097 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatih, dl: DL, VT: ResTy, Op1: Const);
3098 SDNode *MpyNodeOp1 =
3099 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: ResTy,
3100 Op1: N->getOperand(Num: 1), Op2: SDValue(SplatPseudoNode, 0));
3101 SDNode *MpyNodeOp2 =
3102 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: ResTy,
3103 Op1: N->getOperand(Num: 2), Op2: SDValue(SplatPseudoNode, 0));
3104
3105 SDValue LoRegOp1 = CurDAG->getTargetExtractSubreg(
3106 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp1, 0));
3107 SDValue HiRegOp1 = CurDAG->getTargetExtractSubreg(
3108 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp1, 0));
3109 SDValue LoRegOp2 = CurDAG->getTargetExtractSubreg(
3110 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp2, 0));
3111 SDValue HiRegOp2 = CurDAG->getTargetExtractSubreg(
3112 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp2, 0));
3113
3114 SDNode *LoAddNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vadd_qf32, dl: DL,
3115 VT: MVT::v32i32, Op1: LoRegOp1, Op2: LoRegOp2);
3116 SDNode *HiAddNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vadd_qf32, dl: DL,
3117 VT: MVT::v32i32, Op1: HiRegOp1, Op2: HiRegOp2);
3118
3119 SDNode *ConvLoNode = CurDAG->getMachineNode(
3120 Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL, VT: MVT::v32i32, Op1: SDValue(LoAddNode, 0));
3121 SDNode *ConvHiNode = CurDAG->getMachineNode(
3122 Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL, VT: MVT::v32i32, Op1: SDValue(HiAddNode, 0));
3123
3124 SDValue RC =
3125 CurDAG->getTargetConstant(Val: Hexagon::HvxWRRegClassID, DL, VT: MVT::i32);
3126 SDValue SubRegL = CurDAG->getTargetConstant(Val: Hexagon::vsub_lo, DL, VT: MVT::i32);
3127 SDValue SubRegH = CurDAG->getTargetConstant(Val: Hexagon::vsub_hi, DL, VT: MVT::i32);
3128 const SDValue Ops[] = {RC, SDValue(ConvHiNode, 0), SubRegH,
3129 SDValue(ConvLoNode, 0), SubRegL};
3130 SDNode *RS = CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL,
3131 VT: MVT::v64i32, Ops);
3132
3133 ReplaceUses(F: SDValue(N, 0), T: SDValue(RS, 0));
3134 CurDAG->RemoveDeadNode(N);
3135 return;
3136 }
3137 // v1:0.sf = vsub(v0.hf,v1.hf) is translated to
3138 // r2 = #15360; v2.h = vsplat(r2);
3139 // v5:4.qf32 = vmpy(v1.hf,v2.hf); v31:30.qf32 = vmpy(v0.hf,v2.hf)
3140 // v4.qf32 = vsub(v30.qf32,v4.qf32); v3.qf32 = vsub(v31.qf32,v5.qf32)
3141 // v0.sf = v4.qf32; v1.sf = v3.qf32
3142 // Widen the hf operands to sf by doing a widening multiply with 1.0f
3143 // and perform the sub.
3144 case Intrinsic::hexagon_V6_vsub_sf_hf_128B: {
3145 SDValue Const = CurDAG->getTargetConstant(Val: 0x3C00U, DL, VT: MVT::i32);
3146 SDNode *SplatPseudoNode =
3147 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatih, dl: DL, VT: ResTy, Op1: Const);
3148 SDNode *MpyNodeOp1 =
3149 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: ResTy,
3150 Op1: N->getOperand(Num: 1), Op2: SDValue(SplatPseudoNode, 0));
3151 SDNode *MpyNodeOp2 =
3152 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: ResTy,
3153 Op1: N->getOperand(Num: 2), Op2: SDValue(SplatPseudoNode, 0));
3154
3155 SDValue LoRegOp1 = CurDAG->getTargetExtractSubreg(
3156 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp1, 0));
3157 SDValue HiRegOp1 = CurDAG->getTargetExtractSubreg(
3158 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp1, 0));
3159 SDValue LoRegOp2 = CurDAG->getTargetExtractSubreg(
3160 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp2, 0));
3161 SDValue HiRegOp2 = CurDAG->getTargetExtractSubreg(
3162 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNodeOp2, 0));
3163
3164 SDNode *LoSubNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vsub_qf32, dl: DL,
3165 VT: MVT::v32i32, Op1: LoRegOp1, Op2: LoRegOp2);
3166 SDNode *HiSubNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vsub_qf32, dl: DL,
3167 VT: MVT::v32i32, Op1: HiRegOp1, Op2: HiRegOp2);
3168
3169 SDNode *ConvLoNode = CurDAG->getMachineNode(
3170 Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL, VT: MVT::v32i32, Op1: SDValue(LoSubNode, 0));
3171 SDNode *ConvHiNode = CurDAG->getMachineNode(
3172 Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL, VT: MVT::v32i32, Op1: SDValue(HiSubNode, 0));
3173
3174 SDValue RC =
3175 CurDAG->getTargetConstant(Val: Hexagon::HvxWRRegClassID, DL, VT: MVT::i32);
3176 SDValue SubRegL = CurDAG->getTargetConstant(Val: Hexagon::vsub_lo, DL, VT: MVT::i32);
3177 SDValue SubRegH = CurDAG->getTargetConstant(Val: Hexagon::vsub_hi, DL, VT: MVT::i32);
3178 const SDValue Ops[] = {RC, SDValue(ConvHiNode, 0), SubRegH,
3179 SDValue(ConvLoNode, 0), SubRegL};
3180 SDNode *RS = CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL,
3181 VT: MVT::v64i32, Ops);
3182
3183 ReplaceUses(F: SDValue(N, 0), T: SDValue(RS, 0));
3184 CurDAG->RemoveDeadNode(N);
3185 return;
3186 }
3187 // v0.w = vfmv(v1.w) is translated to v0 = v1.
3188 case Intrinsic::hexagon_V6_vassign_fp_128B: {
3189 SDNode *AssignNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vassign, dl: DL, VT: ResTy,
3190 Op1: N->getOperand(Num: 1));
3191 ReplaceUses(F: SDValue(N, 0), T: SDValue(AssignNode, 0));
3192 CurDAG->RemoveDeadNode(N);
3193 return;
3194 }
3195 // v0.hf = vmpy(v0.hf,v1.hf) is translated to
3196 // v1:0.qf32 = vmpy(v0.hf,v1.hf); v0.hf = v1:0.qf32.
3197 case Intrinsic::hexagon_V6_vmpy_hf_hf_128B: {
3198 SDNode *MpyNode =
3199 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: MVT::v64i32,
3200 Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3201 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_hf_qf32, dl: DL,
3202 VT: ResTy, Op1: SDValue(MpyNode, 0));
3203 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3204 CurDAG->RemoveDeadNode(N);
3205 return;
3206 }
3207 // v0.sf = vmpy(v0.sf,v1.sf) is translated to
3208 // v2.qf32 = vmpy(v0.sf,v1.sf); v0.sf = v2.qf32.
3209 case Intrinsic::hexagon_V6_vmpy_sf_sf_128B: {
3210 SDNode *MpyNode =
3211 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_sf, dl: DL, VT: ResTy,
3212 Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3213 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL,
3214 VT: ResTy, Op1: SDValue(MpyNode, 0));
3215 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3216 CurDAG->RemoveDeadNode(N);
3217 return;
3218 }
3219 // v1:0.sf = vmpy(v0.hf,v1.hf) is translated to
3220 // v3:2.qf32 = vmpy(v0.hf,v1.hf); v0.sf = v2.qf32 ; v1.sf = v3.qf32.
3221 case Intrinsic::hexagon_V6_vmpy_sf_hf_128B: {
3222 SDNode *MpyNode =
3223 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: ResTy,
3224 Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3225 SDValue LoReg = CurDAG->getTargetExtractSubreg(
3226 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3227 SDValue HiReg = CurDAG->getTargetExtractSubreg(
3228 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3229 SDNode *ConvLoNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL,
3230 VT: MVT::v32i32, Op1: LoReg);
3231 SDNode *ConvHiNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL,
3232 VT: MVT::v32i32, Op1: HiReg);
3233
3234 SDValue RC =
3235 CurDAG->getTargetConstant(Val: Hexagon::HvxWRRegClassID, DL, VT: MVT::i32);
3236 SDValue SubRegL = CurDAG->getTargetConstant(Val: Hexagon::vsub_lo, DL, VT: MVT::i32);
3237 SDValue SubRegH = CurDAG->getTargetConstant(Val: Hexagon::vsub_hi, DL, VT: MVT::i32);
3238 const SDValue Ops[] = {RC, SDValue(ConvHiNode, 0), SubRegH,
3239 SDValue(ConvLoNode, 0), SubRegL};
3240 SDNode *RS = CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL,
3241 VT: MVT::v64i32, Ops);
3242
3243 ReplaceUses(F: SDValue(N, 0), T: SDValue(RS, 0));
3244 CurDAG->RemoveDeadNode(N);
3245 return;
3246 }
3247 // v0.hf += vmpy(v1.hf,v2.hf) is translated to
3248 // v7:6.qf32 = vmpy(v1.hf, v2.hf) // widening multiply
3249 // V5:4.qf32 = vmpy(v0.hf,1.0) // Convert accum to qf32
3250 // V4.qf32 = vadd(V6.qf32, V4.qf32) // accumulation
3251 // V5.qf32 = vadd(V7.qf32, V5.qf32) // accumulation
3252 // V4.hf = V5:4.qf32
3253 case Intrinsic::hexagon_V6_vmpy_hf_hf_acc_128B: {
3254 SDNode *MpyNode =
3255 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: MVT::v64i32,
3256 Op1: N->getOperand(Num: 2), Op2: N->getOperand(Num: 3));
3257
3258 SDValue Const = CurDAG->getTargetConstant(Val: 0x3C00U, DL, VT: MVT::i32);
3259 SDNode *SplatConstNode =
3260 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatih, dl: DL, VT: MVT::v64i32, Op1: Const);
3261 SDNode *WidenAcc =
3262 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: MVT::v64i32,
3263 Op1: N->getOperand(Num: 1), Op2: SDValue(SplatConstNode, 0));
3264
3265 SDValue LoMpyNode = CurDAG->getTargetExtractSubreg(
3266 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3267 SDValue HiMpyNode = CurDAG->getTargetExtractSubreg(
3268 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3269 SDValue LoWidenAcc = CurDAG->getTargetExtractSubreg(
3270 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(WidenAcc, 0));
3271 SDValue HiWidenAcc = CurDAG->getTargetExtractSubreg(
3272 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(WidenAcc, 0));
3273
3274 SDNode *LoAddNode = CurDAG->getMachineNode(
3275 Opcode: Hexagon::V6_vadd_qf32, dl: DL, VT: MVT::v32i32, Op1: LoWidenAcc, Op2: LoMpyNode);
3276 SDNode *HiAddNode = CurDAG->getMachineNode(
3277 Opcode: Hexagon::V6_vadd_qf32, dl: DL, VT: MVT::v32i32, Op1: HiWidenAcc, Op2: HiMpyNode);
3278
3279 SDValue RC =
3280 CurDAG->getTargetConstant(Val: Hexagon::HvxWRRegClassID, DL, VT: MVT::i32);
3281 SDValue SubRegL = CurDAG->getTargetConstant(Val: Hexagon::vsub_lo, DL, VT: MVT::i32);
3282 SDValue SubRegH = CurDAG->getTargetConstant(Val: Hexagon::vsub_hi, DL, VT: MVT::i32);
3283 const SDValue Ops[] = {RC, SDValue(HiAddNode, 0), SubRegH,
3284 SDValue(LoAddNode, 0), SubRegL};
3285 SDNode *RS = CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL,
3286 VT: MVT::v64i32, Ops);
3287
3288 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_hf_qf32, dl: DL,
3289 VT: ResTy, Op1: SDValue(RS, 0));
3290 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3291 CurDAG->RemoveDeadNode(N);
3292 return;
3293 }
3294 // v1:0.sf += vmpy(v2.hf,v3.hf) is translated to
3295 // v3:2.qf32 = vmpy(v2.hf,v3.hf);
3296 // v0.qf32 = vadd(v2.qf32,v0.sf); v1.qf32 = vadd(v3.qf32,v1.sf);
3297 // v0.sf = v0.qf32; v1.sf = v1.qf32
3298 case Intrinsic::hexagon_V6_vmpy_sf_hf_acc_128B: {
3299 SDNode *MpyNode =
3300 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: ResTy,
3301 Op1: N->getOperand(Num: 2), Op2: N->getOperand(Num: 3));
3302
3303 SDValue LoRegMpy = CurDAG->getTargetExtractSubreg(
3304 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3305 SDValue HiRegMpy = CurDAG->getTargetExtractSubreg(
3306 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3307
3308 SDValue LoRegDest = CurDAG->getTargetExtractSubreg(
3309 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: N->getOperand(Num: 1));
3310 SDValue HiRegDest = CurDAG->getTargetExtractSubreg(
3311 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: N->getOperand(Num: 1));
3312
3313 SDNode *LoAddNode = CurDAG->getMachineNode(
3314 Opcode: Hexagon::V6_vadd_qf32_mix, dl: DL, VT: MVT::v32i32, Op1: LoRegMpy, Op2: LoRegDest);
3315 SDNode *HiAddNode = CurDAG->getMachineNode(
3316 Opcode: Hexagon::V6_vadd_qf32_mix, dl: DL, VT: MVT::v32i32, Op1: HiRegMpy, Op2: HiRegDest);
3317
3318 SDNode *ConvLoNode = CurDAG->getMachineNode(
3319 Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL, VT: MVT::v32i32, Op1: SDValue(LoAddNode, 0));
3320 SDNode *ConvHiNode = CurDAG->getMachineNode(
3321 Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL, VT: MVT::v32i32, Op1: SDValue(HiAddNode, 0));
3322
3323 SDValue RC =
3324 CurDAG->getTargetConstant(Val: Hexagon::HvxWRRegClassID, DL, VT: MVT::i32);
3325 SDValue SubRegL = CurDAG->getTargetConstant(Val: Hexagon::vsub_lo, DL, VT: MVT::i32);
3326 SDValue SubRegH = CurDAG->getTargetConstant(Val: Hexagon::vsub_hi, DL, VT: MVT::i32);
3327 const SDValue Ops[] = {RC, SDValue(ConvHiNode, 0), SubRegH,
3328 SDValue(ConvLoNode, 0), SubRegL};
3329 SDNode *RS = CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL,
3330 VT: MVT::v64i32, Ops);
3331
3332 ReplaceUses(F: SDValue(N, 0), T: SDValue(RS, 0));
3333 CurDAG->RemoveDeadNode(N);
3334 return;
3335 }
3336 // v0.hf = vfmin(v0.hf,v1.hf) is translated to v0.hf = vmin(v0.hf,v1.hf).
3337 case Intrinsic::hexagon_V6_vfmin_hf_128B: {
3338 SDNode *MinNode = CurDAG->getMachineNode(
3339 Opcode: Hexagon::V6_vmin_hf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3340 ReplaceUses(F: SDValue(N, 0), T: SDValue(MinNode, 0));
3341 CurDAG->RemoveDeadNode(N);
3342 return;
3343 }
3344 // v0.sf = vfmin(v0.sf,v1.sf) is translated to v0.sf = vmin(v0.sf,v1.sf).
3345 case Intrinsic::hexagon_V6_vfmin_sf_128B: {
3346 SDNode *MinNode = CurDAG->getMachineNode(
3347 Opcode: Hexagon::V6_vmin_sf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3348 ReplaceUses(F: SDValue(N, 0), T: SDValue(MinNode, 0));
3349 CurDAG->RemoveDeadNode(N);
3350 return;
3351 }
3352 // v0.hf = vfmax(v0.hf,v1.hf) is translated to v0.hf = vmax(v0.hf,v1.hf).
3353 case Intrinsic::hexagon_V6_vfmax_hf_128B: {
3354 SDNode *MaxNode = CurDAG->getMachineNode(
3355 Opcode: Hexagon::V6_vmax_hf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3356 ReplaceUses(F: SDValue(N, 0), T: SDValue(MaxNode, 0));
3357 CurDAG->RemoveDeadNode(N);
3358 return;
3359 }
3360 // v0.sf = vfmax(v0.sf,v1.sf) is translated to v0.sf = vmax(v0.sf,v1.sf).
3361 case Intrinsic::hexagon_V6_vfmax_sf_128B: {
3362 SDNode *MaxNode = CurDAG->getMachineNode(
3363 Opcode: Hexagon::V6_vmax_sf, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1), Op2: N->getOperand(Num: 2));
3364 ReplaceUses(F: SDValue(N, 0), T: SDValue(MaxNode, 0));
3365 CurDAG->RemoveDeadNode(N);
3366 return;
3367 }
3368 // v0.hf = vabs(v0.hf) is translated to
3369 // r2 = #32767 ; v1.h = vsplat(r2) ; v0 = vand(v0,v1)
3370 // Reset the sign bit by splatting 0x7FFF and does a vector and.
3371 case Intrinsic::hexagon_V6_vabs_hf_128B: {
3372 SDValue Const = CurDAG->getTargetConstant(Val: 0x7FFFU, DL, VT: MVT::i32);
3373 SDNode *SplatPseudoNode =
3374 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatih, dl: DL, VT: ResTy, Op1: Const);
3375 SDNode *AndNode =
3376 CurDAG->getMachineNode(Opcode: Hexagon::V6_vand, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1),
3377 Op2: SDValue(SplatPseudoNode, 0));
3378 ReplaceUses(F: SDValue(N, 0), T: SDValue(AndNode, 0));
3379 CurDAG->RemoveDeadNode(N);
3380 return;
3381 }
3382 // v0.sf = vabs(v0.sf) is translated to
3383 // r2 = ##2147483647 ; v1 = vsplat(r2) ; v0 = vand(v0,v1)
3384 // Reset the sign bit by splatting 0x7FFF FFFF and does a vector and.
3385 case Intrinsic::hexagon_V6_vabs_sf_128B: {
3386 SDValue Const = CurDAG->getTargetConstant(Val: 0x7FFFFFFFU, DL, VT: MVT::i32);
3387 SDNode *SplatPseudoNode =
3388 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatiw, dl: DL, VT: ResTy, Op1: Const);
3389 SDNode *AndNode =
3390 CurDAG->getMachineNode(Opcode: Hexagon::V6_vand, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1),
3391 Op2: SDValue(SplatPseudoNode, 0));
3392 ReplaceUses(F: SDValue(N, 0), T: SDValue(AndNode, 0));
3393 CurDAG->RemoveDeadNode(N);
3394 return;
3395 }
3396 // v0.hf = vfneg(v0.hf) is translated to
3397 // r2 = #32768 ; v1.h = vsplat(r2) ; v0 = vxor(v0,v1)
3398 // Flip the sign bit by splatting 0x8000 and does a vector xor.
3399 case Intrinsic::hexagon_V6_vfneg_hf_128B: {
3400 SDValue Const = CurDAG->getTargetConstant(Val: 0x8000U, DL, VT: MVT::i32);
3401 SDNode *SplatPseudoNode =
3402 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatih, dl: DL, VT: ResTy, Op1: Const);
3403 SDNode *XorNode =
3404 CurDAG->getMachineNode(Opcode: Hexagon::V6_vxor, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1),
3405 Op2: SDValue(SplatPseudoNode, 0));
3406 ReplaceUses(F: SDValue(N, 0), T: SDValue(XorNode, 0));
3407 CurDAG->RemoveDeadNode(N);
3408 return;
3409 }
3410 // v0.sf = vfneg(v0.sf) is translated to
3411 // r2 = ##-2147483648 ; v1 = vsplat(r2) ; v0 = vxor(v0,v1)
3412 // Flip the sign bit by splatting 0x8000 0000 and does a vector xor.
3413 case Intrinsic::hexagon_V6_vfneg_sf_128B: {
3414 SDValue Const = CurDAG->getTargetConstant(Val: 0x80000000U, DL, VT: MVT::i32);
3415 SDNode *SplatPseudoNode =
3416 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatiw, dl: DL, VT: ResTy, Op1: Const);
3417 SDNode *XorNode =
3418 CurDAG->getMachineNode(Opcode: Hexagon::V6_vxor, dl: DL, VT: ResTy, Op1: N->getOperand(Num: 1),
3419 Op2: SDValue(SplatPseudoNode, 0));
3420 ReplaceUses(F: SDValue(N, 0), T: SDValue(XorNode, 0));
3421 CurDAG->RemoveDeadNode(N);
3422 return;
3423 }
3424 // v0.h = vcvt(v0.hf) is translated to v0.hf = v0.h.
3425 case Intrinsic::hexagon_V6_vcvt_hf_h_128B: {
3426 SDNode *ConvNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_hf_h, dl: DL, VT: ResTy,
3427 Op1: N->getOperand(Num: 1));
3428 ReplaceUses(F: SDValue(N, 0), T: SDValue(ConvNode, 0));
3429 CurDAG->RemoveDeadNode(N);
3430 return;
3431 }
3432 // v1:0.sf = vcvt(v0.hf) is translated to
3433 // r2 = #15360; v1.h = vsplat(r2); v3:2.qf32 = vmpy(v0.hf,v1.hf);
3434 // v0.sf = v2.qf32; v1.sf = v3.qf32
3435 // Do a widening multiply with 1.0f to convert the hf to sf.
3436 case Intrinsic::hexagon_V6_vcvt_sf_hf_128B: {
3437 SDValue Const = CurDAG->getTargetConstant(Val: 0x3C00U, DL, VT: MVT::i32);
3438 SDNode *SplatPseudoNode =
3439 CurDAG->getMachineNode(Opcode: Hexagon::PS_vsplatih, dl: DL, VT: ResTy, Op1: Const);
3440 SDNode *MpyNode =
3441 CurDAG->getMachineNode(Opcode: Hexagon::V6_vmpy_qf32_hf, dl: DL, VT: ResTy,
3442 Op1: N->getOperand(Num: 1), Op2: SDValue(SplatPseudoNode, 0));
3443
3444 SDValue LoReg = CurDAG->getTargetExtractSubreg(
3445 SRIdx: Hexagon::vsub_lo, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3446 SDValue HiReg = CurDAG->getTargetExtractSubreg(
3447 SRIdx: Hexagon::vsub_hi, DL, VT: MVT::v32i32, Operand: SDValue(MpyNode, 0));
3448 SDNode *ConvLoNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL,
3449 VT: MVT::v32i32, Op1: LoReg);
3450 SDNode *ConvHiNode = CurDAG->getMachineNode(Opcode: Hexagon::V6_vconv_sf_qf32, dl: DL,
3451 VT: MVT::v32i32, Op1: HiReg);
3452
3453 SDValue RC =
3454 CurDAG->getTargetConstant(Val: Hexagon::HvxWRRegClassID, DL, VT: MVT::i32);
3455 SDValue SubRegL = CurDAG->getTargetConstant(Val: Hexagon::vsub_lo, DL, VT: MVT::i32);
3456 SDValue SubRegH = CurDAG->getTargetConstant(Val: Hexagon::vsub_hi, DL, VT: MVT::i32);
3457 const SDValue Ops[] = {RC, SDValue(ConvHiNode, 0), SubRegH,
3458 SDValue(ConvLoNode, 0), SubRegL};
3459 SDNode *RS = CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL,
3460 VT: MVT::v64i32, Ops);
3461
3462 ReplaceUses(F: SDValue(N, 0), T: SDValue(RS, 0));
3463 CurDAG->RemoveDeadNode(N);
3464 return;
3465 }
3466
3467 default:
3468 llvm_unreachable("Unexpected HVX IEEE intrinsic: no QFloat translation");
3469 return;
3470 }
3471
3472 return;
3473}
3474