1//===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "HexagonISelLowering.h"
10#include "HexagonRegisterInfo.h"
11#include "HexagonSubtarget.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/MemoryLocation.h"
15#include "llvm/CodeGen/MachineBasicBlock.h"
16#include "llvm/CodeGen/MachineFunction.h"
17#include "llvm/CodeGen/MachineInstr.h"
18#include "llvm/CodeGen/MachineOperand.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/TargetInstrInfo.h"
21#include "llvm/IR/IntrinsicsHexagon.h"
22#include "llvm/Support/CommandLine.h"
23
24#include <algorithm>
25#include <string>
26#include <utility>
27
28using namespace llvm;
29
30static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
31 cl::Hidden, cl::init(Val: 16),
32 cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
33
34static cl::opt<bool>
35 EnableFpFastConvert("hexagon-fp-fast-convert", cl::Hidden, cl::init(Val: false),
36 cl::desc("Enable FP fast conversion routine."));
37
38static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
39static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
40static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
41static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
42
43static const unsigned MaxExpandMLA = 8;
44
45static std::tuple<unsigned, unsigned, unsigned> getIEEEProperties(MVT Ty) {
46 // For a float scalar type, return (exp-bits, exp-bias, fraction-bits)
47 MVT ElemTy = Ty.getScalarType();
48 switch (ElemTy.SimpleTy) {
49 case MVT::f16:
50 return std::make_tuple(args: 5, args: 15, args: 10);
51 case MVT::f32:
52 return std::make_tuple(args: 8, args: 127, args: 23);
53 case MVT::f64:
54 return std::make_tuple(args: 11, args: 1023, args: 52);
55 default:
56 break;
57 }
58 llvm_unreachable(("Unexpected type: " + EVT(ElemTy).getEVTString()).c_str());
59}
60
61void
62HexagonTargetLowering::initializeHVXLowering() {
63 if (Subtarget.useHVX64BOps()) {
64 addRegisterClass(VT: MVT::v64i8, RC: &Hexagon::HvxVRRegClass);
65 addRegisterClass(VT: MVT::v32i16, RC: &Hexagon::HvxVRRegClass);
66 addRegisterClass(VT: MVT::v16i32, RC: &Hexagon::HvxVRRegClass);
67 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxWRRegClass);
68 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxWRRegClass);
69 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxWRRegClass);
70 // These "short" boolean vector types should be legal because
71 // they will appear as results of vector compares. If they were
72 // not legal, type legalization would try to make them legal
73 // and that would require using operations that do not use or
74 // produce such types. That, in turn, would imply using custom
75 // nodes, which would be unoptimizable by the DAG combiner.
76 // The idea is to rely on target-independent operations as much
77 // as possible.
78 addRegisterClass(VT: MVT::v16i1, RC: &Hexagon::HvxQRRegClass);
79 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
80 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
81 } else if (Subtarget.useHVX128BOps()) {
82 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxVRRegClass);
83 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxVRRegClass);
84 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxVRRegClass);
85 addRegisterClass(VT: MVT::v256i8, RC: &Hexagon::HvxWRRegClass);
86 addRegisterClass(VT: MVT::v128i16, RC: &Hexagon::HvxWRRegClass);
87 addRegisterClass(VT: MVT::v64i32, RC: &Hexagon::HvxWRRegClass);
88 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
89 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
90 addRegisterClass(VT: MVT::v128i1, RC: &Hexagon::HvxQRRegClass);
91 if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) {
92 addRegisterClass(VT: MVT::v32f32, RC: &Hexagon::HvxVRRegClass);
93 addRegisterClass(VT: MVT::v64f16, RC: &Hexagon::HvxVRRegClass);
94 addRegisterClass(VT: MVT::v64f32, RC: &Hexagon::HvxWRRegClass);
95 addRegisterClass(VT: MVT::v128f16, RC: &Hexagon::HvxWRRegClass);
96 }
97 if (Subtarget.useHVXV81Ops()) {
98 addRegisterClass(VT: MVT::v64bf16, RC: &Hexagon::HvxVRRegClass);
99 addRegisterClass(VT: MVT::v128bf16, RC: &Hexagon::HvxWRRegClass);
100 }
101 }
102
103 // Set up operation actions.
104
105 bool Use64b = Subtarget.useHVX64BOps();
106 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
107 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
108 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
109 MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32;
110 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
111
112 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
113 setOperationAction(Op: Opc, VT: FromTy, Action: Promote);
114 AddPromotedToType(Opc, OrigVT: FromTy, DestVT: ToTy);
115 };
116
117 // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
118 // Note: v16i1 -> i16 is handled in type legalization instead of op
119 // legalization.
120 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
121 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
122 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
123 setOperationAction(Op: ISD::BITCAST, VT: MVT::v16i1, Action: Custom);
124 setOperationAction(Op: ISD::BITCAST, VT: MVT::v128i1, Action: Custom);
125 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
126 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteV, Action: Legal);
127 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteW, Action: Legal);
128 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
129
130 if (Subtarget.useHVX128BOps()) {
131 setOperationAction(Op: ISD::BITCAST, VT: MVT::v32i1, Action: Custom);
132 setOperationAction(Op: ISD::BITCAST, VT: MVT::v64i1, Action: Custom);
133 setOperationAction(Op: ISD::STORE, VT: MVT::v32i1, Action: Custom);
134 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i1, Action: Custom);
135 setOperationAction(Op: ISD::STORE, VT: MVT::v64i1, Action: Custom);
136 setOperationAction(Op: ISD::LOAD, VT: MVT::v64i1, Action: Custom);
137 setOperationAction(Op: ISD::STORE, VT: MVT::v128i1, Action: Custom);
138 setOperationAction(Op: ISD::LOAD, VT: MVT::v128i1, Action: Custom);
139 }
140 if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() &&
141 Subtarget.useHVXFloatingPoint()) {
142
143 static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 };
144 static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 };
145
146 for (MVT T : FloatV) {
147 setOperationAction(Op: ISD::FADD, VT: T, Action: Legal);
148 setOperationAction(Op: ISD::FSUB, VT: T, Action: Legal);
149 setOperationAction(Op: ISD::FMUL, VT: T, Action: Legal);
150 setOperationAction(Op: ISD::FMINIMUMNUM, VT: T, Action: Legal);
151 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: T, Action: Legal);
152 setOperationAction(Op: ISD::FMINIMUM, VT: T, Action: Legal);
153 setOperationAction(Op: ISD::FMAXIMUM, VT: T, Action: Legal);
154 setOperationAction(Op: ISD::FMINNUM, VT: T, Action: Custom);
155 setOperationAction(Op: ISD::FMAXNUM, VT: T, Action: Custom);
156 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: T, Action: Custom);
157 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: T, Action: Custom);
158 setOperationAction(Op: ISD::VECREDUCE_FMINIMUM, VT: T, Action: Custom);
159 setOperationAction(Op: ISD::VECREDUCE_FMAXIMUM, VT: T, Action: Custom);
160
161 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
162 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
163
164 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
165 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
166
167 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
168 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
169 // Custom-lower BUILD_VECTOR. The standard (target-independent)
170 // handling of it would convert it to a load, which is not always
171 // the optimal choice.
172 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
173 }
174
175
176 // BUILD_VECTOR with f16 operands cannot be promoted without
177 // promoting the result, so lower the node to vsplat or constant pool
178 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
179 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::f16, Action: Custom);
180
181 // Vector shuffle is always promoted to ByteV and a bitcast to f16 is
182 // generated.
183 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128f16, ByteW);
184 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f16, ByteV);
185 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f32, ByteW);
186 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v32f32, ByteV);
187
188 // For HVX <v81 there is no hardware float-equality instruction; only
189 // float-GT (V6_vgtsf/V6_vgthf) is available. The integer-equality
190 // fallback (V6_veqw/V6_veqh) silently treats NaN as equal to itself
191 // because the bit patterns match. Mark SETCC as Custom for the
192 // single-vector float types so we can synthesise the correct
193 // ordered-equal predicate in LowerHvxFpSetoeq.
194 if (!Subtarget.useHVXV81Ops())
195 for (MVT T : FloatV)
196 setOperationAction(Op: ISD::SETCC, VT: T, Action: Custom);
197
198 if (Subtarget.useHVXV81Ops()) {
199 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128bf16, ByteW);
200 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64bf16, ByteV);
201 setPromoteTo(ISD::SETCC, MVT::v64bf16, MVT::v64f32);
202 setPromoteTo(ISD::FADD, MVT::v64bf16, MVT::v64f32);
203 setPromoteTo(ISD::FSUB, MVT::v64bf16, MVT::v64f32);
204 setPromoteTo(ISD::FMUL, MVT::v64bf16, MVT::v64f32);
205 setPromoteTo(ISD::FMINNUM, MVT::v64bf16, MVT::v64f32);
206 setPromoteTo(ISD::FMAXNUM, MVT::v64bf16, MVT::v64f32);
207
208 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::v64bf16, Action: Legal);
209 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
210 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
211
212 setOperationAction(Op: ISD::LOAD, VT: MVT::v128bf16, Action: Custom);
213 setOperationAction(Op: ISD::STORE, VT: MVT::v128bf16, Action: Custom);
214
215 setOperationAction(Op: ISD::MLOAD, VT: MVT::v64bf16, Action: Custom);
216 setOperationAction(Op: ISD::MSTORE, VT: MVT::v64bf16, Action: Custom);
217 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v64bf16, Action: Custom);
218 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v64bf16, Action: Custom);
219
220 setOperationAction(Op: ISD::MLOAD, VT: MVT::v128bf16, Action: Custom);
221 setOperationAction(Op: ISD::MSTORE, VT: MVT::v128bf16, Action: Custom);
222 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v128bf16, Action: Custom);
223 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v128bf16, Action: Custom);
224
225 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::bf16, Action: Custom);
226 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::bf16, Action: Custom);
227 }
228
229 for (MVT P : FloatW) {
230 setOperationAction(Op: ISD::LOAD, VT: P, Action: Custom);
231 setOperationAction(Op: ISD::STORE, VT: P, Action: Custom);
232 setOperationAction(Op: ISD::FADD, VT: P, Action: Custom);
233 setOperationAction(Op: ISD::FSUB, VT: P, Action: Custom);
234 setOperationAction(Op: ISD::FMUL, VT: P, Action: Custom);
235 setOperationAction(Op: ISD::FMINIMUMNUM, VT: P, Action: Custom);
236 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: P, Action: Custom);
237 setOperationAction(Op: ISD::FMINIMUM, VT: P, Action: Custom);
238 setOperationAction(Op: ISD::FMAXIMUM, VT: P, Action: Custom);
239 setOperationAction(Op: ISD::FMINNUM, VT: P, Action: Custom);
240 setOperationAction(Op: ISD::FMAXNUM, VT: P, Action: Custom);
241 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: P, Action: Custom);
242 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: P, Action: Custom);
243 setOperationAction(Op: ISD::VECREDUCE_FMINIMUM, VT: P, Action: Custom);
244 setOperationAction(Op: ISD::VECREDUCE_FMAXIMUM, VT: P, Action: Custom);
245 setOperationAction(Op: ISD::SETCC, VT: P, Action: Custom);
246 setOperationAction(Op: ISD::VSELECT, VT: P, Action: Custom);
247
248 // Custom-lower BUILD_VECTOR. The standard (target-independent)
249 // handling of it would convert it to a load, which is not always
250 // the optimal choice.
251 setOperationAction(Op: ISD::BUILD_VECTOR, VT: P, Action: Custom);
252 // Make concat-vectors custom to handle concats of more than 2 vectors.
253 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: P, Action: Custom);
254
255 setOperationAction(Op: ISD::MLOAD, VT: P, Action: Custom);
256 setOperationAction(Op: ISD::MSTORE, VT: P, Action: Custom);
257 }
258
259 if (Subtarget.useHVXQFloatOps()) {
260 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Custom);
261 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
262 } else if (Subtarget.useHVXIEEEFPOps()) {
263 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Legal);
264 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
265 }
266 }
267
268 for (MVT T : LegalV) {
269 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
270 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
271
272 setOperationAction(Op: ISD::ABS, VT: T, Action: Legal);
273 setOperationAction(Op: ISD::AND, VT: T, Action: Legal);
274 setOperationAction(Op: ISD::OR, VT: T, Action: Legal);
275 setOperationAction(Op: ISD::XOR, VT: T, Action: Legal);
276 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
277 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
278 setOperationAction(Op: ISD::MUL, VT: T, Action: Legal);
279 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Legal);
280 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Legal);
281 setOperationAction(Op: ISD::SELECT, VT: T, Action: Legal);
282 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
283 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
284 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
285 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
286 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
287 if (T != ByteV) {
288 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
289 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
290 setOperationAction(Op: ISD::BSWAP, VT: T, Action: Legal);
291 }
292
293 setOperationAction(Op: ISD::SMIN, VT: T, Action: Legal);
294 setOperationAction(Op: ISD::SMAX, VT: T, Action: Legal);
295 if (T.getScalarType() != MVT::i32) {
296 setOperationAction(Op: ISD::UMIN, VT: T, Action: Legal);
297 setOperationAction(Op: ISD::UMAX, VT: T, Action: Legal);
298 }
299
300 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
301 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
302 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
303 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
304 if (T.getScalarType() != MVT::i32) {
305 setOperationAction(Op: ISD::MULHS, VT: T, Action: Legal);
306 setOperationAction(Op: ISD::MULHU, VT: T, Action: Legal);
307 }
308
309 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
310 // Make concat-vectors custom to handle concats of more than 2 vectors.
311 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
312 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
313 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: T, Action: Custom);
314 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
315 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: T, Action: Custom);
316 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
317 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
318 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
319 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
320 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
321 if (T != ByteV) {
322 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
323 // HVX only has shifts of words and halfwords.
324 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
325 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
326 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
327
328 // Promote all shuffles to operate on vectors of bytes.
329 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
330 }
331
332 if (Subtarget.useHVXFloatingPoint()) {
333 // Same action for both QFloat and IEEE.
334 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
335 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
336 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
337 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
338 }
339
340 setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Expand);
341 setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Expand);
342 setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Expand);
343 setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Expand);
344 setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Expand);
345 setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Expand);
346 setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Expand);
347 }
348
349 for (MVT T : LegalW) {
350 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
351 // independent) handling of it would convert it to a load, which is
352 // not always the optimal choice.
353 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
354 // Make concat-vectors custom to handle concats of more than 2 vectors.
355 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
356
357 // Custom-lower these operations for pairs. Expand them into a concat
358 // of the corresponding operations on individual vectors.
359 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
360 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
361 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
362 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Custom);
363 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
364 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
365 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
366 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Custom);
367
368 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
369 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
370 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
371 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
372 setOperationAction(Op: ISD::ABS, VT: T, Action: Custom);
373 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Custom);
374 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
375 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Custom);
376
377 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
378 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
379 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
380 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
381 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
382 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
383 setOperationAction(Op: ISD::MUL, VT: T, Action: Custom);
384 setOperationAction(Op: ISD::MULHS, VT: T, Action: Custom);
385 setOperationAction(Op: ISD::MULHU, VT: T, Action: Custom);
386 setOperationAction(Op: ISD::AND, VT: T, Action: Custom);
387 setOperationAction(Op: ISD::OR, VT: T, Action: Custom);
388 setOperationAction(Op: ISD::XOR, VT: T, Action: Custom);
389 setOperationAction(Op: ISD::SETCC, VT: T, Action: Custom);
390 setOperationAction(Op: ISD::VSELECT, VT: T, Action: Custom);
391 if (T != ByteW) {
392 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
393 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
394 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
395
396 // Promote all shuffles to operate on vectors of bytes.
397 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
398 }
399 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
400 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
401
402 setOperationAction(Op: ISD::SMIN, VT: T, Action: Custom);
403 setOperationAction(Op: ISD::SMAX, VT: T, Action: Custom);
404 if (T.getScalarType() != MVT::i32) {
405 setOperationAction(Op: ISD::UMIN, VT: T, Action: Custom);
406 setOperationAction(Op: ISD::UMAX, VT: T, Action: Custom);
407 }
408
409 if (Subtarget.useHVXFloatingPoint()) {
410 // Same action for both QFloat and IEEE.
411 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
412 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
413 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
414 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
415 }
416 }
417
418 // Legalize all of these to HexagonISD::[SU]MUL_LOHI.
419 setOperationAction(Op: ISD::MULHS, VT: WordV, Action: Custom); // -> _LOHI
420 setOperationAction(Op: ISD::MULHU, VT: WordV, Action: Custom); // -> _LOHI
421 setOperationAction(Op: ISD::SMUL_LOHI, VT: WordV, Action: Custom);
422 setOperationAction(Op: ISD::UMUL_LOHI, VT: WordV, Action: Custom);
423
424 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v64f16, Action: Expand);
425 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v64f16, Action: Expand);
426 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v64f16, Action: Expand);
427 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v64f16, Action: Expand);
428 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v64f16, Action: Expand);
429 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v64f16, Action: Expand);
430 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v64f16, Action: Expand);
431 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v64f16, Action: Expand);
432 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v64f16, Action: Expand);
433 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v64f16, Action: Expand);
434 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v64f16, Action: Expand);
435 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v64f16, Action: Expand);
436 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v64f16, Action: Expand);
437 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v64f16, Action: Expand);
438
439 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v32f32, Action: Expand);
440 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v32f32, Action: Expand);
441 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v32f32, Action: Expand);
442 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v32f32, Action: Expand);
443 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v32f32, Action: Expand);
444 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v32f32, Action: Expand);
445 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v32f32, Action: Expand);
446 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v32f32, Action: Expand);
447 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v32f32, Action: Expand);
448 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v32f32, Action: Expand);
449 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v32f32, Action: Expand);
450 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v32f32, Action: Expand);
451 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v32f32, Action: Expand);
452 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v32f32, Action: Expand);
453
454 // Boolean vectors.
455
456 for (MVT T : LegalW) {
457 // Boolean types for vector pairs will overlap with the boolean
458 // types for single vectors, e.g.
459 // v64i8 -> v64i1 (single)
460 // v64i16 -> v64i1 (pair)
461 // Set these actions first, and allow the single actions to overwrite
462 // any duplicates.
463 MVT BoolW = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
464 setOperationAction(Op: ISD::SETCC, VT: BoolW, Action: Custom);
465 setOperationAction(Op: ISD::AND, VT: BoolW, Action: Custom);
466 setOperationAction(Op: ISD::OR, VT: BoolW, Action: Custom);
467 setOperationAction(Op: ISD::XOR, VT: BoolW, Action: Custom);
468 // Masked load/store takes a mask that may need splitting.
469 setOperationAction(Op: ISD::MLOAD, VT: BoolW, Action: Custom);
470 setOperationAction(Op: ISD::MSTORE, VT: BoolW, Action: Custom);
471 }
472
473 for (MVT T : LegalV) {
474 MVT BoolV = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
475 setOperationAction(Op: ISD::BUILD_VECTOR, VT: BoolV, Action: Custom);
476 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: BoolV, Action: Custom);
477 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: BoolV, Action: Custom);
478 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: BoolV, Action: Custom);
479 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: BoolV, Action: Custom);
480 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: BoolV, Action: Custom);
481 setOperationAction(Op: ISD::SELECT, VT: BoolV, Action: Custom);
482 setOperationAction(Op: ISD::AND, VT: BoolV, Action: Legal);
483 setOperationAction(Op: ISD::OR, VT: BoolV, Action: Legal);
484 setOperationAction(Op: ISD::XOR, VT: BoolV, Action: Legal);
485 }
486
487 if (Use64b) {
488 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
489 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
490 } else {
491 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
492 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
493 }
494
495 // Handle store widening for short vectors.
496 unsigned HwLen = Subtarget.getVectorLength();
497 for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
498 if (ElemTy == MVT::i1)
499 continue;
500 int ElemWidth = ElemTy.getFixedSizeInBits();
501 int MaxElems = (8*HwLen) / ElemWidth;
502 for (int N = 2; N < MaxElems; N *= 2) {
503 MVT VecTy = MVT::getVectorVT(VT: ElemTy, NumElements: N);
504 auto Action = getPreferredVectorAction(VT: VecTy);
505 if (Action == TargetLoweringBase::TypeWidenVector) {
506 setOperationAction(Op: ISD::LOAD, VT: VecTy, Action: Custom);
507 setOperationAction(Op: ISD::STORE, VT: VecTy, Action: Custom);
508 setOperationAction(Op: ISD::SETCC, VT: VecTy, Action: Custom);
509 setOperationAction(Op: ISD::TRUNCATE, VT: VecTy, Action: Custom);
510 setOperationAction(Op: ISD::ANY_EXTEND, VT: VecTy, Action: Custom);
511 setOperationAction(Op: ISD::SIGN_EXTEND, VT: VecTy, Action: Custom);
512 setOperationAction(Op: ISD::ZERO_EXTEND, VT: VecTy, Action: Custom);
513 if (Subtarget.useHVXFloatingPoint()) {
514 setOperationAction(Op: ISD::FP_TO_SINT, VT: VecTy, Action: Custom);
515 setOperationAction(Op: ISD::FP_TO_UINT, VT: VecTy, Action: Custom);
516 setOperationAction(Op: ISD::SINT_TO_FP, VT: VecTy, Action: Custom);
517 setOperationAction(Op: ISD::UINT_TO_FP, VT: VecTy, Action: Custom);
518 }
519
520 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: N);
521 if (!isTypeLegal(VT: BoolTy))
522 setOperationAction(Op: ISD::SETCC, VT: BoolTy, Action: Custom);
523 }
524 }
525 }
526
527 // Include cases which are not hander earlier
528 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v32i1, Action: Custom);
529 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v64i1, Action: Custom);
530 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v32i1, Action: Custom);
531
532 setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT});
533
534 setTargetDAGCombine({ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA,
535 ISD::PARTIAL_REDUCE_SUMLA});
536
537 // Partial MLA reductions.
538 {
539 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
540 ISD::PARTIAL_REDUCE_UMLA,
541 ISD::PARTIAL_REDUCE_SUMLA};
542
543 auto HvxType = [=](MVT ScalarT, unsigned Factor = 1) {
544 return MVT::getVectorVT(VT: ScalarT, NumElements: Subtarget.getVectorLength() * Factor *
545 8 / ScalarT.getSizeInBits());
546 };
547
548 // Tuple of (Acc element type, input element type, vector pair).
549 // The assumption is both the input and reduction result are of the same
550 // size so the reduction ratio is the same as the ratio of element type
551 // sizes. This may not hold for all available instructions.
552 typedef std::tuple<MVT, MVT, bool> ReductionSignature;
553
554 static const std::vector<ReductionSignature> NativeReductions = {
555 {MVT::i32, MVT::i8, false},
556 };
557
558 for (const auto &R : NativeReductions) {
559
560 MVT AccType = std::get<0>(t: R);
561 MVT InputType = std::get<1>(t: R);
562 unsigned Factor = std::get<2>(t: R) ? 2 : 1;
563
564 // The native size is legal.
565 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: HvxType(AccType), InputVT: HvxType(InputType),
566 Action: Legal);
567
568 // Allow custom partial MLA reductions on larger vectors than legally
569 // supported. These reduction must be declared as Custom (or Legal)
570 // for foldPartialReduceMLAMulOp() to fold the multiply by one pattern
571 // inserted when the partial reduction intrinsic is converted to
572 // PARTIAL_REDUCE_U/S/SUMLA. Otherwise, the Split action will apply
573 // on the original pattern, including the extensions and multiplies,
574 // which will make it impossible to match.
575 // There are two independent ways to extend the
576 // input size: 1. to concatenate the result - output vector is
577 // proportionally extended, 2) to reduce the result - the output vector
578 // size stays the same. We limit allowed combinations so that the total
579 // number of generated reduction instructions is limited by a constant
580 // number. This limit is arbitrary and can be revised. On one hand, it is
581 // convenient to have more choices; on the other hand, there is a
582 // diminishing benefit of very long sequences, which should probably be
583 // written as loops instead.
584 for (unsigned ConcatFactor = 1; ConcatFactor <= MaxExpandMLA;
585 ConcatFactor <<= 1)
586 for (unsigned ReductionFactor = 1; ReductionFactor <= MaxExpandMLA;
587 ReductionFactor <<= 1)
588 if (ConcatFactor * ReductionFactor != 1 &&
589 ConcatFactor * ReductionFactor <= MaxExpandMLA)
590 setPartialReduceMLAAction(
591 Opcodes: MLAOps, AccVT: HvxType(AccType, Factor * ConcatFactor),
592 InputVT: HvxType(InputType, Factor * ConcatFactor * ReductionFactor),
593 Action: Custom);
594 }
595 }
596}
597
598unsigned
599HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
600 // Early exit for invalid input types
601 if (!VecTy.isVector())
602 return ~0u;
603
604 MVT ElemTy = VecTy.getVectorElementType();
605 unsigned VecLen = VecTy.getVectorNumElements();
606 unsigned HwLen = Subtarget.getVectorLength();
607
608 // Split vectors of i1 that exceed byte vector length.
609 if (ElemTy == MVT::i1 && VecLen > HwLen)
610 return TargetLoweringBase::TypeSplitVector;
611
612 ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
613 // For shorter vectors of i1, widen them if any of the corresponding
614 // vectors of integers needs to be widened.
615 if (ElemTy == MVT::i1) {
616 for (MVT T : Tys) {
617 assert(T != MVT::i1);
618 auto A = getPreferredHvxVectorAction(VecTy: MVT::getVectorVT(VT: T, NumElements: VecLen));
619 if (A != ~0u)
620 return A;
621 }
622 return ~0u;
623 }
624
625 // If the size of VecTy is at least half of the vector length,
626 // widen the vector. Note: the threshold was not selected in
627 // any scientific way.
628 if (llvm::is_contained(Range&: Tys, Element: ElemTy)) {
629 unsigned VecWidth = VecTy.getSizeInBits();
630 unsigned HwWidth = 8*HwLen;
631 if (VecWidth > 2*HwWidth)
632 return TargetLoweringBase::TypeSplitVector;
633
634 bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0;
635 if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth)
636 return TargetLoweringBase::TypeWidenVector;
637 if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
638 return TargetLoweringBase::TypeWidenVector;
639 }
640
641 // Defer to default.
642 return ~0u;
643}
644
645unsigned
646HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const {
647 unsigned Opc = Op.getOpcode();
648 switch (Opc) {
649 case HexagonISD::SMUL_LOHI:
650 case HexagonISD::UMUL_LOHI:
651 case HexagonISD::USMUL_LOHI:
652 return TargetLoweringBase::Custom;
653 }
654 return TargetLoweringBase::Legal;
655}
656
657SDValue
658HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
659 const SDLoc &dl, SelectionDAG &DAG) const {
660 SmallVector<SDValue,4> IntOps;
661 IntOps.push_back(Elt: DAG.getConstant(Val: IntId, DL: dl, VT: MVT::i32));
662 append_range(C&: IntOps, R&: Ops);
663 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: ResTy, Ops: IntOps);
664}
665
666MVT
667HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
668 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
669
670 MVT ElemTy = Tys.first.getVectorElementType();
671 return MVT::getVectorVT(VT: ElemTy, NumElements: Tys.first.getVectorNumElements() +
672 Tys.second.getVectorNumElements());
673}
674
675HexagonTargetLowering::TypePair
676HexagonTargetLowering::typeSplit(MVT VecTy) const {
677 assert(VecTy.isVector());
678 unsigned NumElem = VecTy.getVectorNumElements();
679 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
680 MVT HalfTy = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: NumElem/2);
681 return { HalfTy, HalfTy };
682}
683
684MVT
685HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
686 MVT ElemTy = VecTy.getVectorElementType();
687 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() * Factor);
688 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
689}
690
691MVT
692HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
693 MVT ElemTy = VecTy.getVectorElementType();
694 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() / Factor);
695 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
696}
697
698SDValue
699HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
700 SelectionDAG &DAG) const {
701 if (ty(Op: Vec).getVectorElementType() == ElemTy)
702 return Vec;
703 MVT CastTy = tyVector(Ty: Vec.getValueType().getSimpleVT(), ElemTy);
704 return DAG.getBitcast(VT: CastTy, V: Vec);
705}
706
707SDValue
708HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
709 SelectionDAG &DAG) const {
710 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: typeJoin(Tys: ty(Ops)),
711 N1: Ops.first, N2: Ops.second);
712}
713
714HexagonTargetLowering::VectorPair
715HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
716 SelectionDAG &DAG) const {
717 TypePair Tys = typeSplit(VecTy: ty(Op: Vec));
718 if (Vec.getOpcode() == HexagonISD::QCAT)
719 return VectorPair(Vec.getOperand(i: 0), Vec.getOperand(i: 1));
720 return DAG.SplitVector(N: Vec, DL: dl, LoVT: Tys.first, HiVT: Tys.second);
721}
722
723bool
724HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
725 return Subtarget.isHVXVectorType(VecTy: Ty) &&
726 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
727}
728
729bool
730HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
731 return Subtarget.isHVXVectorType(VecTy: Ty) &&
732 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
733}
734
735bool
736HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
737 return Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true) &&
738 Ty.getVectorElementType() == MVT::i1;
739}
740
741bool HexagonTargetLowering::allowsHvxMemoryAccess(
742 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
743 // Bool vectors are excluded by default, but make it explicit to
744 // emphasize that bool vectors cannot be loaded or stored.
745 // Also, disallow double vector stores (to prevent unnecessary
746 // store widening in DAG combiner).
747 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
748 return false;
749 if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
750 return false;
751 if (Fast)
752 *Fast = 1;
753 return true;
754}
755
756bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
757 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
758 if (!Subtarget.isHVXVectorType(VecTy))
759 return false;
760 // XXX Should this be false? vmemu are a bit slower than vmem.
761 if (Fast)
762 *Fast = 1;
763 return true;
764}
765
766void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection(
767 MachineInstr &MI, SDNode *Node) const {
768 unsigned Opc = MI.getOpcode();
769 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
770 MachineBasicBlock &MB = *MI.getParent();
771 MachineFunction &MF = *MB.getParent();
772 MachineRegisterInfo &MRI = MF.getRegInfo();
773 DebugLoc DL = MI.getDebugLoc();
774 auto At = MI.getIterator();
775
776 switch (Opc) {
777 case Hexagon::PS_vsplatib:
778 if (Subtarget.useHVXV62Ops()) {
779 // SplatV = A2_tfrsi #imm
780 // OutV = V6_lvsplatb SplatV
781 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
782 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
783 .add(MO: MI.getOperand(i: 1));
784 Register OutV = MI.getOperand(i: 0).getReg();
785 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
786 .addReg(RegNo: SplatV);
787 } else {
788 // SplatV = A2_tfrsi #imm:#imm:#imm:#imm
789 // OutV = V6_lvsplatw SplatV
790 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
791 const MachineOperand &InpOp = MI.getOperand(i: 1);
792 assert(InpOp.isImm());
793 uint32_t V = InpOp.getImm() & 0xFF;
794 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
795 .addImm(Val: V << 24 | V << 16 | V << 8 | V);
796 Register OutV = MI.getOperand(i: 0).getReg();
797 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
798 }
799 MB.erase(I: At);
800 break;
801 case Hexagon::PS_vsplatrb:
802 if (Subtarget.useHVXV62Ops()) {
803 // OutV = V6_lvsplatb Inp
804 Register OutV = MI.getOperand(i: 0).getReg();
805 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
806 .add(MO: MI.getOperand(i: 1));
807 } else {
808 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
809 const MachineOperand &InpOp = MI.getOperand(i: 1);
810 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::S2_vsplatrb), DestReg: SplatV)
811 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
812 Register OutV = MI.getOperand(i: 0).getReg();
813 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV)
814 .addReg(RegNo: SplatV);
815 }
816 MB.erase(I: At);
817 break;
818 case Hexagon::PS_vsplatih:
819 if (Subtarget.useHVXV62Ops()) {
820 // SplatV = A2_tfrsi #imm
821 // OutV = V6_lvsplath SplatV
822 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
823 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
824 .add(MO: MI.getOperand(i: 1));
825 Register OutV = MI.getOperand(i: 0).getReg();
826 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
827 .addReg(RegNo: SplatV);
828 } else {
829 // SplatV = A2_tfrsi #imm:#imm
830 // OutV = V6_lvsplatw SplatV
831 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
832 const MachineOperand &InpOp = MI.getOperand(i: 1);
833 assert(InpOp.isImm());
834 uint32_t V = InpOp.getImm() & 0xFFFF;
835 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
836 .addImm(Val: V << 16 | V);
837 Register OutV = MI.getOperand(i: 0).getReg();
838 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
839 }
840 MB.erase(I: At);
841 break;
842 case Hexagon::PS_vsplatrh:
843 if (Subtarget.useHVXV62Ops()) {
844 // OutV = V6_lvsplath Inp
845 Register OutV = MI.getOperand(i: 0).getReg();
846 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
847 .add(MO: MI.getOperand(i: 1));
848 } else {
849 // SplatV = A2_combine_ll Inp, Inp
850 // OutV = V6_lvsplatw SplatV
851 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
852 const MachineOperand &InpOp = MI.getOperand(i: 1);
853 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_combine_ll), DestReg: SplatV)
854 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg())
855 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
856 Register OutV = MI.getOperand(i: 0).getReg();
857 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
858 }
859 MB.erase(I: At);
860 break;
861 case Hexagon::PS_vsplatiw:
862 case Hexagon::PS_vsplatrw:
863 if (Opc == Hexagon::PS_vsplatiw) {
864 // SplatV = A2_tfrsi #imm
865 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
866 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
867 .add(MO: MI.getOperand(i: 1));
868 MI.getOperand(i: 1).ChangeToRegister(Reg: SplatV, isDef: false);
869 }
870 // OutV = V6_lvsplatw SplatV/Inp
871 MI.setDesc(TII.get(Opcode: Hexagon::V6_lvsplatw));
872 break;
873 }
874}
875
876SDValue
877HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
878 SelectionDAG &DAG) const {
879 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
880 ElemIdx = DAG.getBitcast(VT: MVT::i32, V: ElemIdx);
881
882 unsigned ElemWidth = ElemTy.getSizeInBits();
883 if (ElemWidth == 8)
884 return ElemIdx;
885
886 unsigned L = Log2_32(Value: ElemWidth/8);
887 const SDLoc &dl(ElemIdx);
888 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32,
889 Ops: {ElemIdx, DAG.getConstant(Val: L, DL: dl, VT: MVT::i32)});
890}
891
892SDValue
893HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
894 SelectionDAG &DAG) const {
895 unsigned ElemWidth = ElemTy.getSizeInBits();
896 assert(ElemWidth >= 8 && ElemWidth <= 32);
897 if (ElemWidth == 32)
898 return Idx;
899
900 if (ty(Op: Idx) != MVT::i32)
901 Idx = DAG.getBitcast(VT: MVT::i32, V: Idx);
902 const SDLoc &dl(Idx);
903 SDValue Mask = DAG.getConstant(Val: 32/ElemWidth - 1, DL: dl, VT: MVT::i32);
904 SDValue SubIdx = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, Ops: {Idx, Mask});
905 return SubIdx;
906}
907
908SDValue
909HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
910 SDValue Op1, ArrayRef<int> Mask,
911 SelectionDAG &DAG) const {
912 MVT OpTy = ty(Op: Op0);
913 assert(OpTy == ty(Op1));
914
915 MVT ElemTy = OpTy.getVectorElementType();
916 if (ElemTy == MVT::i8)
917 return DAG.getVectorShuffle(VT: OpTy, dl, N1: Op0, N2: Op1, Mask);
918 assert(ElemTy.getSizeInBits() >= 8);
919
920 MVT ResTy = tyVector(Ty: OpTy, ElemTy: MVT::i8);
921 unsigned ElemSize = ElemTy.getSizeInBits() / 8;
922
923 SmallVector<int,128> ByteMask;
924 for (int M : Mask) {
925 if (M < 0) {
926 for (unsigned I = 0; I != ElemSize; ++I)
927 ByteMask.push_back(Elt: -1);
928 } else {
929 int NewM = M*ElemSize;
930 for (unsigned I = 0; I != ElemSize; ++I)
931 ByteMask.push_back(Elt: NewM+I);
932 }
933 }
934 assert(ResTy.getVectorNumElements() == ByteMask.size());
935 return DAG.getVectorShuffle(VT: ResTy, dl, N1: opCastElem(Vec: Op0, ElemTy: MVT::i8, DAG),
936 N2: opCastElem(Vec: Op1, ElemTy: MVT::i8, DAG), Mask: ByteMask);
937}
938
939SDValue
940HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
941 const SDLoc &dl, MVT VecTy,
942 SelectionDAG &DAG) const {
943 unsigned VecLen = Values.size();
944 MachineFunction &MF = DAG.getMachineFunction();
945 MVT ElemTy = VecTy.getVectorElementType();
946 unsigned ElemWidth = ElemTy.getSizeInBits();
947 unsigned HwLen = Subtarget.getVectorLength();
948
949 unsigned ElemSize = ElemWidth / 8;
950 assert(ElemSize*VecLen == HwLen);
951 SmallVector<SDValue,32> Words;
952
953 if (VecTy.getVectorElementType() != MVT::i32 &&
954 !(Subtarget.useHVXFloatingPoint() &&
955 VecTy.getVectorElementType() == MVT::f32)) {
956 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
957 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
958 MVT PartVT = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: OpsPerWord);
959 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
960 SDValue W = buildVector32(Elem: Values.slice(N: i, M: OpsPerWord), dl, VecTy: PartVT, DAG);
961 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: W));
962 }
963 } else {
964 for (SDValue V : Values)
965 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V));
966 }
967 auto isSplat = [] (ArrayRef<SDValue> Values, SDValue &SplatV) {
968 unsigned NumValues = Values.size();
969 assert(NumValues > 0);
970 bool IsUndef = true;
971 for (unsigned i = 0; i != NumValues; ++i) {
972 if (Values[i].isUndef())
973 continue;
974 IsUndef = false;
975 if (!SplatV.getNode())
976 SplatV = Values[i];
977 else if (SplatV != Values[i])
978 return false;
979 }
980 if (IsUndef)
981 SplatV = Values[0];
982 return true;
983 };
984
985 unsigned NumWords = Words.size();
986 SDValue SplatV;
987 bool IsSplat = isSplat(Words, SplatV);
988 if (IsSplat && isUndef(Op: SplatV))
989 return DAG.getUNDEF(VT: VecTy);
990 if (IsSplat) {
991 assert(SplatV.getNode());
992 if (isNullConstant(V: SplatV))
993 return getZero(dl, Ty: VecTy, DAG);
994 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
995 SDValue S = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordTy, Operand: SplatV);
996 return DAG.getBitcast(VT: VecTy, V: S);
997 }
998
999 // Delay recognizing constant vectors until here, so that we can generate
1000 // a vsplat.
1001 SmallVector<ConstantInt*, 128> Consts(VecLen);
1002 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
1003 if (AllConst) {
1004 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
1005 (Constant**)Consts.end());
1006 Constant *CV = ConstantVector::get(V: Tmp);
1007 Align Alignment(HwLen);
1008 SDValue CP = LowerConstantPool(
1009 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
1010 DAG);
1011 return DAG.getLoad(VT: VecTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
1012 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
1013 }
1014
1015 // A special case is a situation where the vector is built entirely from
1016 // elements extracted from another vector. This could be done via a shuffle
1017 // more efficiently, but typically, the size of the source vector will not
1018 // match the size of the vector being built (which precludes the use of a
1019 // shuffle directly).
1020 // This only handles a single source vector, and the vector being built
1021 // should be of a sub-vector type of the source vector type.
1022 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
1023 SmallVectorImpl<int> &SrcIdx) {
1024 SDValue Vec;
1025 for (SDValue V : Values) {
1026 if (isUndef(Op: V)) {
1027 SrcIdx.push_back(Elt: -1);
1028 continue;
1029 }
1030 if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1031 return false;
1032 // All extracts should come from the same vector.
1033 SDValue T = V.getOperand(i: 0);
1034 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
1035 return false;
1036 Vec = T;
1037 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
1038 if (C == nullptr)
1039 return false;
1040 int I = C->getSExtValue();
1041 assert(I >= 0 && "Negative element index");
1042 SrcIdx.push_back(Elt: I);
1043 }
1044 SrcVec = Vec;
1045 return true;
1046 };
1047
1048 SmallVector<int,128> ExtIdx;
1049 SDValue ExtVec;
1050 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
1051 MVT ExtTy = ty(Op: ExtVec);
1052 unsigned ExtLen = ExtTy.getVectorNumElements();
1053 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
1054 // Construct a new shuffle mask that will produce a vector with the same
1055 // number of elements as the input vector, and such that the vector we
1056 // want will be the initial subvector of it.
1057 SmallVector<int,128> Mask;
1058 BitVector Used(ExtLen);
1059
1060 for (int M : ExtIdx) {
1061 Mask.push_back(Elt: M);
1062 if (M >= 0)
1063 Used.set(M);
1064 }
1065 // Fill the rest of the mask with the unused elements of ExtVec in hopes
1066 // that it will result in a permutation of ExtVec's elements. It's still
1067 // fine if it doesn't (e.g. if undefs are present, or elements are
1068 // repeated), but permutations can always be done efficiently via vdelta
1069 // and vrdelta.
1070 for (unsigned I = 0; I != ExtLen; ++I) {
1071 if (Mask.size() == ExtLen)
1072 break;
1073 if (!Used.test(Idx: I))
1074 Mask.push_back(Elt: I);
1075 }
1076
1077 SDValue S = DAG.getVectorShuffle(VT: ExtTy, dl, N1: ExtVec,
1078 N2: DAG.getUNDEF(VT: ExtTy), Mask);
1079 return ExtLen == VecLen ? S : LoHalf(V: S, DAG);
1080 }
1081 }
1082
1083 // Find most common element to initialize vector with. This is to avoid
1084 // unnecessary vinsert/valign for cases where the same value is present
1085 // many times. Creates a histogram of the vector's elements to find the
1086 // most common element n.
1087 assert(4*Words.size() == Subtarget.getVectorLength());
1088 int VecHist[32];
1089 int n = 0;
1090 for (unsigned i = 0; i != NumWords; ++i) {
1091 VecHist[i] = 0;
1092 if (Words[i].isUndef())
1093 continue;
1094 for (unsigned j = i; j != NumWords; ++j)
1095 if (Words[i] == Words[j])
1096 VecHist[i]++;
1097
1098 if (VecHist[i] > VecHist[n])
1099 n = i;
1100 }
1101
1102 SDValue HalfV = getZero(dl, Ty: VecTy, DAG);
1103 if (VecHist[n] > 1) {
1104 // Always splat at word (i32) granularity so that the SPLAT_VECTOR node
1105 // is selected as PS_vsplatrw (word broadcast) rather than PS_vsplatrb
1106 // (byte broadcast of the low byte only), which would corrupt multi-byte
1107 // element types.
1108 MVT WordVecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
1109 SDValue WordSplat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordVecTy, Operand: Words[n]);
1110 SDValue SplatV = DAG.getBitcast(VT: VecTy, V: WordSplat);
1111 HalfV = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: VecTy,
1112 Ops: {HalfV, SplatV, DAG.getConstant(Val: HwLen/2, DL: dl, VT: MVT::i32)});
1113 }
1114 SDValue HalfV0 = HalfV;
1115 SDValue HalfV1 = HalfV;
1116
1117 // Construct two halves in parallel, then or them together. Rn and Rm count
1118 // number of rotations needed before the next element. One last rotation is
1119 // performed post-loop to position the last element.
1120 int Rn = 0, Rm = 0;
1121 SDValue Sn, Sm;
1122 SDValue N = HalfV0;
1123 SDValue M = HalfV1;
1124 for (unsigned i = 0; i != NumWords/2; ++i) {
1125 // Rotate by element count since last insertion.
1126 if (Words[i] != Words[n] || VecHist[n] <= 1) {
1127 Sn = DAG.getConstant(Val: Rn, DL: dl, VT: MVT::i32);
1128 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1129 N = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1130 Ops: {HalfV0, Words[i]});
1131 Rn = 0;
1132 }
1133 if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) {
1134 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1135 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1136 M = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1137 Ops: {HalfV1, Words[i+NumWords/2]});
1138 Rm = 0;
1139 }
1140 Rn += 4;
1141 Rm += 4;
1142 }
1143 // Perform last rotation.
1144 Sn = DAG.getConstant(Val: Rn+HwLen/2, DL: dl, VT: MVT::i32);
1145 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1146 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1147 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1148
1149 SDValue T0 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV0);
1150 SDValue T1 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV1);
1151
1152 SDValue DstV = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ty(Op: T0), Ops: {T0, T1});
1153
1154 SDValue OutV =
1155 DAG.getBitcast(VT: tyVector(Ty: ty(Op: DstV), ElemTy: VecTy.getVectorElementType()), V: DstV);
1156 return OutV;
1157}
1158
1159SDValue
1160HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
1161 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
1162 MVT PredTy = ty(Op: PredV);
1163 unsigned HwLen = Subtarget.getVectorLength();
1164 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1165
1166 if (Subtarget.isHVXVectorType(VecTy: PredTy, IncludeBool: true)) {
1167 // Move the vector predicate SubV to a vector register, and scale it
1168 // down to match the representation (bytes per type element) that VecV
1169 // uses. The scaling down will pick every 2nd or 4th (every Scale-th
1170 // in general) element and put them at the front of the resulting
1171 // vector. This subvector will then be inserted into the Q2V of VecV.
1172 // To avoid having an operation that generates an illegal type (short
1173 // vector), generate a full size vector.
1174 //
1175 SDValue T = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: PredV);
1176 SmallVector<int,128> Mask(HwLen);
1177 // Scale = BitBytes(PredV) / Given BitBytes.
1178 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
1179 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
1180
1181 for (unsigned i = 0; i != HwLen; ++i) {
1182 unsigned Num = i % Scale;
1183 unsigned Off = i / Scale;
1184 Mask[BlockLen*Num + Off] = i;
1185 }
1186 SDValue S = DAG.getVectorShuffle(VT: ByteTy, dl, N1: T, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1187 if (!ZeroFill)
1188 return S;
1189 // Fill the bytes beyond BlockLen with 0s.
1190 // V6_pred_scalar2 cannot fill the entire predicate, so it only works
1191 // when BlockLen < HwLen.
1192 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1193 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1194 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1195 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1196 SDValue M = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Q);
1197 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ByteTy, N1: S, N2: M);
1198 }
1199
1200 // Make sure that this is a valid scalar predicate.
1201 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
1202
1203 unsigned Bytes = 8 / PredTy.getVectorNumElements();
1204 SmallVector<SDValue,4> Words[2];
1205 unsigned IdxW = 0;
1206
1207 SDValue W0 = isUndef(Op: PredV)
1208 ? DAG.getUNDEF(VT: MVT::i64)
1209 : DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: PredV);
1210 Words[IdxW].push_back(Elt: HiHalf(V: W0, DAG));
1211 Words[IdxW].push_back(Elt: LoHalf(V: W0, DAG));
1212
1213 while (Bytes < BitBytes) {
1214 IdxW ^= 1;
1215 Words[IdxW].clear();
1216
1217 if (Bytes < 4) {
1218 for (const SDValue &W : Words[IdxW ^ 1]) {
1219 SDValue T = expandPredicate(Vec32: W, dl, DAG);
1220 Words[IdxW].push_back(Elt: HiHalf(V: T, DAG));
1221 Words[IdxW].push_back(Elt: LoHalf(V: T, DAG));
1222 }
1223 } else {
1224 for (const SDValue &W : Words[IdxW ^ 1]) {
1225 Words[IdxW].push_back(Elt: W);
1226 Words[IdxW].push_back(Elt: W);
1227 }
1228 }
1229 Bytes *= 2;
1230 }
1231
1232 assert(Bytes == BitBytes);
1233 SDValue Vec = ZeroFill ? getZero(dl, Ty: ByteTy, DAG) : DAG.getUNDEF(VT: ByteTy);
1234 SDValue S4 = DAG.getConstant(Val: HwLen-4, DL: dl, VT: MVT::i32);
1235 for (const SDValue &W : Words[IdxW]) {
1236 Vec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Vec, N2: S4);
1237 Vec = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: ByteTy, N1: Vec, N2: W);
1238 }
1239
1240 return Vec;
1241}
1242
1243SDValue
1244HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
1245 const SDLoc &dl, MVT VecTy,
1246 SelectionDAG &DAG) const {
1247 // Construct a vector V of bytes, such that a comparison V >u 0 would
1248 // produce the required vector predicate.
1249 unsigned VecLen = Values.size();
1250 unsigned HwLen = Subtarget.getVectorLength();
1251 assert(VecLen <= HwLen || VecLen == 8*HwLen);
1252 SmallVector<SDValue,128> Bytes;
1253 bool AllT = true, AllF = true;
1254
1255 auto IsTrue = [] (SDValue V) {
1256 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1257 return !N->isZero();
1258 return false;
1259 };
1260 auto IsFalse = [] (SDValue V) {
1261 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1262 return N->isZero();
1263 return false;
1264 };
1265
1266 if (VecLen <= HwLen) {
1267 // In the hardware, each bit of a vector predicate corresponds to a byte
1268 // of a vector register. Calculate how many bytes does a bit of VecTy
1269 // correspond to.
1270 assert(HwLen % VecLen == 0);
1271 unsigned BitBytes = HwLen / VecLen;
1272 for (SDValue V : Values) {
1273 AllT &= IsTrue(V);
1274 AllF &= IsFalse(V);
1275
1276 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(Op: V, DL: dl, VT: MVT::i8)
1277 : DAG.getUNDEF(VT: MVT::i8);
1278 for (unsigned B = 0; B != BitBytes; ++B)
1279 Bytes.push_back(Elt: Ext);
1280 }
1281 } else {
1282 // There are as many i1 values, as there are bits in a vector register.
1283 // Divide the values into groups of 8 and check that each group consists
1284 // of the same value (ignoring undefs).
1285 for (unsigned I = 0; I != VecLen; I += 8) {
1286 unsigned B = 0;
1287 // Find the first non-undef value in this group.
1288 for (; B != 8; ++B) {
1289 if (!Values[I+B].isUndef())
1290 break;
1291 }
1292 SDValue F = Values[I+B];
1293 AllT &= IsTrue(F);
1294 AllF &= IsFalse(F);
1295
1296 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(Op: F, DL: dl, VT: MVT::i8)
1297 : DAG.getUNDEF(VT: MVT::i8);
1298 Bytes.push_back(Elt: Ext);
1299 // Verify that the rest of values in the group are the same as the
1300 // first.
1301 for (; B != 8; ++B)
1302 assert(Values[I+B].isUndef() || Values[I+B] == F);
1303 }
1304 }
1305
1306 if (AllT)
1307 return DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: VecTy);
1308 if (AllF)
1309 return DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: VecTy);
1310
1311 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1312 SDValue ByteVec = buildHvxVectorReg(Values: Bytes, dl, VecTy: ByteTy, DAG);
1313 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1314}
1315
1316SDValue
1317HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
1318 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1319 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1320
1321 unsigned ElemWidth = ElemTy.getSizeInBits();
1322 assert(ElemWidth >= 8 && ElemWidth <= 32);
1323 (void)ElemWidth;
1324
1325 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1326 SDValue ExWord = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1327 Ops: {VecV, ByteIdx});
1328 if (ElemTy == MVT::i32)
1329 return ExWord;
1330
1331 // Have an extracted word, need to extract the smaller element out of it.
1332 // 1. Extract the bits of (the original) IdxV that correspond to the index
1333 // of the desired element in the 32-bit word.
1334 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1335 // 2. Extract the element from the word.
1336 SDValue ExVec = DAG.getBitcast(VT: tyVector(Ty: ty(Op: ExWord), ElemTy), V: ExWord);
1337 return extractVector(VecV: ExVec, IdxV: SubIdx, dl, ValTy: ElemTy, ResTy: MVT::i32, DAG);
1338}
1339
1340SDValue
1341HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
1342 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1343 // Implement other return types if necessary.
1344 assert(ResTy == MVT::i1);
1345
1346 unsigned HwLen = Subtarget.getVectorLength();
1347 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1348 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1349
1350 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1351 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1352 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1353
1354 SDValue ExtB = extractHvxElementReg(VecV: ByteVec, IdxV, dl, ResTy: MVT::i32, DAG);
1355 SDValue Zero = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1356 return getInstr(MachineOpc: Hexagon::C2_cmpgtui, dl, Ty: MVT::i1, Ops: {ExtB, Zero}, DAG);
1357}
1358
1359SDValue
1360HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
1361 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1362 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1363
1364 unsigned ElemWidth = ElemTy.getSizeInBits();
1365 assert(ElemWidth >= 8 && ElemWidth <= 32);
1366 (void)ElemWidth;
1367
1368 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
1369 SDValue ByteIdxV) {
1370 MVT VecTy = ty(Op: VecV);
1371 unsigned HwLen = Subtarget.getVectorLength();
1372 SDValue MaskV =
1373 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
1374 Ops: {ByteIdxV, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)});
1375 SDValue RotV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {VecV, MaskV});
1376 SDValue InsV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy, Ops: {RotV, ValV});
1377 SDValue SubV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1378 Ops: {DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32), MaskV});
1379 SDValue TorV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {InsV, SubV});
1380 return TorV;
1381 };
1382
1383 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1384 if (ElemTy == MVT::i32)
1385 return InsertWord(VecV, ValV, ByteIdx);
1386
1387 // If this is not inserting a 32-bit word, convert it into such a thing.
1388 // 1. Extract the existing word from the target vector.
1389 SDValue WordIdx = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
1390 Ops: {ByteIdx, DAG.getConstant(Val: 2, DL: dl, VT: MVT::i32)});
1391 SDValue Ext = extractHvxElementReg(VecV: opCastElem(Vec: VecV, ElemTy: MVT::i32, DAG), IdxV: WordIdx,
1392 dl, ResTy: MVT::i32, DAG);
1393
1394 // 2. Treating the extracted word as a 32-bit vector, insert the given
1395 // value into it.
1396 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1397 MVT SubVecTy = tyVector(Ty: ty(Op: Ext), ElemTy);
1398 SDValue Ins = insertVector(VecV: DAG.getBitcast(VT: SubVecTy, V: Ext),
1399 ValV, IdxV: SubIdx, dl, ValTy: ElemTy, DAG);
1400
1401 // 3. Insert the 32-bit word back into the original vector.
1402 return InsertWord(VecV, Ins, ByteIdx);
1403}
1404
1405SDValue
1406HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
1407 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1408 unsigned HwLen = Subtarget.getVectorLength();
1409 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1410 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1411
1412 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1413 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1414 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1415 ValV = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i32, Operand: ValV);
1416
1417 SDValue InsV = insertHvxElementReg(VecV: ByteVec, IdxV, ValV, dl, DAG);
1418 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ty(Op: VecV), Operand: InsV);
1419}
1420
1421SDValue
1422HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV,
1423 SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1424 MVT VecTy = ty(Op: VecV);
1425 unsigned HwLen = Subtarget.getVectorLength();
1426 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1427 MVT ElemTy = VecTy.getVectorElementType();
1428 unsigned ElemWidth = ElemTy.getSizeInBits();
1429
1430 // If the source vector is a vector pair, get the single vector containing
1431 // the subvector of interest. The subvector will never overlap two single
1432 // vectors.
1433 if (isHvxPairTy(Ty: VecTy)) {
1434 unsigned SubIdx = Hexagon::vsub_lo;
1435 if (Idx * ElemWidth >= 8 * HwLen) {
1436 SubIdx = Hexagon::vsub_hi;
1437 Idx -= VecTy.getVectorNumElements() / 2;
1438 }
1439
1440 VecTy = typeSplit(VecTy).first;
1441 VecV = DAG.getTargetExtractSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV);
1442 if (VecTy == ResTy)
1443 return VecV;
1444 }
1445
1446 // The only meaningful subvectors of a single HVX vector are those that
1447 // fit in a scalar register.
1448 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
1449
1450 MVT WordTy = tyVector(Ty: VecTy, ElemTy: MVT::i32);
1451 SDValue WordVec = DAG.getBitcast(VT: WordTy, V: VecV);
1452 unsigned WordIdx = (Idx*ElemWidth) / 32;
1453
1454 SDValue W0Idx = DAG.getConstant(Val: WordIdx, DL: dl, VT: MVT::i32);
1455 SDValue W0 = extractHvxElementReg(VecV: WordVec, IdxV: W0Idx, dl, ResTy: MVT::i32, DAG);
1456 if (ResTy.getSizeInBits() == 32)
1457 return DAG.getBitcast(VT: ResTy, V: W0);
1458
1459 SDValue W1Idx = DAG.getConstant(Val: WordIdx+1, DL: dl, VT: MVT::i32);
1460 SDValue W1 = extractHvxElementReg(VecV: WordVec, IdxV: W1Idx, dl, ResTy: MVT::i32, DAG);
1461 SDValue WW = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::i64, DAG);
1462 return DAG.getBitcast(VT: ResTy, V: WW);
1463}
1464
1465SDValue
1466HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
1467 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1468 MVT VecTy = ty(Op: VecV);
1469 unsigned HwLen = Subtarget.getVectorLength();
1470 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1471 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1472 // IdxV is required to be a constant.
1473 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1474
1475 unsigned ResLen = ResTy.getVectorNumElements();
1476 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1477 unsigned Offset = Idx * BitBytes;
1478 SDValue Undef = DAG.getUNDEF(VT: ByteTy);
1479 SmallVector<int,128> Mask;
1480
1481 if (Subtarget.isHVXVectorType(VecTy: ResTy, IncludeBool: true)) {
1482 // Converting between two vector predicates. Since the result is shorter
1483 // than the source, it will correspond to a vector predicate with the
1484 // relevant bits replicated. The replication count is the ratio of the
1485 // source and target vector lengths.
1486 unsigned Rep = VecTy.getVectorNumElements() / ResLen;
1487 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
1488 for (unsigned i = 0; i != HwLen/Rep; ++i) {
1489 for (unsigned j = 0; j != Rep; ++j)
1490 Mask.push_back(Elt: i + Offset);
1491 }
1492 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1493 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: ShuffV);
1494 }
1495
1496 // Converting between a vector predicate and a scalar predicate. In the
1497 // vector predicate, a group of BitBytes bits will correspond to a single
1498 // i1 element of the source vector type. Those bits will all have the same
1499 // value. The same will be true for ByteVec, where each byte corresponds
1500 // to a bit in the vector predicate.
1501 // The algorithm is to traverse the ByteVec, going over the i1 values from
1502 // the source vector, and generate the corresponding representation in an
1503 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
1504 // elements so that the interesting 8 bytes will be in the low end of the
1505 // vector.
1506 unsigned Rep = 8 / ResLen;
1507 // Make sure the output fill the entire vector register, so repeat the
1508 // 8-byte groups as many times as necessary.
1509 for (unsigned r = 0; r != HwLen / 8; ++r) {
1510 // This will generate the indexes of the 8 interesting bytes.
1511 for (unsigned i = 0; i != ResLen; ++i) {
1512 for (unsigned j = 0; j != Rep; ++j)
1513 Mask.push_back(Elt: Offset + i*BitBytes);
1514 }
1515 }
1516
1517 SDValue Zero = getZero(dl, Ty: MVT::i32, DAG);
1518 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1519 // Combine the two low words from ShuffV into a v8i8, and byte-compare
1520 // them against 0.
1521 SDValue W0 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32, Ops: {ShuffV, Zero});
1522 SDValue W1 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1523 Ops: {ShuffV, DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32)});
1524 SDValue Vec64 = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::v8i8, DAG);
1525 return getInstr(MachineOpc: Hexagon::A4_vcmpbgtui, dl, Ty: ResTy,
1526 Ops: {Vec64, DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32)}, DAG);
1527}
1528
1529SDValue
1530HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
1531 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1532 MVT VecTy = ty(Op: VecV);
1533 MVT SubTy = ty(Op: SubV);
1534 unsigned HwLen = Subtarget.getVectorLength();
1535 MVT ElemTy = VecTy.getVectorElementType();
1536 unsigned ElemWidth = ElemTy.getSizeInBits();
1537
1538 bool IsPair = isHvxPairTy(Ty: VecTy);
1539 MVT SingleTy = MVT::getVectorVT(VT: ElemTy, NumElements: (8*HwLen)/ElemWidth);
1540 // The two single vectors that VecV consists of, if it's a pair.
1541 SDValue V0, V1;
1542 SDValue SingleV = VecV;
1543 SDValue PickHi;
1544
1545 if (IsPair) {
1546 V0 = LoHalf(V: VecV, DAG);
1547 V1 = HiHalf(V: VecV, DAG);
1548
1549 SDValue HalfV = DAG.getConstant(Val: SingleTy.getVectorNumElements(),
1550 DL: dl, VT: MVT::i32);
1551 PickHi = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: IdxV, RHS: HalfV, Cond: ISD::SETUGT);
1552 if (isHvxSingleTy(Ty: SubTy)) {
1553 if (const auto *CN = dyn_cast<const ConstantSDNode>(Val: IdxV.getNode())) {
1554 unsigned Idx = CN->getZExtValue();
1555 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
1556 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
1557 return DAG.getTargetInsertSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV, Subreg: SubV);
1558 }
1559 // If IdxV is not a constant, generate the two variants: with the
1560 // SubV as the high and as the low subregister, and select the right
1561 // pair based on the IdxV.
1562 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SubV, V1});
1563 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SubV});
1564 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1565 }
1566 // The subvector being inserted must be entirely contained in one of
1567 // the vectors V0 or V1. Set SingleV to the correct one, and update
1568 // IdxV to be the index relative to the beginning of that vector.
1569 SDValue S = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: IdxV, N2: HalfV);
1570 IdxV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: PickHi, N2: S, N3: IdxV);
1571 SingleV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: SingleTy, N1: PickHi, N2: V1, N3: V0);
1572 }
1573
1574 // The only meaningful subvectors of a single HVX vector are those that
1575 // fit in a scalar register.
1576 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
1577 // Convert IdxV to be index in bytes.
1578 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1579 if (!IdxN || !IdxN->isZero()) {
1580 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1581 N2: DAG.getConstant(Val: ElemWidth/8, DL: dl, VT: MVT::i32));
1582 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: IdxV);
1583 }
1584 // When inserting a single word, the rotation back to the original position
1585 // would be by HwLen-Idx, but if two words are inserted, it will need to be
1586 // by (HwLen-4)-Idx.
1587 unsigned RolBase = HwLen;
1588 if (SubTy.getSizeInBits() == 32) {
1589 SDValue V = DAG.getBitcast(VT: MVT::i32, V: SubV);
1590 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: V);
1591 } else {
1592 SDValue V = DAG.getBitcast(VT: MVT::i64, V: SubV);
1593 SDValue R0 = LoHalf(V, DAG);
1594 SDValue R1 = HiHalf(V, DAG);
1595 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R0);
1596 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV,
1597 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
1598 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R1);
1599 RolBase = HwLen-4;
1600 }
1601 // If the vector wasn't ror'ed, don't ror it back.
1602 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1603 SDValue RolV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1604 N1: DAG.getConstant(Val: RolBase, DL: dl, VT: MVT::i32), N2: IdxV);
1605 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: RolV);
1606 }
1607
1608 if (IsPair) {
1609 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SingleV, V1});
1610 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SingleV});
1611 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1612 }
1613 return SingleV;
1614}
1615
1616SDValue
1617HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1618 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1619 MVT VecTy = ty(Op: VecV);
1620 MVT SubTy = ty(Op: SubV);
1621 assert(Subtarget.isHVXVectorType(VecTy, true));
1622 // VecV is an HVX vector predicate. SubV may be either an HVX vector
1623 // predicate as well, or it can be a scalar predicate.
1624
1625 unsigned VecLen = VecTy.getVectorNumElements();
1626 unsigned HwLen = Subtarget.getVectorLength();
1627 assert(HwLen % VecLen == 0 && "Unexpected vector type");
1628
1629 unsigned Scale = VecLen / SubTy.getVectorNumElements();
1630 unsigned BitBytes = HwLen / VecLen;
1631 unsigned BlockLen = HwLen / Scale;
1632
1633 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1634 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1635 SDValue ByteSub = createHvxPrefixPred(PredV: SubV, dl, BitBytes, ZeroFill: false, DAG);
1636 SDValue ByteIdx;
1637
1638 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1639 if (!IdxN || !IdxN->isZero()) {
1640 ByteIdx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1641 N2: DAG.getConstant(Val: BitBytes, DL: dl, VT: MVT::i32));
1642 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteIdx);
1643 }
1644
1645 // ByteVec is the target vector VecV rotated in such a way that the
1646 // subvector should be inserted at index 0. Generate a predicate mask
1647 // and use vmux to do the insertion.
1648 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1649 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1650 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1651 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1652 ByteVec = getInstr(MachineOpc: Hexagon::V6_vmux, dl, Ty: ByteTy, Ops: {Q, ByteSub, ByteVec}, DAG);
1653 // Rotate ByteVec back, and convert to a vector predicate.
1654 if (!IdxN || !IdxN->isZero()) {
1655 SDValue HwLenV = DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32);
1656 SDValue ByteXdi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: HwLenV, N2: ByteIdx);
1657 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteXdi);
1658 }
1659 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1660}
1661
1662SDValue
1663HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1664 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1665 // Sign- and any-extending of a vector predicate to a vector register is
1666 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1667 // a vector of 1s (where the 1s are of type matching the vector type).
1668 assert(Subtarget.isHVXVectorType(ResTy));
1669 if (!ZeroExt)
1670 return DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ResTy, Operand: VecV);
1671
1672 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1673 SDValue True = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
1674 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1675 SDValue False = getZero(dl, Ty: ResTy, DAG);
1676 return DAG.getSelect(DL: dl, VT: ResTy, Cond: VecV, LHS: True, RHS: False);
1677}
1678
1679SDValue
1680HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1681 MVT ResTy, SelectionDAG &DAG) const {
1682 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1683 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1684 // vector register. The remaining bits of the vector register are
1685 // unspecified.
1686
1687 MachineFunction &MF = DAG.getMachineFunction();
1688 unsigned HwLen = Subtarget.getVectorLength();
1689 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1690 MVT PredTy = ty(Op: VecQ);
1691 unsigned PredLen = PredTy.getVectorNumElements();
1692 assert(HwLen % PredLen == 0);
1693 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8*HwLen/PredLen), NumElements: PredLen);
1694
1695 Type *Int8Ty = Type::getInt8Ty(C&: *DAG.getContext());
1696 SmallVector<Constant*, 128> Tmp;
1697 // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
1698 // These are bytes with the LSB rotated left with respect to their index.
1699 for (unsigned i = 0; i != HwLen/8; ++i) {
1700 for (unsigned j = 0; j != 8; ++j)
1701 Tmp.push_back(Elt: ConstantInt::get(Ty: Int8Ty, V: 1ull << j));
1702 }
1703 Constant *CV = ConstantVector::get(V: Tmp);
1704 Align Alignment(HwLen);
1705 SDValue CP = LowerConstantPool(
1706 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
1707 DAG);
1708 SDValue Bytes =
1709 DAG.getLoad(VT: ByteTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
1710 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
1711
1712 // Select the bytes that correspond to true bits in the vector predicate.
1713 SDValue Sel = DAG.getSelect(DL: dl, VT: VecTy, Cond: VecQ, LHS: DAG.getBitcast(VT: VecTy, V: Bytes),
1714 RHS: getZero(dl, Ty: VecTy, DAG));
1715 // Calculate the OR of all bytes in each group of 8. That will compress
1716 // all the individual bits into a single byte.
1717 // First, OR groups of 4, via vrmpy with 0x01010101.
1718 SDValue All1 =
1719 DAG.getSplatBuildVector(VT: MVT::v4i8, DL: dl, Op: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1720 SDValue Vrmpy = getInstr(MachineOpc: Hexagon::V6_vrmpyub, dl, Ty: ByteTy, Ops: {Sel, All1}, DAG);
1721 // Then rotate the accumulated vector by 4 bytes, and do the final OR.
1722 SDValue Rot = getInstr(MachineOpc: Hexagon::V6_valignbi, dl, Ty: ByteTy,
1723 Ops: {Vrmpy, Vrmpy, DAG.getTargetConstant(Val: 4, DL: dl, VT: MVT::i32)}, DAG);
1724 SDValue Vor = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, Ops: {Vrmpy, Rot});
1725
1726 // Pick every 8th byte and coalesce them at the beginning of the output.
1727 // For symmetry, coalesce every 1+8th byte after that, then every 2+8th
1728 // byte and so on.
1729 SmallVector<int,128> Mask;
1730 for (unsigned i = 0; i != HwLen; ++i)
1731 Mask.push_back(Elt: (8*i) % HwLen + i/(HwLen/8));
1732 SDValue Collect =
1733 DAG.getVectorShuffle(VT: ByteTy, dl, N1: Vor, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1734 return DAG.getBitcast(VT: ResTy, V: Collect);
1735}
1736
1737SDValue
1738HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed,
1739 const SDLoc &dl, SelectionDAG &DAG) const {
1740 // Take a vector and resize the element type to match the given type.
1741 MVT InpTy = ty(Op: VecV);
1742 if (InpTy == ResTy)
1743 return VecV;
1744
1745 unsigned InpWidth = InpTy.getSizeInBits();
1746 unsigned ResWidth = ResTy.getSizeInBits();
1747
1748 if (InpTy.isFloatingPoint()) {
1749 return InpWidth < ResWidth
1750 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: ResTy, Operand: VecV)
1751 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ResTy, N1: VecV,
1752 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
1753 }
1754
1755 assert(InpTy.isInteger());
1756
1757 if (InpWidth < ResWidth) {
1758 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1759 return DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ResTy, Operand: VecV);
1760 } else {
1761 unsigned NarOpc = Signed ? HexagonISD::SSAT : HexagonISD::USAT;
1762 return DAG.getNode(Opcode: NarOpc, DL: dl, VT: ResTy, N1: VecV, N2: DAG.getValueType(ResTy));
1763 }
1764}
1765
1766SDValue
1767HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx,
1768 SelectionDAG &DAG) const {
1769 assert(ty(Vec).getSizeInBits() % SubTy.getSizeInBits() == 0);
1770
1771 const SDLoc &dl(Vec);
1772 unsigned ElemIdx = SubIdx * SubTy.getVectorNumElements();
1773 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubTy,
1774 Ops: {Vec, DAG.getConstant(Val: ElemIdx, DL: dl, VT: MVT::i32)});
1775}
1776
1777SDValue
1778HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1779 const {
1780 const SDLoc &dl(Op);
1781 MVT VecTy = ty(Op);
1782
1783 unsigned Size = Op.getNumOperands();
1784 SmallVector<SDValue,128> Ops;
1785 for (unsigned i = 0; i != Size; ++i)
1786 Ops.push_back(Elt: Op.getOperand(i));
1787
1788 if (VecTy.getVectorElementType() == MVT::i1)
1789 return buildHvxVectorPred(Values: Ops, dl, VecTy, DAG);
1790
1791 // In case of MVT::f16 BUILD_VECTOR, since MVT::f16 is
1792 // not a legal type, just bitcast the node to use i16
1793 // types and bitcast the result back to f16
1794 if (VecTy.getVectorElementType() == MVT::f16 ||
1795 VecTy.getVectorElementType() == MVT::bf16) {
1796 SmallVector<SDValue, 64> NewOps;
1797 for (unsigned i = 0; i != Size; i++)
1798 NewOps.push_back(Elt: DAG.getBitcast(VT: MVT::i16, V: Ops[i]));
1799
1800 SDValue T0 =
1801 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: tyVector(Ty: VecTy, ElemTy: MVT::i16), Ops: NewOps);
1802 return DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: VecTy.getVectorElementType()), V: T0);
1803 }
1804
1805 // First, split the BUILD_VECTOR for vector pairs. We could generate
1806 // some pairs directly (via splat), but splats should be generated
1807 // by the combiner prior to getting here.
1808 if (VecTy.getSizeInBits() == 16 * Subtarget.getVectorLength()) {
1809 ArrayRef<SDValue> A(Ops);
1810 MVT SingleTy = typeSplit(VecTy).first;
1811 SDValue V0 = buildHvxVectorReg(Values: A.take_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1812 SDValue V1 = buildHvxVectorReg(Values: A.drop_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1813 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, N1: V0, N2: V1);
1814 }
1815
1816 return buildHvxVectorReg(Values: Ops, dl, VecTy, DAG);
1817}
1818
1819SDValue
1820HexagonTargetLowering::LowerHvxSplatVector(SDValue Op, SelectionDAG &DAG)
1821 const {
1822 const SDLoc &dl(Op);
1823 MVT VecTy = ty(Op);
1824 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
1825
1826 if (ArgTy == MVT::f16 || ArgTy == MVT::bf16) {
1827 MVT SplatTy = MVT::getVectorVT(VT: MVT::i16, NumElements: VecTy.getVectorNumElements());
1828 SDValue ToInt16 = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: 0));
1829 SDValue ToInt32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: ToInt16);
1830 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: SplatTy, Operand: ToInt32);
1831 return DAG.getBitcast(VT: VecTy, V: Splat);
1832 }
1833
1834 return SDValue();
1835}
1836
1837SDValue
1838HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1839 const {
1840 // Vector concatenation of two integer (non-bool) vectors does not need
1841 // special lowering. Custom-lower concats of bool vectors and expand
1842 // concats of more than 2 vectors.
1843 MVT VecTy = ty(Op);
1844 const SDLoc &dl(Op);
1845 unsigned NumOp = Op.getNumOperands();
1846 if (VecTy.getVectorElementType() != MVT::i1) {
1847 if (NumOp == 2)
1848 return Op;
1849 // Expand the other cases into a build-vector.
1850 SmallVector<SDValue,8> Elems;
1851 for (SDValue V : Op.getNode()->ops())
1852 DAG.ExtractVectorElements(Op: V, Args&: Elems);
1853 // A vector of i16 will be broken up into a build_vector of i16's.
1854 // This is a problem, since at the time of operation legalization,
1855 // all operations are expected to be type-legalized, and i16 is not
1856 // a legal type. If any of the extracted elements is not of a valid
1857 // type, sign-extend it to a valid one.
1858 for (SDValue &V : Elems) {
1859 MVT Ty = ty(Op: V);
1860 if (!isTypeLegal(VT: Ty)) {
1861 MVT NTy = typeLegalize(Ty, DAG);
1862 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1863 V = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NTy,
1864 N1: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NTy,
1865 N1: V.getOperand(i: 0), N2: V.getOperand(i: 1)),
1866 N2: DAG.getValueType(Ty));
1867 continue;
1868 }
1869 // A few less complicated cases.
1870 switch (V.getOpcode()) {
1871 case ISD::Constant:
1872 V = DAG.getSExtOrTrunc(Op: V, DL: dl, VT: NTy);
1873 break;
1874 case ISD::UNDEF:
1875 V = DAG.getUNDEF(VT: NTy);
1876 break;
1877 case ISD::TRUNCATE:
1878 V = V.getOperand(i: 0);
1879 break;
1880 default:
1881 llvm_unreachable("Unexpected vector element");
1882 }
1883 }
1884 }
1885 return DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Elems);
1886 }
1887
1888 assert(VecTy.getVectorElementType() == MVT::i1);
1889 unsigned HwLen = Subtarget.getVectorLength();
1890 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1891
1892 SDValue Op0 = Op.getOperand(i: 0);
1893
1894 // If the operands are HVX types (i.e. not scalar predicates), then
1895 // defer the concatenation, and create QCAT instead.
1896 if (Subtarget.isHVXVectorType(VecTy: ty(Op: Op0), IncludeBool: true)) {
1897 if (NumOp == 2)
1898 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: Op0, N2: Op.getOperand(i: 1));
1899
1900 ArrayRef<SDUse> U(Op.getNode()->ops());
1901 SmallVector<SDValue, 4> SV(U);
1902 ArrayRef<SDValue> Ops(SV);
1903
1904 MVT HalfTy = typeSplit(VecTy).first;
1905 SDValue V0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1906 Ops: Ops.take_front(N: NumOp/2));
1907 SDValue V1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1908 Ops: Ops.take_back(N: NumOp/2));
1909 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: V0, N2: V1);
1910 }
1911
1912 // Count how many bytes (in a vector register) each bit in VecTy
1913 // corresponds to.
1914 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1915
1916 // Make sure that createHvxPrefixPred will only ever need to expand
1917 // the predicate, i.e. bytes-per-bit in the input is not greater than
1918 // the target bytes-per-bit in the result.
1919 SDValue Combined = combineConcatOfScalarPreds(Op, BitBytes, DAG);
1920 SmallVector<SDValue,8> Prefixes;
1921 for (SDValue V : Combined.getNode()->op_values()) {
1922 SDValue P = createHvxPrefixPred(PredV: V, dl, BitBytes, ZeroFill: true, DAG);
1923 Prefixes.push_back(Elt: P);
1924 }
1925
1926 unsigned InpLen = ty(Op: Combined.getOperand(i: 0)).getVectorNumElements();
1927 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1928 SDValue S = DAG.getConstant(Val: HwLen - InpLen*BitBytes, DL: dl, VT: MVT::i32);
1929 SDValue Res = getZero(dl, Ty: ByteTy, DAG);
1930 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1931 Res = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Res, N2: S);
1932 Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, N1: Res, N2: Prefixes[e-i-1]);
1933 }
1934 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: Res);
1935}
1936
1937SDValue
1938HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1939 const {
1940 // Change the type of the extracted element to i32.
1941 SDValue VecV = Op.getOperand(i: 0);
1942 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1943 const SDLoc &dl(Op);
1944 SDValue IdxV = Op.getOperand(i: 1);
1945 if (ElemTy == MVT::i1)
1946 return extractHvxElementPred(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1947
1948 return extractHvxElementReg(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1949}
1950
1951SDValue
1952HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1953 const {
1954 const SDLoc &dl(Op);
1955 SDValue VecV = Op.getOperand(i: 0);
1956 SDValue ValV = Op.getOperand(i: 1);
1957 SDValue IdxV = Op.getOperand(i: 2);
1958 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1959 if (ElemTy == MVT::i1)
1960 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1961
1962 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1963}
1964
1965SDValue
1966HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1967 const {
1968 SDValue SrcV = Op.getOperand(i: 0);
1969 MVT SrcTy = ty(Op: SrcV);
1970 MVT DstTy = ty(Op);
1971 SDValue IdxV = Op.getOperand(i: 1);
1972 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1973 assert(Idx % DstTy.getVectorNumElements() == 0);
1974 (void)Idx;
1975 const SDLoc &dl(Op);
1976
1977 MVT ElemTy = SrcTy.getVectorElementType();
1978 if (ElemTy == MVT::i1)
1979 return extractHvxSubvectorPred(VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1980
1981 return extractHvxSubvectorReg(OrigOp: Op, VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1982}
1983
1984SDValue
1985HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1986 const {
1987 // Idx does not need to be a constant.
1988 SDValue VecV = Op.getOperand(i: 0);
1989 SDValue ValV = Op.getOperand(i: 1);
1990 SDValue IdxV = Op.getOperand(i: 2);
1991
1992 const SDLoc &dl(Op);
1993 MVT VecTy = ty(Op: VecV);
1994 MVT ElemTy = VecTy.getVectorElementType();
1995 if (ElemTy == MVT::i1)
1996 return insertHvxSubvectorPred(VecV, SubV: ValV, IdxV, dl, DAG);
1997
1998 return insertHvxSubvectorReg(VecV, SubV: ValV, IdxV, dl, DAG);
1999}
2000
2001SDValue
2002HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
2003 // Lower any-extends of boolean vectors to sign-extends, since they
2004 // translate directly to Q2V. Zero-extending could also be done equally
2005 // fast, but Q2V is used/recognized in more places.
2006 // For all other vectors, use zero-extend.
2007 MVT ResTy = ty(Op);
2008 SDValue InpV = Op.getOperand(i: 0);
2009 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2010 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2011 return LowerHvxSignExt(Op, DAG);
2012 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: ResTy, Operand: InpV);
2013}
2014
2015SDValue
2016HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
2017 MVT ResTy = ty(Op);
2018 SDValue InpV = Op.getOperand(i: 0);
2019 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2020 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2021 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: false, DAG);
2022 return Op;
2023}
2024
2025SDValue
2026HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
2027 MVT ResTy = ty(Op);
2028 SDValue InpV = Op.getOperand(i: 0);
2029 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2030 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2031 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: true, DAG);
2032 return Op;
2033}
2034
2035SDValue
2036HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
2037 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
2038 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
2039 const SDLoc &dl(Op);
2040 MVT ResTy = ty(Op);
2041 SDValue InpV = Op.getOperand(i: 0);
2042 assert(ResTy == ty(InpV));
2043
2044 // Calculate the vectors of 1 and bitwidth(x).
2045 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2046 unsigned ElemWidth = ElemTy.getSizeInBits();
2047
2048 SDValue Vec1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2049 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
2050 SDValue VecW = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2051 Operand: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2052 SDValue VecN1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2053 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
2054
2055 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
2056 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
2057 // it separately in custom combine or selection).
2058 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy,
2059 Ops: {DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {InpV, VecN1}),
2060 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {InpV, Vec1})});
2061 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy,
2062 Ops: {VecW, DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: ResTy, Operand: A)});
2063}
2064
2065SDValue
2066HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
2067 const SDLoc &dl(Op);
2068 MVT ResTy = ty(Op);
2069 assert(ResTy.getVectorElementType() == MVT::i32);
2070
2071 SDValue Vs = Op.getOperand(i: 0);
2072 SDValue Vt = Op.getOperand(i: 1);
2073
2074 SDVTList ResTys = DAG.getVTList(VT1: ResTy, VT2: ResTy);
2075 unsigned Opc = Op.getOpcode();
2076
2077 // On HVX v62+ producing the full product is cheap, so legalize MULH to LOHI.
2078 if (Opc == ISD::MULHU)
2079 return DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2080 if (Opc == ISD::MULHS)
2081 return DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2082
2083#ifndef NDEBUG
2084 Op.dump(&DAG);
2085#endif
2086 llvm_unreachable("Unexpected mulh operation");
2087}
2088
2089SDValue
2090HexagonTargetLowering::LowerHvxMulLoHi(SDValue Op, SelectionDAG &DAG) const {
2091 const SDLoc &dl(Op);
2092 unsigned Opc = Op.getOpcode();
2093 SDValue Vu = Op.getOperand(i: 0);
2094 SDValue Vv = Op.getOperand(i: 1);
2095
2096 // If the HI part is not used, convert it to a regular MUL.
2097 if (auto HiVal = Op.getValue(R: 1); HiVal.use_empty()) {
2098 // Need to preserve the types and the number of values.
2099 SDValue Hi = DAG.getUNDEF(VT: ty(Op: HiVal));
2100 SDValue Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: ty(Op), Ops: {Vu, Vv});
2101 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2102 }
2103
2104 bool SignedVu = Opc == HexagonISD::SMUL_LOHI;
2105 bool SignedVv = Opc == HexagonISD::SMUL_LOHI || Opc == HexagonISD::USMUL_LOHI;
2106
2107 // Legal on HVX v62+, but lower it here because patterns can't handle multi-
2108 // valued nodes.
2109 if (Subtarget.useHVXV62Ops())
2110 return emitHvxMulLoHiV62(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2111
2112 if (Opc == HexagonISD::SMUL_LOHI) {
2113 // Direct MULHS expansion is cheaper than doing the whole SMUL_LOHI,
2114 // for other signedness LOHI is cheaper.
2115 if (auto LoVal = Op.getValue(R: 0); LoVal.use_empty()) {
2116 SDValue Hi = emitHvxMulHsV60(A: Vu, B: Vv, dl, DAG);
2117 SDValue Lo = DAG.getUNDEF(VT: ty(Op: LoVal));
2118 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2119 }
2120 }
2121
2122 return emitHvxMulLoHiV60(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2123}
2124
2125SDValue
2126HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
2127 SDValue Val = Op.getOperand(i: 0);
2128 MVT ResTy = ty(Op);
2129 MVT ValTy = ty(Op: Val);
2130 const SDLoc &dl(Op);
2131
2132 if (isHvxBoolTy(Ty: ValTy) && ResTy.isScalarInteger()) {
2133 unsigned HwLen = Subtarget.getVectorLength();
2134 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
2135
2136 // When the predicate is shorter than the predicate register, each boolean
2137 // is represented by multiple consecutive bits in the input register.
2138 // Condense the bits so each boolean is represented by one bit. This only
2139 // handles 2x and 4x compaction ratios.
2140 unsigned PredLen = ValTy.getVectorNumElements();
2141 if (PredLen < HwLen) {
2142 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2143 Val = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Val);
2144 if (HwLen > PredLen * 2) {
2145 assert(HwLen == PredLen * 4);
2146 PredLen *= 2;
2147 Val = getInstr(MachineOpc: Hexagon::V6_vdealh, dl, Ty: ByteTy, Ops: Val, DAG);
2148 }
2149 if (HwLen > PredLen) {
2150 assert(HwLen == PredLen * 2);
2151 Val = getInstr(MachineOpc: Hexagon::V6_vdealb, dl, Ty: ByteTy, Ops: Val, DAG);
2152 }
2153 Val = DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ValTy, Operand: Val);
2154 }
2155
2156 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2157 unsigned BitWidth = ResTy.getSizeInBits();
2158
2159 if (BitWidth < 64) {
2160 SDValue W0 = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
2161 dl, ResTy: MVT::i32, DAG);
2162 if (BitWidth == 32)
2163 return W0;
2164 assert(BitWidth < 32u);
2165 return DAG.getZExtOrTrunc(Op: W0, DL: dl, VT: ResTy);
2166 }
2167
2168 // The result is >= 64 bits. The only options are 64 or 128.
2169 assert(BitWidth == 64 || BitWidth == 128);
2170 SmallVector<SDValue,4> Words;
2171 for (unsigned i = 0; i != BitWidth/32; ++i) {
2172 SDValue W = extractHvxElementReg(
2173 VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl, ResTy: MVT::i32, DAG);
2174 Words.push_back(Elt: W);
2175 }
2176 SmallVector<SDValue,2> Combines;
2177 assert(Words.size() % 2 == 0);
2178 for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
2179 SDValue C = getCombine(Hi: Words[i+1], Lo: Words[i], dl, ResTy: MVT::i64, DAG);
2180 Combines.push_back(Elt: C);
2181 }
2182
2183 if (BitWidth == 64)
2184 return Combines[0];
2185
2186 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: ResTy, Ops: Combines);
2187 }
2188
2189 // Handle bitcast from i32, v2i16, and v4i8 to v32i1.
2190 // Splat the input into a 32-element i32 vector, then AND each element
2191 // with a unique bitmask to isolate individual bits.
2192 auto bitcastI32ToV32I1 = [&](SDValue Val32) {
2193 assert(Val32.getValueType().getSizeInBits() == 32 &&
2194 "Input must be 32 bits");
2195 MVT VecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: 32);
2196 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Val32);
2197 SmallVector<SDValue, 32> Mask;
2198 for (unsigned i = 0; i < 32; ++i)
2199 Mask.push_back(Elt: DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32));
2200
2201 SDValue MaskVec = DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Mask);
2202 SDValue Anded = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: VecTy, N1: Splat, N2: MaskVec);
2203 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: MVT::v32i1, Operand: Anded);
2204 };
2205 // === Case: v32i1 ===
2206 if (ResTy == MVT::v32i1 &&
2207 (ValTy == MVT::i32 || ValTy == MVT::v2i16 || ValTy == MVT::v4i8) &&
2208 Subtarget.useHVX128BOps()) {
2209 SDValue Val32 = Val;
2210 if (ValTy == MVT::v2i16 || ValTy == MVT::v4i8)
2211 Val32 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Val);
2212 return bitcastI32ToV32I1(Val32);
2213 }
2214 // === Case: v64i1 ===
2215 if (ResTy == MVT::v64i1 && ValTy == MVT::i64 && Subtarget.useHVX128BOps()) {
2216 // Split i64 into lo/hi 32-bit halves.
2217 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Val);
2218 SDValue HiShifted = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Val,
2219 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64));
2220 SDValue Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: HiShifted);
2221
2222 // Reuse the same 32-bit logic twice.
2223 SDValue LoRes = bitcastI32ToV32I1(Lo);
2224 SDValue HiRes = bitcastI32ToV32I1(Hi);
2225
2226 // Concatenate into a v64i1 predicate.
2227 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v64i1, N1: LoRes, N2: HiRes);
2228 }
2229
2230 if (isHvxBoolTy(Ty: ResTy) && ValTy.isScalarInteger()) {
2231 // Handle bitcast from i128 -> v128i1 and i64 -> v64i1.
2232 unsigned BitWidth = ValTy.getSizeInBits();
2233 unsigned HwLen = Subtarget.getVectorLength();
2234 assert(BitWidth == HwLen);
2235
2236 MVT ValAsVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: BitWidth / 8);
2237 SDValue ValAsVec = DAG.getBitcast(VT: ValAsVecTy, V: Val);
2238 // Splat each byte of Val 8 times.
2239 // Bytes = [(b0)x8, (b1)x8, ...., (b15)x8]
2240 // where b0, b1,..., b15 are least to most significant bytes of I.
2241 SmallVector<SDValue, 128> Bytes;
2242 // Tmp: 0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80, 0x01,0x02,0x04,0x08,...
2243 // These are bytes with the LSB rotated left with respect to their index.
2244 SmallVector<SDValue, 128> Tmp;
2245 for (unsigned I = 0; I != HwLen / 8; ++I) {
2246 SDValue Idx = DAG.getConstant(Val: I, DL: dl, VT: MVT::i32);
2247 SDValue Byte =
2248 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i8, N1: ValAsVec, N2: Idx);
2249 for (unsigned J = 0; J != 8; ++J) {
2250 Bytes.push_back(Elt: Byte);
2251 Tmp.push_back(Elt: DAG.getConstant(Val: 1ull << J, DL: dl, VT: MVT::i8));
2252 }
2253 }
2254
2255 MVT ConstantVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2256 SDValue ConstantVec = DAG.getBuildVector(VT: ConstantVecTy, DL: dl, Ops: Tmp);
2257 SDValue I2V = buildHvxVectorReg(Values: Bytes, dl, VecTy: ConstantVecTy, DAG);
2258
2259 // Each Byte in the I2V will be set iff corresponding bit is set in Val.
2260 I2V = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ConstantVecTy, Ops: {I2V, ConstantVec});
2261 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: I2V);
2262 }
2263
2264 return Op;
2265}
2266
2267SDValue HexagonTargetLowering::LowerHvxStore(SDValue Op,
2268 SelectionDAG &DAG) const {
2269 const SDLoc &dl(Op);
2270 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
2271 SDValue Val = SN->getValue();
2272 MVT ValTy = ty(Op: Val);
2273
2274 // Check if this is a store of an HVX bool vector (predicate)
2275 if (!isHvxBoolTy(Ty: ValTy))
2276 return SDValue();
2277
2278 unsigned NumBits = ValTy.getVectorNumElements();
2279 MachineMemOperand *MMO = SN->getMemOperand();
2280
2281 // Check alignment requirements based on predicate size
2282 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2283 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2284 return SDValue();
2285
2286 unsigned HwLen = Subtarget.getVectorLength();
2287 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
2288
2289 // Compress the predicate into a vector register
2290 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2291
2292 // Extract words from the compressed vector
2293 SmallVector<SDValue, 4> Words;
2294 for (unsigned i = 0; i != NumBits / 32; ++i) {
2295 SDValue W = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl,
2296 ResTy: MVT::i32, DAG);
2297 Words.push_back(Elt: W);
2298 }
2299
2300 SDValue Chain = SN->getChain();
2301 SDValue BasePtr = SN->getBasePtr();
2302 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2303
2304 if (NumBits == 32)
2305 return DAG.getStore(Chain, dl, Val: Words[0], Ptr: BasePtr, PtrInfo,
2306 Alignment: MMO->getBaseAlign());
2307
2308 if (NumBits == 64) {
2309 SDValue W64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2310 return DAG.getStore(Chain, dl, Val: W64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2311 }
2312
2313 if (NumBits == 128) {
2314 SDValue Lo64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2315 SDValue Hi64 = getCombine(Hi: Words[3], Lo: Words[2], dl, ResTy: MVT::i64, DAG);
2316
2317 Chain =
2318 DAG.getStore(Chain, dl, Val: Lo64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2319
2320 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2321 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2322 return DAG.getStore(Chain, dl, Val: Hi64, Ptr: Ptr8, PtrInfo: PtrInfo.getWithOffset(O: 8),
2323 Alignment: Align(8));
2324 }
2325
2326 return SDValue();
2327}
2328
2329SDValue HexagonTargetLowering::LowerHvxLoad(SDValue Op,
2330 SelectionDAG &DAG) const {
2331 const SDLoc &dl(Op);
2332 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
2333 MVT ResTy = ty(Op);
2334
2335 // Check if this is a load of an HVX bool vector (predicate)
2336 if (!isHvxBoolTy(Ty: ResTy))
2337 return SDValue();
2338
2339 unsigned NumBits = ResTy.getVectorNumElements();
2340 MachineMemOperand *MMO = LN->getMemOperand();
2341
2342 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2343 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2344 return SDValue();
2345
2346 SDValue Chain = LN->getChain();
2347 SDValue BasePtr = LN->getBasePtr();
2348 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2349
2350 if (NumBits == 32) {
2351 SDValue W32 =
2352 DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2353 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v32i1, Operand: W32);
2354 SDValue Ops[] = {Pred, W32.getValue(R: 1)};
2355 return DAG.getMergeValues(Ops, dl);
2356 }
2357
2358 if (NumBits == 64) {
2359 SDValue W64 =
2360 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2361 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: W64);
2362 SDValue Ops[] = {Pred, W64.getValue(R: 1)};
2363 return DAG.getMergeValues(Ops, dl);
2364 }
2365
2366 if (NumBits == 128) {
2367 SDValue Lo64 =
2368 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2369 Chain = Lo64.getValue(R: 1);
2370
2371 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2372 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2373 SDValue Hi64 = DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: Ptr8,
2374 PtrInfo: PtrInfo.getWithOffset(O: 8), Alignment: Align(8));
2375
2376 SDValue LoPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Lo64);
2377 SDValue HiPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Hi64);
2378 SDValue Pred =
2379 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v128i1, N1: LoPred, N2: HiPred);
2380
2381 SDValue Ops[] = {Pred, Hi64.getValue(R: 1)};
2382 return DAG.getMergeValues(Ops, dl);
2383 }
2384
2385 return SDValue();
2386}
2387
2388SDValue
2389HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
2390 // Sign- and zero-extends are legal.
2391 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
2392 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL: SDLoc(Op), VT: ty(Op),
2393 Operand: Op.getOperand(i: 0));
2394}
2395
2396SDValue
2397HexagonTargetLowering::LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const {
2398 MVT ResTy = ty(Op);
2399 if (ResTy.getVectorElementType() != MVT::i1)
2400 return Op;
2401
2402 const SDLoc &dl(Op);
2403 unsigned HwLen = Subtarget.getVectorLength();
2404 unsigned VecLen = ResTy.getVectorNumElements();
2405 assert(HwLen % VecLen == 0);
2406 unsigned ElemSize = HwLen / VecLen;
2407
2408 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemSize * 8), NumElements: VecLen);
2409 SDValue S =
2410 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: Op.getOperand(i: 0),
2411 N2: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 1)),
2412 N3: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 2)));
2413 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: S);
2414}
2415
2416SDValue
2417HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
2418 if (SDValue S = getVectorShiftByInt(Op, DAG))
2419 return S;
2420 return Op;
2421}
2422
2423SDValue
2424HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op,
2425 SelectionDAG &DAG) const {
2426 unsigned Opc = Op.getOpcode();
2427 assert(Opc == ISD::FSHL || Opc == ISD::FSHR);
2428
2429 // Make sure the shift amount is within the range of the bitwidth
2430 // of the element type.
2431 SDValue A = Op.getOperand(i: 0);
2432 SDValue B = Op.getOperand(i: 1);
2433 SDValue S = Op.getOperand(i: 2);
2434
2435 MVT InpTy = ty(Op: A);
2436 MVT ElemTy = InpTy.getVectorElementType();
2437
2438 const SDLoc &dl(Op);
2439 unsigned ElemWidth = ElemTy.getSizeInBits();
2440 bool IsLeft = Opc == ISD::FSHL;
2441
2442 // The expansion into regular shifts produces worse code for i8 and for
2443 // right shift of i32 on v65+.
2444 bool UseShifts = ElemTy != MVT::i8;
2445 if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32)
2446 UseShifts = false;
2447
2448 if (SDValue SplatV = getSplatValue(Op: S, DAG); SplatV && UseShifts) {
2449 // If this is a funnel shift by a scalar, lower it into regular shifts.
2450 SDValue Mask = DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: MVT::i32);
2451 SDValue ModS =
2452 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
2453 Ops: {DAG.getZExtOrTrunc(Op: SplatV, DL: dl, VT: MVT::i32), Mask});
2454 SDValue NegS =
2455 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2456 Ops: {DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32), ModS});
2457 SDValue IsZero =
2458 DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ModS, RHS: getZero(dl, Ty: MVT::i32, DAG), Cond: ISD::SETEQ);
2459 // FSHL A, B => A << | B >>n
2460 // FSHR A, B => A <<n | B >>
2461 SDValue Part1 =
2462 DAG.getNode(Opcode: HexagonISD::VASL, DL: dl, VT: InpTy, Ops: {A, IsLeft ? ModS : NegS});
2463 SDValue Part2 =
2464 DAG.getNode(Opcode: HexagonISD::VLSR, DL: dl, VT: InpTy, Ops: {B, IsLeft ? NegS : ModS});
2465 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Part1, Part2});
2466 // If the shift amount was 0, pick A or B, depending on the direction.
2467 // The opposite shift will also be by 0, so the "Or" will be incorrect.
2468 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InpTy, Ops: {IsZero, (IsLeft ? A : B), Or});
2469 }
2470
2471 SDValue Mask = DAG.getSplatBuildVector(
2472 VT: InpTy, DL: dl, Op: DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: ElemTy));
2473
2474 unsigned MOpc = Opc == ISD::FSHL ? HexagonISD::MFSHL : HexagonISD::MFSHR;
2475 return DAG.getNode(Opcode: MOpc, DL: dl, VT: ty(Op),
2476 Ops: {A, B, DAG.getNode(Opcode: ISD::AND, DL: dl, VT: InpTy, Ops: {S, Mask})});
2477}
2478
2479SDValue
2480HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
2481 const SDLoc &dl(Op);
2482 unsigned IntNo = Op.getConstantOperandVal(i: 0);
2483 SmallVector<SDValue> Ops(Op->ops());
2484
2485 auto Swap = [&](SDValue P) {
2486 return DAG.getMergeValues(Ops: {P.getValue(R: 1), P.getValue(R: 0)}, dl);
2487 };
2488
2489 switch (IntNo) {
2490 case Intrinsic::hexagon_V6_pred_typecast:
2491 case Intrinsic::hexagon_V6_pred_typecast_128B: {
2492 MVT ResTy = ty(Op), InpTy = ty(Op: Ops[1]);
2493 if (isHvxBoolTy(Ty: ResTy) && isHvxBoolTy(Ty: InpTy)) {
2494 if (ResTy == InpTy)
2495 return Ops[1];
2496 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: ResTy, Operand: Ops[1]);
2497 }
2498 break;
2499 }
2500 case Intrinsic::hexagon_V6_vmpyss_parts:
2501 case Intrinsic::hexagon_V6_vmpyss_parts_128B:
2502 return Swap(DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2503 Ops: {Ops[1], Ops[2]}));
2504 case Intrinsic::hexagon_V6_vmpyuu_parts:
2505 case Intrinsic::hexagon_V6_vmpyuu_parts_128B:
2506 return Swap(DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2507 Ops: {Ops[1], Ops[2]}));
2508 case Intrinsic::hexagon_V6_vmpyus_parts:
2509 case Intrinsic::hexagon_V6_vmpyus_parts_128B: {
2510 return Swap(DAG.getNode(Opcode: HexagonISD::USMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2511 Ops: {Ops[1], Ops[2]}));
2512 }
2513 } // switch
2514
2515 return Op;
2516}
2517
2518SDValue
2519HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const {
2520 const SDLoc &dl(Op);
2521 unsigned HwLen = Subtarget.getVectorLength();
2522 MachineFunction &MF = DAG.getMachineFunction();
2523 auto *MaskN = cast<MaskedLoadStoreSDNode>(Val: Op.getNode());
2524 SDValue Mask = MaskN->getMask();
2525 SDValue Chain = MaskN->getChain();
2526 SDValue Base = MaskN->getBasePtr();
2527 auto *MemOp = MF.getMachineMemOperand(MMO: MaskN->getMemOperand(), Offset: 0, Size: HwLen);
2528
2529 unsigned Opc = Op->getOpcode();
2530 assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE);
2531
2532 if (Opc == ISD::MLOAD) {
2533 MVT ValTy = ty(Op);
2534 SDValue Load = DAG.getLoad(VT: ValTy, dl, Chain, Ptr: Base, MMO: MemOp);
2535 SDValue Thru = cast<MaskedLoadSDNode>(Val: MaskN)->getPassThru();
2536 if (isUndef(Op: Thru))
2537 return Load;
2538 SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: Mask, N2: Load, N3: Thru);
2539 return DAG.getMergeValues(Ops: {VSel, Load.getValue(R: 1)}, dl);
2540 }
2541
2542 // MSTORE
2543 // HVX only has aligned masked stores.
2544
2545 // TODO: Fold negations of the mask into the store.
2546 unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
2547 SDValue Value = cast<MaskedStoreSDNode>(Val: MaskN)->getValue();
2548 SDValue Offset0 = DAG.getTargetConstant(Val: 0, DL: dl, VT: ty(Op: Base));
2549
2550 if (MaskN->getAlign().value() % HwLen == 0) {
2551 SDValue Store = getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2552 Ops: {Mask, Base, Offset0, Value, Chain}, DAG);
2553 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Store.getNode()), NewMemRefs: {MemOp});
2554 return Store;
2555 }
2556
2557 // Unaligned case.
2558 auto StoreAlign = [&](SDValue V, SDValue A) {
2559 SDValue Z = getZero(dl, Ty: ty(Op: V), DAG);
2560 // TODO: use funnel shifts?
2561 // vlalign(Vu,Vv,Rt) rotates the pair Vu:Vv left by Rt and takes the
2562 // upper half.
2563 SDValue LoV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {V, Z, A}, DAG);
2564 SDValue HiV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {Z, V, A}, DAG);
2565 return std::make_pair(x&: LoV, y&: HiV);
2566 };
2567
2568 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2569 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
2570 SDValue MaskV = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Mask);
2571 VectorPair Tmp = StoreAlign(MaskV, Base);
2572 VectorPair MaskU = {DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.first),
2573 DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.second)};
2574 VectorPair ValueU = StoreAlign(Value, Base);
2575
2576 SDValue Offset1 = DAG.getTargetConstant(Val: HwLen, DL: dl, VT: MVT::i32);
2577 SDValue StoreLo =
2578 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2579 Ops: {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
2580 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreLo.getNode()), NewMemRefs: {MemOp});
2581
2582 // If the store fits within one HwLen-aligned block, the high half's predicate
2583 // is always all-zeros and the vmem(Base+HwLen) can be elided entirely.
2584 // Proof: addr % StoreAlign == 0 and StoreMemSize <= StoreAlign implies
2585 // addr % HwLen <= HwLen - StoreAlign, so addr % HwLen + StoreMemSize
2586 // <= HwLen.
2587 // Without this guard, Hexagon v73+ probes the TLB for vmem(Base+HwLen) even
2588 // when the predicate is all-zeros, causing a TLBMISS if that page is
2589 // unmapped.
2590 uint64_t StoreMemSize = MaskN->getMemoryVT().getStoreSize().getFixedValue();
2591 if (StoreMemSize <= MaskN->getAlign().value())
2592 return StoreLo;
2593
2594 SDValue StoreHi =
2595 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2596 Ops: {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
2597 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreHi.getNode()), NewMemRefs: {MemOp});
2598 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: {StoreLo, StoreHi});
2599}
2600
2601SDValue HexagonTargetLowering::LowerHvxFpExtend(SDValue Op,
2602 SelectionDAG &DAG) const {
2603 // This conversion only applies to QFloat. IEEE extension from f16 to f32
2604 // is legal (done via a pattern).
2605 assert(Subtarget.useHVXQFloatOps());
2606
2607 assert(Op->getOpcode() == ISD::FP_EXTEND);
2608
2609 MVT VecTy = ty(Op);
2610 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
2611 const SDLoc &dl(Op);
2612
2613 if (ArgTy == MVT::v64bf16) {
2614 MVT HalfTy = typeSplit(VecTy).first;
2615 SDValue BF16Vec = Op.getOperand(i: 0);
2616 SDValue Zeroes =
2617 getInstr(MachineOpc: Hexagon::V6_vxor, dl, Ty: HalfTy, Ops: {BF16Vec, BF16Vec}, DAG);
2618 // Interleave zero vector with the bf16 vector, with zeroes in the lower
2619 // half of each 32 bit lane, effectively extending the bf16 values to fp32
2620 // values.
2621 SDValue ShuffVec =
2622 getInstr(MachineOpc: Hexagon::V6_vshufoeh, dl, Ty: VecTy, Ops: {BF16Vec, Zeroes}, DAG);
2623 VectorPair VecPair = opSplit(Vec: ShuffVec, dl, DAG);
2624 SDValue Result = getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2625 Ops: {VecPair.second, VecPair.first,
2626 DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)},
2627 DAG);
2628 return Result;
2629 }
2630
2631 assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16);
2632
2633 SDValue F16Vec = Op.getOperand(i: 0);
2634
2635 APFloat FloatVal = APFloat(1.0f);
2636 bool Ignored;
2637 FloatVal.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
2638 SDValue Fp16Ones = DAG.getConstantFP(Val: FloatVal, DL: dl, VT: ArgTy);
2639 SDValue VmpyVec =
2640 getInstr(MachineOpc: Hexagon::V6_vmpy_qf32_hf, dl, Ty: VecTy, Ops: {F16Vec, Fp16Ones}, DAG);
2641
2642 MVT HalfTy = typeSplit(VecTy).first;
2643 VectorPair Pair = opSplit(Vec: VmpyVec, dl, DAG);
2644 SDValue LoVec =
2645 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.first}, DAG);
2646 SDValue HiVec =
2647 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.second}, DAG);
2648
2649 SDValue ShuffVec =
2650 getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2651 Ops: {HiVec, LoVec, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)}, DAG);
2652
2653 return ShuffVec;
2654}
2655
2656SDValue
2657HexagonTargetLowering::LowerHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
2658 // Catch invalid conversion ops (just in case).
2659 assert(Op.getOpcode() == ISD::FP_TO_SINT ||
2660 Op.getOpcode() == ISD::FP_TO_UINT);
2661
2662 MVT ResTy = ty(Op);
2663 MVT FpTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2664 MVT IntTy = ResTy.getVectorElementType();
2665
2666 if (Subtarget.useHVXIEEEFPOps()) {
2667 // There are only conversions from f16.
2668 if (FpTy == MVT::f16) {
2669 // Other int types aren't legal in HVX, so we shouldn't see them here.
2670 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2671 // Conversions to i8 and i16 are legal.
2672 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2673 return Op;
2674 }
2675 }
2676
2677 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2678 return EqualizeFpIntConversion(Op, DAG);
2679
2680 return ExpandHvxFpToInt(Op, DAG);
2681}
2682
2683// For vector type v32i1 uint_to_fp/sint_to_fp to v32f32:
2684// R1 = #1, R2 holds the v32i1 param
2685// V1 = vsplat(R1)
2686// V2 = vsplat(R2)
2687// Q0 = vand(V1,R1)
2688// V0.w=prefixsum(Q0)
2689// V0.w=vsub(V0.w,V1.w)
2690// V2.w = vlsr(V2.w,V0.w)
2691// V2 = vand(V2,V1)
2692// V2.sf = V2.w
2693SDValue HexagonTargetLowering::LowerHvxPred32ToFp(SDValue PredOp,
2694 SelectionDAG &DAG) const {
2695
2696 MVT ResTy = ty(Op: PredOp);
2697 const SDLoc &dl(PredOp);
2698
2699 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2700 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2701 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2702 Op1: SDValue(RegConst, 0));
2703 SDNode *PredTransfer =
2704 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2705 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2706 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2707 Op1: SDValue(PredTransfer, 0));
2708 SDNode *SplatParam = DAG.getMachineNode(
2709 Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2710 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: PredOp.getOperand(i: 0)));
2711 SDNode *Vsub =
2712 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2713 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2714 SDNode *IndexShift =
2715 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2716 Op1: SDValue(SplatParam, 0), Op2: SDValue(Vsub, 0));
2717 SDNode *MaskOff =
2718 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2719 Op1: SDValue(IndexShift, 0), Op2: SDValue(SplatConst, 0));
2720 SDNode *Convert = DAG.getMachineNode(Opcode: Hexagon::V6_vconv_sf_w, dl, VT: ResTy,
2721 Op1: SDValue(MaskOff, 0));
2722 return SDValue(Convert, 0);
2723}
2724
2725// For vector type v64i1 uint_to_fo to v64f16:
2726// i64 R32 = bitcast v64i1 R3:2 (R3:2 holds v64i1)
2727// R3 = subreg_high (R32)
2728// R2 = subreg_low (R32)
2729// R1 = #1
2730// V1 = vsplat(R1)
2731// V2 = vsplat(R2)
2732// V3 = vsplat(R3)
2733// Q0 = vand(V1,R1)
2734// V0.w=prefixsum(Q0)
2735// V0.w=vsub(V0.w,V1.w)
2736// V2.w = vlsr(V2.w,V0.w)
2737// V3.w = vlsr(V3.w,V0.w)
2738// V2 = vand(V2,V1)
2739// V3 = vand(V3,V1)
2740// V2.h = vpacke(V3.w,V2.w)
2741// V2.hf = V2.h
2742SDValue HexagonTargetLowering::LowerHvxPred64ToFp(SDValue PredOp,
2743 SelectionDAG &DAG) const {
2744
2745 MVT ResTy = ty(Op: PredOp);
2746 const SDLoc &dl(PredOp);
2747
2748 SDValue Inp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: PredOp.getOperand(i: 0));
2749 // Get the hi and lo regs
2750 SDValue HiReg =
2751 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_hi, DL: dl, VT: MVT::i32, Operand: Inp);
2752 SDValue LoReg =
2753 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_lo, DL: dl, VT: MVT::i32, Operand: Inp);
2754 // Get constant #1 and splat into vector V1
2755 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2756 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2757 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2758 Op1: SDValue(RegConst, 0));
2759 // Splat the hi and lo args
2760 SDNode *SplatHi =
2761 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2762 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: HiReg));
2763 SDNode *SplatLo =
2764 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2765 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: LoReg));
2766 // vand between splatted const and const
2767 SDNode *PredTransfer =
2768 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2769 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2770 // Get the prefixsum
2771 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2772 Op1: SDValue(PredTransfer, 0));
2773 // Get the vsub
2774 SDNode *Vsub =
2775 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2776 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2777 // Get vlsr for hi and lo
2778 SDNode *IndexShift_hi =
2779 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2780 Op1: SDValue(SplatHi, 0), Op2: SDValue(Vsub, 0));
2781 SDNode *IndexShift_lo =
2782 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2783 Op1: SDValue(SplatLo, 0), Op2: SDValue(Vsub, 0));
2784 // Get vand of hi and lo
2785 SDNode *MaskOff_hi =
2786 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2787 Op1: SDValue(IndexShift_hi, 0), Op2: SDValue(SplatConst, 0));
2788 SDNode *MaskOff_lo =
2789 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2790 Op1: SDValue(IndexShift_lo, 0), Op2: SDValue(SplatConst, 0));
2791 // Pack them
2792 SDNode *Pack =
2793 DAG.getMachineNode(Opcode: Hexagon::V6_vpackeh, dl, VT: MVT::v64i16,
2794 Op1: SDValue(MaskOff_hi, 0), Op2: SDValue(MaskOff_lo, 0));
2795 SDNode *Convert =
2796 DAG.getMachineNode(Opcode: Hexagon::V6_vconv_hf_h, dl, VT: ResTy, Op1: SDValue(Pack, 0));
2797 return SDValue(Convert, 0);
2798}
2799
2800SDValue
2801HexagonTargetLowering::LowerHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
2802 // Catch invalid conversion ops (just in case).
2803 assert(Op.getOpcode() == ISD::SINT_TO_FP ||
2804 Op.getOpcode() == ISD::UINT_TO_FP);
2805
2806 MVT ResTy = ty(Op);
2807 MVT IntTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2808 MVT FpTy = ResTy.getVectorElementType();
2809
2810 if (Op.getOpcode() == ISD::UINT_TO_FP || Op.getOpcode() == ISD::SINT_TO_FP) {
2811 if (ResTy == MVT::v32f32 && ty(Op: Op.getOperand(i: 0)) == MVT::v32i1)
2812 return LowerHvxPred32ToFp(PredOp: Op, DAG);
2813 if (ResTy == MVT::v64f16 && ty(Op: Op.getOperand(i: 0)) == MVT::v64i1)
2814 return LowerHvxPred64ToFp(PredOp: Op, DAG);
2815 }
2816
2817 if (Subtarget.useHVXIEEEFPOps()) {
2818 // There are only conversions to f16.
2819 if (FpTy == MVT::f16) {
2820 // Other int types aren't legal in HVX, so we shouldn't see them here.
2821 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2822 // i8, i16 -> f16 is legal.
2823 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2824 return Op;
2825 }
2826 }
2827
2828 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2829 return EqualizeFpIntConversion(Op, DAG);
2830
2831 return ExpandHvxIntToFp(Op, DAG);
2832}
2833
2834HexagonTargetLowering::TypePair
2835HexagonTargetLowering::typeExtendToWider(MVT Ty0, MVT Ty1) const {
2836 // Compare the widths of elements of the two types, and extend the narrower
2837 // type to match the with of the wider type. For vector types, apply this
2838 // to the element type.
2839 assert(Ty0.isVector() == Ty1.isVector());
2840
2841 MVT ElemTy0 = Ty0.getScalarType();
2842 MVT ElemTy1 = Ty1.getScalarType();
2843
2844 unsigned Width0 = ElemTy0.getSizeInBits();
2845 unsigned Width1 = ElemTy1.getSizeInBits();
2846 unsigned MaxWidth = std::max(a: Width0, b: Width1);
2847
2848 auto getScalarWithWidth = [](MVT ScalarTy, unsigned Width) {
2849 if (ScalarTy.isInteger())
2850 return MVT::getIntegerVT(BitWidth: Width);
2851 assert(ScalarTy.isFloatingPoint());
2852 return MVT::getFloatingPointVT(BitWidth: Width);
2853 };
2854
2855 MVT WideETy0 = getScalarWithWidth(ElemTy0, MaxWidth);
2856 MVT WideETy1 = getScalarWithWidth(ElemTy1, MaxWidth);
2857
2858 if (!Ty0.isVector()) {
2859 // Both types are scalars.
2860 return {WideETy0, WideETy1};
2861 }
2862
2863 // Vector types.
2864 unsigned NumElem = Ty0.getVectorNumElements();
2865 assert(NumElem == Ty1.getVectorNumElements());
2866
2867 return {MVT::getVectorVT(VT: WideETy0, NumElements: NumElem),
2868 MVT::getVectorVT(VT: WideETy1, NumElements: NumElem)};
2869}
2870
2871HexagonTargetLowering::TypePair
2872HexagonTargetLowering::typeWidenToWider(MVT Ty0, MVT Ty1) const {
2873 // Compare the numbers of elements of two vector types, and widen the
2874 // narrower one to match the number of elements in the wider one.
2875 assert(Ty0.isVector() && Ty1.isVector());
2876
2877 unsigned Len0 = Ty0.getVectorNumElements();
2878 unsigned Len1 = Ty1.getVectorNumElements();
2879 if (Len0 == Len1)
2880 return {Ty0, Ty1};
2881
2882 unsigned MaxLen = std::max(a: Len0, b: Len1);
2883 return {MVT::getVectorVT(VT: Ty0.getVectorElementType(), NumElements: MaxLen),
2884 MVT::getVectorVT(VT: Ty1.getVectorElementType(), NumElements: MaxLen)};
2885}
2886
2887MVT
2888HexagonTargetLowering::typeLegalize(MVT Ty, SelectionDAG &DAG) const {
2889 EVT LegalTy = getTypeToTransformTo(Context&: *DAG.getContext(), VT: Ty);
2890 assert(LegalTy.isSimple());
2891 return LegalTy.getSimpleVT();
2892}
2893
2894MVT
2895HexagonTargetLowering::typeWidenToHvx(MVT Ty) const {
2896 unsigned HwWidth = 8 * Subtarget.getVectorLength();
2897 assert(Ty.getSizeInBits() <= HwWidth);
2898 if (Ty.getSizeInBits() == HwWidth)
2899 return Ty;
2900
2901 MVT ElemTy = Ty.getScalarType();
2902 return MVT::getVectorVT(VT: ElemTy, NumElements: HwWidth / ElemTy.getSizeInBits());
2903}
2904
2905HexagonTargetLowering::VectorPair
2906HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B,
2907 const SDLoc &dl, bool Signed, SelectionDAG &DAG) const {
2908 // Compute A+B, return {A+B, O}, where O = vector predicate indicating
2909 // whether an overflow has occurred.
2910 MVT ResTy = ty(Op: A);
2911 assert(ResTy == ty(B));
2912 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: ResTy.getVectorNumElements());
2913
2914 if (!Signed) {
2915 // V62+ has V6_vaddcarry, but it requires input predicate, so it doesn't
2916 // save any instructions.
2917 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2918 SDValue Ovf = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Add, RHS: A, Cond: ISD::SETULT);
2919 return {Add, Ovf};
2920 }
2921
2922 // Signed overflow has happened, if:
2923 // (A, B have the same sign) and (A+B has a different sign from either)
2924 // i.e. (~A xor B) & ((A+B) xor B), then check the sign bit
2925 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2926 SDValue NotA =
2927 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {A, DAG.getAllOnesConstant(DL: dl, VT: ResTy)});
2928 SDValue Xor0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {NotA, B});
2929 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {Add, B});
2930 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy, Ops: {Xor0, Xor1});
2931 SDValue MSB =
2932 DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: getZero(dl, Ty: ResTy, DAG), Cond: ISD::SETLT);
2933 return {Add, MSB};
2934}
2935
2936HexagonTargetLowering::VectorPair
2937HexagonTargetLowering::emitHvxShiftRightRnd(SDValue Val, unsigned Amt,
2938 bool Signed, SelectionDAG &DAG) const {
2939 // Shift Val right by Amt bits, round the result to the nearest integer,
2940 // tie-break by rounding halves to even integer.
2941
2942 const SDLoc &dl(Val);
2943 MVT ValTy = ty(Op: Val);
2944
2945 // This should also work for signed integers.
2946 //
2947 // uint tmp0 = inp + ((1 << (Amt-1)) - 1);
2948 // bool ovf = (inp > tmp0);
2949 // uint rup = inp & (1 << (Amt+1));
2950 //
2951 // uint tmp1 = inp >> (Amt-1); // tmp1 == tmp2 iff
2952 // uint tmp2 = tmp0 >> (Amt-1); // the Amt-1 lower bits were all 0
2953 // uint tmp3 = tmp2 + rup;
2954 // uint frac = (tmp1 != tmp2) ? tmp2 >> 1 : tmp3 >> 1;
2955 unsigned ElemWidth = ValTy.getVectorElementType().getSizeInBits();
2956 MVT ElemTy = MVT::getIntegerVT(BitWidth: ElemWidth);
2957 MVT IntTy = tyVector(Ty: ValTy, ElemTy);
2958 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: IntTy.getVectorNumElements());
2959 unsigned ShRight = Signed ? ISD::SRA : ISD::SRL;
2960
2961 SDValue Inp = DAG.getBitcast(VT: IntTy, V: Val);
2962 SDValue LowBits = DAG.getConstant(Val: (1ull << (Amt - 1)) - 1, DL: dl, VT: IntTy);
2963
2964 SDValue AmtP1 = DAG.getConstant(Val: 1ull << Amt, DL: dl, VT: IntTy);
2965 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntTy, Ops: {Inp, AmtP1});
2966 SDValue Zero = getZero(dl, Ty: IntTy, DAG);
2967 SDValue Bit = DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: Zero, Cond: ISD::SETNE);
2968 SDValue Rup = DAG.getZExtOrTrunc(Op: Bit, DL: dl, VT: IntTy);
2969 auto [Tmp0, Ovf] = emitHvxAddWithOverflow(A: Inp, B: LowBits, dl, Signed, DAG);
2970
2971 SDValue AmtM1 = DAG.getConstant(Val: Amt - 1, DL: dl, VT: IntTy);
2972 SDValue Tmp1 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Inp, N2: AmtM1);
2973 SDValue Tmp2 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Tmp0, N2: AmtM1);
2974 SDValue Tmp3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntTy, N1: Tmp2, N2: Rup);
2975
2976 SDValue Eq = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Tmp1, RHS: Tmp2, Cond: ISD::SETEQ);
2977 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntTy);
2978 SDValue Tmp4 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp2, One});
2979 SDValue Tmp5 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp3, One});
2980 SDValue Mux = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: IntTy, Ops: {Eq, Tmp5, Tmp4});
2981 return {Mux, Ovf};
2982}
2983
2984SDValue
2985HexagonTargetLowering::emitHvxMulHsV60(SDValue A, SDValue B, const SDLoc &dl,
2986 SelectionDAG &DAG) const {
2987 MVT VecTy = ty(Op: A);
2988 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
2989 assert(VecTy.getVectorElementType() == MVT::i32);
2990
2991 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
2992
2993 // mulhs(A,B) =
2994 // = [(Hi(A)*2^16 + Lo(A)) *s (Hi(B)*2^16 + Lo(B))] >> 32
2995 // = [Hi(A)*2^16 *s Hi(B)*2^16 + Hi(A) *su Lo(B)*2^16
2996 // + Lo(A) *us (Hi(B)*2^16 + Lo(B))] >> 32
2997 // = [Hi(A) *s Hi(B)*2^32 + Hi(A) *su Lo(B)*2^16 + Lo(A) *us B] >> 32
2998 // The low half of Lo(A)*Lo(B) will be discarded (it's not added to
2999 // anything, so it cannot produce any carry over to higher bits),
3000 // so everything in [] can be shifted by 16 without loss of precision.
3001 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + Lo(A)*B >> 16] >> 16
3002 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + V6_vmpyewuh(A,B)] >> 16
3003 // The final additions need to make sure to properly maintain any carry-
3004 // out bits.
3005 //
3006 // Hi(B) Lo(B)
3007 // Hi(A) Lo(A)
3008 // --------------
3009 // Lo(B)*Lo(A) | T0 = V6_vmpyewuh(B,A) does this,
3010 // Hi(B)*Lo(A) | + dropping the low 16 bits
3011 // Hi(A)*Lo(B) | T2
3012 // Hi(B)*Hi(A)
3013
3014 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh, dl, Ty: VecTy, Ops: {B, A}, DAG);
3015 // T1 = get Hi(A) into low halves.
3016 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {A, S16}, DAG);
3017 // P0 = interleaved T1.h*B.uh (full precision product)
3018 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyhus, dl, Ty: PairTy, Ops: {T1, B}, DAG);
3019 // T2 = T1.even(h) * B.even(uh), i.e. Hi(A)*Lo(B)
3020 SDValue T2 = LoHalf(V: P0, DAG);
3021 // We need to add T0+T2, recording the carry-out, which will be 1<<16
3022 // added to the final sum.
3023 // P1 = interleaved even/odd 32-bit (unsigned) sums of 16-bit halves
3024 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3025 // P2 = interleaved even/odd 32-bit (signed) sums of 16-bit halves
3026 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vaddhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3027 // T3 = full-precision(T0+T2) >> 16
3028 // The low halves are added-unsigned, the high ones are added-signed.
3029 SDValue T3 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3030 Ops: {HiHalf(V: P2, DAG), LoHalf(V: P1, DAG), S16}, DAG);
3031 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {B, S16}, DAG);
3032 // P3 = interleaved Hi(B)*Hi(A) (full precision),
3033 // which is now Lo(T1)*Lo(T4), so we want to keep the even product.
3034 SDValue P3 = getInstr(MachineOpc: Hexagon::V6_vmpyhv, dl, Ty: PairTy, Ops: {T1, T4}, DAG);
3035 SDValue T5 = LoHalf(V: P3, DAG);
3036 // Add:
3037 SDValue T6 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {T3, T5});
3038 return T6;
3039}
3040
3041SDValue
3042HexagonTargetLowering::emitHvxMulLoHiV60(SDValue A, bool SignedA, SDValue B,
3043 bool SignedB, const SDLoc &dl,
3044 SelectionDAG &DAG) const {
3045 MVT VecTy = ty(Op: A);
3046 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3047 assert(VecTy.getVectorElementType() == MVT::i32);
3048
3049 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
3050
3051 if (SignedA && !SignedB) {
3052 // Make A:unsigned, B:signed.
3053 std::swap(a&: A, b&: B);
3054 std::swap(a&: SignedA, b&: SignedB);
3055 }
3056
3057 // Do halfword-wise multiplications for unsigned*unsigned product, then
3058 // add corrections for signed and unsigned*signed.
3059
3060 SDValue Lo, Hi;
3061
3062 // P0:lo = (uu) products of low halves of A and B,
3063 // P0:hi = (uu) products of high halves.
3064 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, B}, DAG);
3065
3066 // Swap low/high halves in B
3067 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_lvsplatw, dl, Ty: VecTy,
3068 Ops: {DAG.getConstant(Val: 0x02020202, DL: dl, VT: MVT::i32)}, DAG);
3069 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vdelta, dl, Ty: VecTy, Ops: {B, T0}, DAG);
3070 // P1 = products of even/odd halfwords.
3071 // P1:lo = (uu) products of even(A.uh) * odd(B.uh)
3072 // P1:hi = (uu) products of odd(A.uh) * even(B.uh)
3073 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, T1}, DAG);
3074
3075 // P2:lo = low halves of P1:lo + P1:hi,
3076 // P2:hi = high halves of P1:lo + P1:hi.
3077 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy,
3078 Ops: {HiHalf(V: P1, DAG), LoHalf(V: P1, DAG)}, DAG);
3079 // Still need to add the high halves of P0:lo to P2:lo
3080 SDValue T2 =
3081 getInstr(MachineOpc: Hexagon::V6_vlsrw, dl, Ty: VecTy, Ops: {LoHalf(V: P0, DAG), S16}, DAG);
3082 SDValue T3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {LoHalf(V: P2, DAG), T2});
3083
3084 // The high halves of T3 will contribute to the HI part of LOHI.
3085 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3086 Ops: {HiHalf(V: P2, DAG), T3, S16}, DAG);
3087
3088 // The low halves of P2 need to be added to high halves of the LO part.
3089 Lo = getInstr(MachineOpc: Hexagon::V6_vaslw_acc, dl, Ty: VecTy,
3090 Ops: {LoHalf(V: P0, DAG), LoHalf(V: P2, DAG), S16}, DAG);
3091 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {HiHalf(V: P0, DAG), T4});
3092
3093 if (SignedA) {
3094 assert(SignedB && "Signed A and unsigned B should have been inverted");
3095
3096 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3097 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3098 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3099 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3100 SDValue X0 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: VecTy, Ops: {Q0, B, Zero});
3101 SDValue X1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, X0, A}, DAG);
3102 Hi = getInstr(MachineOpc: Hexagon::V6_vsubw, dl, Ty: VecTy, Ops: {Hi, X1}, DAG);
3103 } else if (SignedB) {
3104 // Same correction as for mulhus:
3105 // mulhus(A.uw,B.w) = mulhu(A.uw,B.uw) - (A.w if B < 0)
3106 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3107 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3108 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3109 Hi = getInstr(MachineOpc: Hexagon::V6_vsubwq, dl, Ty: VecTy, Ops: {Q1, Hi, A}, DAG);
3110 } else {
3111 assert(!SignedA && !SignedB);
3112 }
3113
3114 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3115}
3116
3117SDValue
3118HexagonTargetLowering::emitHvxMulLoHiV62(SDValue A, bool SignedA,
3119 SDValue B, bool SignedB,
3120 const SDLoc &dl,
3121 SelectionDAG &DAG) const {
3122 MVT VecTy = ty(Op: A);
3123 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3124 assert(VecTy.getVectorElementType() == MVT::i32);
3125
3126 if (SignedA && !SignedB) {
3127 // Make A:unsigned, B:signed.
3128 std::swap(a&: A, b&: B);
3129 std::swap(a&: SignedA, b&: SignedB);
3130 }
3131
3132 // Do S*S first, then make corrections for U*S or U*U if needed.
3133 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh_64, dl, Ty: PairTy, Ops: {A, B}, DAG);
3134 SDValue P1 =
3135 getInstr(MachineOpc: Hexagon::V6_vmpyowh_64_acc, dl, Ty: PairTy, Ops: {P0, A, B}, DAG);
3136 SDValue Lo = LoHalf(V: P1, DAG);
3137 SDValue Hi = HiHalf(V: P1, DAG);
3138
3139 if (!SignedB) {
3140 assert(!SignedA && "Signed A and unsigned B should have been inverted");
3141 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3142 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3143
3144 // Mulhu(X, Y) = Mulhs(X, Y) + (X, if Y < 0) + (Y, if X < 0).
3145 // def: Pat<(VecI32 (mulhu HVI32:$A, HVI32:$B)),
3146 // (V6_vaddw (HiHalf (Muls64O $A, $B)),
3147 // (V6_vaddwq (V6_vgtw (V6_vd0), $B),
3148 // (V6_vandvqv (V6_vgtw (V6_vd0), $A), $B),
3149 // $A))>;
3150 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3151 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3152 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vandvqv, dl, Ty: VecTy, Ops: {Q0, B}, DAG);
3153 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, T0, A}, DAG);
3154 Hi = getInstr(MachineOpc: Hexagon::V6_vaddw, dl, Ty: VecTy, Ops: {Hi, T1}, DAG);
3155 } else if (!SignedA) {
3156 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3157 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3158
3159 // Mulhus(unsigned X, signed Y) = Mulhs(X, Y) + (Y, if X < 0).
3160 // def: Pat<(VecI32 (HexagonMULHUS HVI32:$A, HVI32:$B)),
3161 // (V6_vaddwq (V6_vgtw (V6_vd0), $A),
3162 // (HiHalf (Muls64O $A, $B)),
3163 // $B)>;
3164 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3165 Hi = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q0, Hi, B}, DAG);
3166 }
3167
3168 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3169}
3170
3171SDValue
3172HexagonTargetLowering::EqualizeFpIntConversion(SDValue Op, SelectionDAG &DAG)
3173 const {
3174 // Rewrite conversion between integer and floating-point in such a way that
3175 // the integer type is extended/narrowed to match the bitwidth of the
3176 // floating-point type, combined with additional integer-integer extensions
3177 // or narrowings to match the original input/result types.
3178 // E.g. f32 -> i8 ==> f32 -> i32 -> i8
3179 //
3180 // The input/result types are not required to be legal, but if they are
3181 // legal, this function should not introduce illegal types.
3182
3183 unsigned Opc = Op.getOpcode();
3184 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT ||
3185 Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3186
3187 SDValue Inp = Op.getOperand(i: 0);
3188 MVT InpTy = ty(Op: Inp);
3189 MVT ResTy = ty(Op);
3190
3191 if (InpTy == ResTy)
3192 return Op;
3193
3194 const SDLoc &dl(Op);
3195 bool Signed = Opc == ISD::FP_TO_SINT || Opc == ISD::SINT_TO_FP;
3196
3197 auto [WInpTy, WResTy] = typeExtendToWider(Ty0: InpTy, Ty1: ResTy);
3198 SDValue WInp = resizeToWidth(VecV: Inp, ResTy: WInpTy, Signed, dl, DAG);
3199 SDValue Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: WResTy, Operand: WInp);
3200 SDValue Res = resizeToWidth(VecV: Conv, ResTy, Signed, dl, DAG);
3201 return Res;
3202}
3203
3204SDValue
3205HexagonTargetLowering::ExpandHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
3206 unsigned Opc = Op.getOpcode();
3207 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT);
3208
3209 const SDLoc &dl(Op);
3210 SDValue Op0 = Op.getOperand(i: 0);
3211 MVT InpTy = ty(Op: Op0);
3212 MVT ResTy = ty(Op);
3213 assert(InpTy.changeTypeToInteger() == ResTy);
3214
3215 // At this point this is an experiment under a flag.
3216 // In arch before V81 the rounding mode is towards nearest value.
3217 // The C/C++ standard requires rounding towards zero:
3218 // C (C99 and later): ISO/IEC 9899:2018 (C18), section 6.3.1.4 — "When a
3219 // finite value of real floating type is converted to an integer type, the
3220 // fractional part is discarded (i.e., the value is truncated toward zero)."
3221 // C++: ISO/IEC 14882:2020 (C++20), section 7.3.7 — "A prvalue of a
3222 // floating-point type can be converted to a prvalue of an integer type. The
3223 // conversion truncates; that is, the fractional part is discarded."
3224 if (InpTy == MVT::v64f16) {
3225 if (Subtarget.useHVXV81Ops()) {
3226 // This is c/c++ compliant
3227 SDValue ConvVec =
3228 getInstr(MachineOpc: Hexagon::V6_vconv_h_hf_rnd, dl, Ty: ResTy, Ops: {Op0}, DAG);
3229 return ConvVec;
3230 } else if (EnableFpFastConvert) {
3231 // Vd32.h=Vu32.hf same as Q6_Vh_equals_Vhf
3232 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_h_hf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3233 return ConvVec;
3234 }
3235 } else if (EnableFpFastConvert && InpTy == MVT::v32f32) {
3236 // Vd32.w=Vu32.sf same as Q6_Vw_equals_Vsf
3237 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_w_sf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3238 return ConvVec;
3239 }
3240
3241 // int32_t conv_f32_to_i32(uint32_t inp) {
3242 // // s | exp8 | frac23
3243 //
3244 // int neg = (int32_t)inp < 0;
3245 //
3246 // // "expm1" is the actual exponent minus 1: instead of "bias", subtract
3247 // // "bias+1". When the encoded exp is "all-1" (i.e. inf/nan), this will
3248 // // produce a large positive "expm1", which will result in max u/int.
3249 // // In all IEEE formats, bias is the largest positive number that can be
3250 // // represented in bias-width bits (i.e. 011..1).
3251 // int32_t expm1 = (inp << 1) - 0x80000000;
3252 // expm1 >>= 24;
3253 //
3254 // // Always insert the "implicit 1". Subnormal numbers will become 0
3255 // // regardless.
3256 // uint32_t frac = (inp << 8) | 0x80000000;
3257 //
3258 // // "frac" is the fraction part represented as Q1.31. If it was
3259 // // interpreted as uint32_t, it would be the fraction part multiplied
3260 // // by 2^31.
3261 //
3262 // // Calculate the amount of right shift, since shifting further to the
3263 // // left would lose significant bits. Limit it to 32, because we want
3264 // // shifts by 32+ to produce 0, whereas V6_vlsrwv treats the shift
3265 // // amount as a 6-bit signed value (so 33 is same as -31, i.e. shift
3266 // // left by 31). "rsh" can be negative.
3267 // int32_t rsh = min(31 - (expm1 + 1), 32);
3268 //
3269 // frac >>= rsh; // rsh == 32 will produce 0
3270 //
3271 // // Everything up to this point is the same for conversion to signed
3272 // // unsigned integer.
3273 //
3274 // if (neg) // Only for signed int
3275 // frac = -frac; //
3276 // if (rsh <= 0 && neg) // bound = neg ? 0x80000000 : 0x7fffffff
3277 // frac = 0x80000000; // frac = rsh <= 0 ? bound : frac
3278 // if (rsh <= 0 && !neg) //
3279 // frac = 0x7fffffff; //
3280 //
3281 // if (neg) // Only for unsigned int
3282 // frac = 0; //
3283 // if (rsh < 0 && !neg) // frac = rsh < 0 ? 0x7fffffff : frac;
3284 // frac = 0x7fffffff; // frac = neg ? 0 : frac;
3285 //
3286 // return frac;
3287 // }
3288
3289 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: ResTy.getVectorElementCount());
3290
3291 // Zero = V6_vd0();
3292 // Neg = V6_vgtw(Zero, Inp);
3293 // One = V6_lvsplatw(1);
3294 // M80 = V6_lvsplatw(0x80000000);
3295 // Exp00 = V6_vaslwv(Inp, One);
3296 // Exp01 = V6_vsubw(Exp00, M80);
3297 // ExpM1 = V6_vasrw(Exp01, 24);
3298 // Frc00 = V6_vaslw(Inp, 8);
3299 // Frc01 = V6_vor(Frc00, M80);
3300 // Rsh00 = V6_vsubw(V6_lvsplatw(30), ExpM1);
3301 // Rsh01 = V6_vminw(Rsh00, V6_lvsplatw(32));
3302 // Frc02 = V6_vlsrwv(Frc01, Rsh01);
3303
3304 // if signed int:
3305 // Bnd = V6_vmux(Neg, M80, V6_lvsplatw(0x7fffffff))
3306 // Pos = V6_vgtw(Rsh01, Zero);
3307 // Frc13 = V6_vsubw(Zero, Frc02);
3308 // Frc14 = V6_vmux(Neg, Frc13, Frc02);
3309 // Int = V6_vmux(Pos, Frc14, Bnd);
3310 //
3311 // if unsigned int:
3312 // Rsn = V6_vgtw(Zero, Rsh01)
3313 // Frc23 = V6_vmux(Rsn, V6_lvsplatw(0x7fffffff), Frc02)
3314 // Int = V6_vmux(Neg, Zero, Frc23)
3315
3316 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: InpTy);
3317 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3318 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias));
3319
3320 SDValue Inp = DAG.getBitcast(VT: ResTy, V: Op0);
3321 SDValue Zero = getZero(dl, Ty: ResTy, DAG);
3322 SDValue Neg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Inp, RHS: Zero, Cond: ISD::SETLT);
3323 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: ResTy);
3324 SDValue M7F = DAG.getConstant(Val: (1ull << (ElemWidth - 1)) - 1, DL: dl, VT: ResTy);
3325 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResTy);
3326 SDValue Exp00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, One});
3327 SDValue Exp01 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Exp00, M80});
3328 SDValue MNE = DAG.getConstant(Val: ElemWidth - ExpWidth, DL: dl, VT: ResTy);
3329 SDValue ExpM1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: ResTy, Ops: {Exp01, MNE});
3330
3331 SDValue ExpW = DAG.getConstant(Val: ExpWidth, DL: dl, VT: ResTy);
3332 SDValue Frc00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, ExpW});
3333 SDValue Frc01 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ResTy, Ops: {Frc00, M80});
3334
3335 SDValue MN2 = DAG.getConstant(Val: ElemWidth - 2, DL: dl, VT: ResTy);
3336 SDValue Rsh00 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {MN2, ExpM1});
3337 SDValue MW = DAG.getConstant(Val: ElemWidth, DL: dl, VT: ResTy);
3338 SDValue Rsh01 = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ResTy, Ops: {Rsh00, MW});
3339 SDValue Frc02 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ResTy, Ops: {Frc01, Rsh01});
3340
3341 SDValue Int;
3342
3343 if (Opc == ISD::FP_TO_SINT) {
3344 SDValue Bnd = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, M80, M7F});
3345 SDValue Pos = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETGT);
3346 SDValue Frc13 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Zero, Frc02});
3347 SDValue Frc14 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, Frc13, Frc02});
3348 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Pos, Frc14, Bnd});
3349 } else {
3350 assert(Opc == ISD::FP_TO_UINT);
3351 SDValue Rsn = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETLT);
3352 SDValue Frc23 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Rsn, N2: M7F, N3: Frc02);
3353 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Neg, N2: Zero, N3: Frc23);
3354 }
3355
3356 return Int;
3357}
3358
3359SDValue
3360HexagonTargetLowering::ExpandHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
3361 unsigned Opc = Op.getOpcode();
3362 assert(Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3363
3364 const SDLoc &dl(Op);
3365 SDValue Op0 = Op.getOperand(i: 0);
3366 MVT InpTy = ty(Op: Op0);
3367 MVT ResTy = ty(Op);
3368 assert(ResTy.changeTypeToInteger() == InpTy);
3369
3370 // uint32_t vnoc1_rnd(int32_t w) {
3371 // int32_t iszero = w == 0;
3372 // int32_t isneg = w < 0;
3373 // uint32_t u = __builtin_HEXAGON_A2_abs(w);
3374 //
3375 // uint32_t norm_left = __builtin_HEXAGON_S2_cl0(u) + 1;
3376 // uint32_t frac0 = (uint64_t)u << norm_left;
3377 //
3378 // // Rounding:
3379 // uint32_t frac1 = frac0 + ((1 << 8) - 1);
3380 // uint32_t renorm = (frac0 > frac1);
3381 // uint32_t rup = (int)(frac0 << 22) < 0;
3382 //
3383 // uint32_t frac2 = frac0 >> 8;
3384 // uint32_t frac3 = frac1 >> 8;
3385 // uint32_t frac = (frac2 != frac3) ? frac3 >> 1 : (frac3 + rup) >> 1;
3386 //
3387 // int32_t exp = 32 - norm_left + renorm + 127;
3388 // exp <<= 23;
3389 //
3390 // uint32_t sign = 0x80000000 * isneg;
3391 // uint32_t f = sign | exp | frac;
3392 // return iszero ? 0 : f;
3393 // }
3394
3395 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: InpTy.getVectorElementCount());
3396 bool Signed = Opc == ISD::SINT_TO_FP;
3397
3398 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: ResTy);
3399 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3400
3401 SDValue Zero = getZero(dl, Ty: InpTy, DAG);
3402 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: InpTy);
3403 SDValue IsZero = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ);
3404 SDValue Abs = Signed ? DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: InpTy, Operand: Op0) : Op0;
3405 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: InpTy, Operand: Abs);
3406 SDValue NLeft = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Clz, One});
3407 SDValue Frac0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy, Ops: {Abs, NLeft});
3408
3409 auto [Frac, Ovf] = emitHvxShiftRightRnd(Val: Frac0, Amt: ExpWidth + 1, Signed: false, DAG);
3410 if (Signed) {
3411 SDValue IsNeg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETLT);
3412 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: InpTy);
3413 SDValue Sign = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsNeg, M80, Zero});
3414 Frac = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Sign, Frac});
3415 }
3416
3417 SDValue Rnrm = DAG.getZExtOrTrunc(Op: Ovf, DL: dl, VT: InpTy);
3418 SDValue Exp0 = DAG.getConstant(Val: ElemWidth + ExpBias, DL: dl, VT: InpTy);
3419 SDValue Exp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Rnrm, Exp0});
3420 SDValue Exp2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: InpTy, Ops: {Exp1, NLeft});
3421 SDValue Exp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy,
3422 Ops: {Exp2, DAG.getConstant(Val: FracWidth, DL: dl, VT: InpTy)});
3423 SDValue Flt0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Frac, Exp3});
3424 SDValue Flt1 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsZero, Zero, Flt0});
3425 SDValue Flt = DAG.getBitcast(VT: ResTy, V: Flt1);
3426
3427 return Flt;
3428}
3429
3430SDValue
3431HexagonTargetLowering::CreateTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3432 unsigned Opc = Op.getOpcode();
3433 unsigned TLOpc;
3434 switch (Opc) {
3435 case ISD::ANY_EXTEND:
3436 case ISD::SIGN_EXTEND:
3437 case ISD::ZERO_EXTEND:
3438 TLOpc = HexagonISD::TL_EXTEND;
3439 break;
3440 case ISD::TRUNCATE:
3441 TLOpc = HexagonISD::TL_TRUNCATE;
3442 break;
3443#ifndef NDEBUG
3444 Op.dump(&DAG);
3445#endif
3446 llvm_unreachable("Unexpected operator");
3447 }
3448
3449 const SDLoc &dl(Op);
3450 return DAG.getNode(Opcode: TLOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0),
3451 N2: DAG.getUNDEF(VT: MVT::i128), // illegal type
3452 N3: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
3453}
3454
3455SDValue
3456HexagonTargetLowering::RemoveTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3457 assert(Op.getOpcode() == HexagonISD::TL_EXTEND ||
3458 Op.getOpcode() == HexagonISD::TL_TRUNCATE);
3459 unsigned Opc = Op.getConstantOperandVal(i: 2);
3460 return DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: ty(Op), Operand: Op.getOperand(i: 0));
3461}
3462
3463HexagonTargetLowering::VectorPair
3464HexagonTargetLowering::SplitVectorOp(SDValue Op, SelectionDAG &DAG) const {
3465 assert(!Op.isMachineOpcode());
3466 SmallVector<SDValue, 2> OpsL, OpsH;
3467 const SDLoc &dl(Op);
3468
3469 auto SplitVTNode = [&DAG, this](const VTSDNode *N) {
3470 MVT Ty = typeSplit(VecTy: N->getVT().getSimpleVT()).first;
3471 SDValue TV = DAG.getValueType(Ty);
3472 return std::make_pair(x&: TV, y&: TV);
3473 };
3474
3475 for (SDValue A : Op.getNode()->ops()) {
3476 auto [Lo, Hi] =
3477 ty(Op: A).isVector() ? opSplit(Vec: A, dl, DAG) : std::make_pair(x&: A, y&: A);
3478 // Special case for type operand.
3479 switch (Op.getOpcode()) {
3480 case ISD::SIGN_EXTEND_INREG:
3481 case HexagonISD::SSAT:
3482 case HexagonISD::USAT:
3483 if (const auto *N = dyn_cast<const VTSDNode>(Val: A.getNode()))
3484 std::tie(args&: Lo, args&: Hi) = SplitVTNode(N);
3485 break;
3486 }
3487 OpsL.push_back(Elt: Lo);
3488 OpsH.push_back(Elt: Hi);
3489 }
3490
3491 MVT ResTy = ty(Op);
3492 MVT HalfTy = typeSplit(VecTy: ResTy).first;
3493 SDValue L = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsL);
3494 SDValue H = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsH);
3495 return {L, H};
3496}
3497
3498SDValue
3499HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
3500 auto *MemN = cast<MemSDNode>(Val: Op.getNode());
3501 unsigned MemOpc = MemN->getOpcode();
3502 EVT MemTy = MemN->getMemoryVT();
3503
3504 if ((MemOpc == ISD::STORE || MemOpc == ISD::LOAD) &&
3505 (!MemTy.isSimple() || !isHvxPairTy(Ty: MemTy.getSimpleVT())))
3506 return Op;
3507
3508 EVT ValueType;
3509 if (MemOpc == ISD::STORE)
3510 ValueType = ty(Op: cast<StoreSDNode>(Val&: Op)->getValue());
3511 else if (MemOpc == ISD::MSTORE)
3512 ValueType = ty(Op: cast<MaskedStoreSDNode>(Val&: Op)->getValue());
3513 else // ISD::LOAD, ISD::MLOAD.
3514 ValueType = MemN->getValueType(ResNo: 0);
3515
3516 EVT LoVT, HiVT;
3517 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: ValueType);
3518
3519 EVT LoMemVT, HiMemVT;
3520 bool HiIsEmpty = false;
3521 std::tie(args&: LoMemVT, args&: HiMemVT) =
3522 DAG.GetDependentSplitDestVTs(VT: MemTy, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
3523
3524 uint64_t LoSize = LoMemVT.getSizeInBits().getFixedValue() / 8;
3525 uint64_t HiSize = HiMemVT.getSizeInBits().getFixedValue() / 8;
3526
3527 const SDLoc &dl(Op);
3528 SDValue Chain = MemN->getChain();
3529 SDValue Base0 = MemN->getBasePtr();
3530 SDValue Base1 =
3531 DAG.getMemBasePlusOffset(Base: Base0, Offset: TypeSize::getFixed(ExactSize: LoSize), DL: dl);
3532
3533 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
3534 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
3535 MachineFunction &MF = DAG.getMachineFunction();
3536 auto MemSize = [=](uint64_t Size) {
3537 return (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
3538 ? (uint64_t)MemoryLocation::UnknownSize
3539 : Size;
3540 };
3541 // MOp1 will not be used if HiIsEmpty for masked loads and stores (MLOAD and
3542 // MSTORE). Non-masked loads and store are always of double-vector size (see
3543 // isHvxPairTy() check above).
3544 MOp0 = MF.getMachineMemOperand(MMO, Offset: 0, Size: MemSize(LoSize));
3545 MOp1 = MF.getMachineMemOperand(MMO, Offset: LoSize, Size: MemSize(HiSize));
3546 }
3547
3548 if (MemOpc == ISD::LOAD) {
3549 assert(cast<LoadSDNode>(Op)->isUnindexed());
3550 SDValue Load0 = DAG.getLoad(VT: LoVT, dl, Chain, Ptr: Base0, MMO: MOp0);
3551 SDValue Load1 = DAG.getLoad(VT: HiVT, dl, Chain, Ptr: Base1, MMO: MOp1);
3552 return DAG.getMergeValues(
3553 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: Load0,
3554 N2: Load1),
3555 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Load0.getValue(R: 1),
3556 N2: Load1.getValue(R: 1))},
3557 dl);
3558 }
3559 if (MemOpc == ISD::STORE) {
3560 assert(cast<StoreSDNode>(Op)->isUnindexed());
3561 VectorPair Vals = opSplit(Vec: cast<StoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3562 SDValue Store0 = DAG.getStore(Chain, dl, Val: Vals.first, Ptr: Base0, MMO: MOp0);
3563 SDValue Store1 = DAG.getStore(Chain, dl, Val: Vals.second, Ptr: Base1, MMO: MOp1);
3564 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store0, N2: Store1);
3565 }
3566
3567 assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE);
3568
3569 auto MaskN = cast<MaskedLoadStoreSDNode>(Val&: Op);
3570 assert(MaskN->isUnindexed());
3571 VectorPair Masks = opSplit(Vec: MaskN->getMask(), dl, DAG);
3572 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3573
3574 if (MemOpc == ISD::MLOAD) {
3575 VectorPair Thru =
3576 opSplit(Vec: cast<MaskedLoadSDNode>(Val&: Op)->getPassThru(), dl, DAG);
3577 SDValue MLoad0 = DAG.getMaskedLoad(VT: LoVT, dl, Chain, Base: Base0, Offset,
3578 Mask: Masks.first, Src0: Thru.first, MemVT: LoMemVT, MMO: MOp0,
3579 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3580
3581 // The hi masked load has zero storage size. We therefore simply set it to
3582 // the low masked load and rely on subsequent removal from the chain as it
3583 // is unused. See DAGTypeLegalizer::SplitVecRes_MLOAD() for the same logic.
3584 SDValue MLoad1 =
3585 HiIsEmpty ? MLoad0
3586 : DAG.getMaskedLoad(VT: HiVT, dl, Chain, Base: Base1, Offset,
3587 Mask: Masks.second, Src0: Thru.second, MemVT: HiMemVT, MMO: MOp1,
3588 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3589 return DAG.getMergeValues(
3590 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: MLoad0,
3591 N2: MLoad1),
3592 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MLoad0.getValue(R: 1),
3593 N2: MLoad1.getValue(R: 1))},
3594 dl);
3595 }
3596 if (MemOpc == ISD::MSTORE) {
3597 VectorPair Vals = opSplit(Vec: cast<MaskedStoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3598 SDValue MStore0 =
3599 DAG.getMaskedStore(Chain, dl, Val: Vals.first, Base: Base0, Offset, Mask: Masks.first,
3600 MemVT: LoMemVT, MMO: MOp0, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3601 if (HiIsEmpty)
3602 return MStore0;
3603 SDValue MStore1 =
3604 DAG.getMaskedStore(Chain, dl, Val: Vals.second, Base: Base1, Offset, Mask: Masks.second,
3605 MemVT: HiMemVT, MMO: MOp1, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3606 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MStore0, N2: MStore1);
3607 }
3608
3609 std::string Name = "Unexpected operation: " + Op->getOperationName(G: &DAG);
3610 llvm_unreachable(Name.c_str());
3611}
3612
3613SDValue
3614HexagonTargetLowering::WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const {
3615 const SDLoc &dl(Op);
3616 auto *LoadN = cast<LoadSDNode>(Val: Op.getNode());
3617 assert(LoadN->isUnindexed() && "Not widening indexed loads yet");
3618 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3619 "Not widening loads of i1 yet");
3620
3621 SDValue Chain = LoadN->getChain();
3622 SDValue Base = LoadN->getBasePtr();
3623 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3624
3625 MVT ResTy = ty(Op);
3626 unsigned HwLen = Subtarget.getVectorLength();
3627 unsigned ResLen = ResTy.getStoreSize();
3628 assert(ResLen < HwLen && "vsetq(v1) prerequisite");
3629
3630 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3631 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3632 Ops: {DAG.getConstant(Val: ResLen, DL: dl, VT: MVT::i32)}, DAG);
3633
3634 MVT LoadTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
3635 MachineFunction &MF = DAG.getMachineFunction();
3636 auto *MemOp = MF.getMachineMemOperand(MMO: LoadN->getMemOperand(), Offset: 0, Size: HwLen);
3637
3638 SDValue Load = DAG.getMaskedLoad(VT: LoadTy, dl, Chain, Base, Offset, Mask,
3639 Src0: DAG.getUNDEF(VT: LoadTy), MemVT: LoadTy, MMO: MemOp,
3640 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3641 SDValue Value = opCastElem(Vec: Load, ElemTy: ResTy.getVectorElementType(), DAG);
3642 return DAG.getMergeValues(Ops: {Value, Load.getValue(R: 1)}, dl);
3643}
3644
3645SDValue
3646HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const {
3647 const SDLoc &dl(Op);
3648 auto *StoreN = cast<StoreSDNode>(Val: Op.getNode());
3649 assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
3650 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3651 "Not widening stores of i1 yet");
3652
3653 SDValue Chain = StoreN->getChain();
3654 SDValue Base = StoreN->getBasePtr();
3655 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3656
3657 SDValue Value = opCastElem(Vec: StoreN->getValue(), ElemTy: MVT::i8, DAG);
3658 MVT ValueTy = ty(Op: Value);
3659 unsigned ValueLen = ValueTy.getVectorNumElements();
3660 unsigned HwLen = Subtarget.getVectorLength();
3661 assert(isPowerOf2_32(ValueLen));
3662
3663 for (unsigned Len = ValueLen; Len < HwLen; ) {
3664 Value = opJoin(Ops: {Value, DAG.getUNDEF(VT: ty(Op: Value))}, dl, DAG);
3665 Len = ty(Op: Value).getVectorNumElements(); // This is Len *= 2
3666 }
3667 assert(ty(Value).getVectorNumElements() == HwLen); // Paranoia
3668
3669 assert(ValueLen < HwLen && "vsetq(v1) prerequisite");
3670 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3671 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3672 Ops: {DAG.getConstant(Val: ValueLen, DL: dl, VT: MVT::i32)}, DAG);
3673 MachineFunction &MF = DAG.getMachineFunction();
3674 auto *MemOp = MF.getMachineMemOperand(MMO: StoreN->getMemOperand(), Offset: 0, Size: HwLen);
3675 return DAG.getMaskedStore(Chain, dl, Val: Value, Base, Offset, Mask,
3676 MemVT: StoreN->getMemoryVT(), MMO: MemOp, AM: ISD::UNINDEXED, IsTruncating: false,
3677 IsCompressing: false);
3678}
3679
3680SDValue
3681HexagonTargetLowering::WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const {
3682 const SDLoc &dl(Op);
3683 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
3684 MVT ElemTy = ty(Op: Op0).getVectorElementType();
3685 unsigned HwLen = Subtarget.getVectorLength();
3686
3687 unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits();
3688 assert(WideOpLen * ElemTy.getSizeInBits() == 8 * HwLen);
3689 MVT WideOpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideOpLen);
3690 if (!Subtarget.isHVXVectorType(VecTy: WideOpTy, IncludeBool: true))
3691 return SDValue();
3692
3693 SDValue WideOp0 = appendUndef(Val: Op0, ResTy: WideOpTy, DAG);
3694 SDValue WideOp1 = appendUndef(Val: Op1, ResTy: WideOpTy, DAG);
3695 EVT ResTy =
3696 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: WideOpTy);
3697 SDValue SetCC = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: ResTy,
3698 Ops: {WideOp0, WideOp1, Op.getOperand(i: 2)});
3699
3700 EVT RetTy = typeLegalize(Ty: ty(Op), DAG);
3701 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3702 Ops: {SetCC, getZero(dl, Ty: MVT::i32, DAG)});
3703}
3704
3705SDValue HexagonTargetLowering::WidenHvxTruncateToBool(SDValue Op,
3706 SelectionDAG &DAG) const {
3707 // Handle truncation to boolean vector where the result boolean type
3708 // needs widening (e.g., v16i32 -> v16i1 where v16i1 is not a standard
3709 // HVX predicate type, or v16i8 -> v16i1 in 128-byte mode).
3710 // Widen the input to HVX width, perform the truncate to the widened
3711 // boolean type, then extract the result.
3712 const SDLoc &dl(Op);
3713 SDValue Inp = Op.getOperand(i: 0);
3714 MVT InpTy = ty(Op: Inp);
3715 MVT ResTy = ty(Op);
3716
3717 assert(ResTy.getVectorElementType() == MVT::i1 &&
3718 "Expected boolean result type");
3719
3720 MVT ElemTy = InpTy.getVectorElementType();
3721 unsigned HwLen = Subtarget.getVectorLength();
3722
3723 // Calculate the widened input type that fills the HVX register.
3724 unsigned WideLen = (8 * HwLen) / ElemTy.getSizeInBits();
3725 MVT WideInpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideLen);
3726 if (!Subtarget.isHVXVectorType(VecTy: WideInpTy, IncludeBool: false))
3727 return SDValue();
3728
3729 // Widen the input to HVX width.
3730 SDValue WideInp = appendUndef(Val: Inp, ResTy: WideInpTy, DAG);
3731
3732 // Perform the truncate to widened boolean type.
3733 MVT WideBoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: WideLen);
3734 SDValue WideTrunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: WideBoolTy, Operand: WideInp);
3735
3736 // Extract the result.
3737 EVT RetTy = typeLegalize(Ty: ResTy, DAG);
3738 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3739 Ops: {WideTrunc, getZero(dl, Ty: MVT::i32, DAG)});
3740}
3741
3742SDValue
3743HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
3744 unsigned Opc = Op.getOpcode();
3745 bool IsPairOp = isHvxPairTy(Ty: ty(Op)) ||
3746 llvm::any_of(Range: Op.getNode()->ops(), P: [this] (SDValue V) {
3747 return isHvxPairTy(Ty: ty(Op: V));
3748 });
3749
3750 if (IsPairOp) {
3751 switch (Opc) {
3752 default:
3753 break;
3754 case ISD::LOAD:
3755 case ISD::STORE:
3756 case ISD::MLOAD:
3757 case ISD::MSTORE:
3758 return SplitHvxMemOp(Op, DAG);
3759 case ISD::SINT_TO_FP:
3760 case ISD::UINT_TO_FP:
3761 case ISD::FP_TO_SINT:
3762 case ISD::FP_TO_UINT:
3763 if (ty(Op).getSizeInBits() == ty(Op: Op.getOperand(i: 0)).getSizeInBits())
3764 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3765 break;
3766 case ISD::ABS:
3767 case ISD::CTPOP:
3768 case ISD::CTLZ:
3769 case ISD::CTTZ:
3770 case ISD::MUL:
3771 case ISD::FADD:
3772 case ISD::FSUB:
3773 case ISD::FMUL:
3774 case ISD::FMINIMUMNUM:
3775 case ISD::FMAXIMUMNUM:
3776 case ISD::FMINIMUM:
3777 case ISD::FMAXIMUM:
3778 case ISD::FMINNUM:
3779 case ISD::FMAXNUM:
3780 case ISD::MULHS:
3781 case ISD::MULHU:
3782 case ISD::AND:
3783 case ISD::OR:
3784 case ISD::XOR:
3785 case ISD::SRA:
3786 case ISD::SHL:
3787 case ISD::SRL:
3788 case ISD::FSHL:
3789 case ISD::FSHR:
3790 case ISD::SMIN:
3791 case ISD::SMAX:
3792 case ISD::UMIN:
3793 case ISD::UMAX:
3794 case ISD::SETCC:
3795 case ISD::VSELECT:
3796 case ISD::SIGN_EXTEND_INREG:
3797 case ISD::SPLAT_VECTOR:
3798 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3799 case ISD::SIGN_EXTEND:
3800 case ISD::ZERO_EXTEND:
3801 // In general, sign- and zero-extends can't be split and still
3802 // be legal. The only exception is extending bool vectors.
3803 if (ty(Op: Op.getOperand(i: 0)).getVectorElementType() == MVT::i1)
3804 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3805 break;
3806 }
3807 }
3808
3809 switch (Opc) {
3810 default:
3811 break;
3812 // clang-format off
3813 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
3814 case ISD::SPLAT_VECTOR: return LowerHvxSplatVector(Op, DAG);
3815 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
3816 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
3817 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
3818 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
3819 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
3820 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
3821 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
3822 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
3823 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
3824 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
3825 case ISD::SELECT: return LowerHvxSelect(Op, DAG);
3826 case ISD::SRA:
3827 case ISD::SHL:
3828 case ISD::SRL: return LowerHvxShift(Op, DAG);
3829 case ISD::FSHL:
3830 case ISD::FSHR: return LowerHvxFunnelShift(Op, DAG);
3831 case ISD::MULHS:
3832 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
3833 case ISD::SMUL_LOHI:
3834 case ISD::UMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3835 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
3836 case ISD::SETCC: {
3837 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
3838 if (CC == ISD::SETOEQ &&
3839 ty(Op: Op.getOperand(i: 0)).getScalarType().isFloatingPoint())
3840 return LowerHvxFpSetoeq(Op, DAG);
3841 return Op;
3842 }
3843 case ISD::INTRINSIC_VOID: return Op;
3844 case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG);
3845 case ISD::MLOAD:
3846 case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG);
3847 // Unaligned loads will be handled by the default lowering.
3848 case ISD::LOAD: return LowerHvxLoad(Op, DAG);
3849 case ISD::STORE: return LowerHvxStore(Op, DAG);
3850 case ISD::FP_EXTEND: return LowerHvxFpExtend(Op, DAG);
3851 case ISD::FP_TO_SINT:
3852 case ISD::FP_TO_UINT: return LowerHvxFpToInt(Op, DAG);
3853 case ISD::SINT_TO_FP:
3854 case ISD::UINT_TO_FP: return LowerHvxIntToFp(Op, DAG);
3855
3856 // Special nodes:
3857 case HexagonISD::SMUL_LOHI:
3858 case HexagonISD::UMUL_LOHI:
3859 case HexagonISD::USMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3860
3861 case ISD::PARTIAL_REDUCE_SMLA:
3862 case ISD::PARTIAL_REDUCE_UMLA:
3863 case ISD::PARTIAL_REDUCE_SUMLA:
3864 return LowerHvxPartialReduceMLA(Op, DAG);
3865 case ISD::VECREDUCE_FMIN:
3866 return LowerHvxVecReduceFMin(Op, DAG);
3867 case ISD::VECREDUCE_FMAX:
3868 return LowerHvxVecReduceFMax(Op, DAG);
3869 case ISD::VECREDUCE_FMINIMUM:
3870 return LowerHvxVecReduceFMinimum(Op, DAG);
3871 case ISD::VECREDUCE_FMAXIMUM:
3872 return LowerHvxVecReduceFMaximum(Op, DAG);
3873 case ISD::FMINNUM:
3874 return LowerHvxFMinNum(Op, DAG);
3875 case ISD::FMAXNUM:
3876 return LowerHvxFMaxNum(Op, DAG);
3877 // clang-format on
3878 }
3879#ifndef NDEBUG
3880 Op.dumpr(&DAG);
3881#endif
3882 llvm_unreachable("Unhandled HVX operation");
3883}
3884
3885SDValue
3886HexagonTargetLowering::ExpandHvxResizeIntoSteps(SDValue Op, SelectionDAG &DAG)
3887 const {
3888 // Rewrite the extension/truncation/saturation op into steps where each
3889 // step changes the type widths by a factor of 2.
3890 // E.g. i8 -> i16 remains unchanged, but i8 -> i32 ==> i8 -> i16 -> i32.
3891 //
3892 // Some of the vector types in Op may not be legal.
3893
3894 unsigned Opc = Op.getOpcode();
3895 switch (Opc) {
3896 case HexagonISD::SSAT:
3897 case HexagonISD::USAT:
3898 case HexagonISD::TL_EXTEND:
3899 case HexagonISD::TL_TRUNCATE:
3900 break;
3901 case ISD::ANY_EXTEND:
3902 case ISD::ZERO_EXTEND:
3903 case ISD::SIGN_EXTEND:
3904 case ISD::TRUNCATE:
3905 llvm_unreachable("ISD:: ops will be auto-folded");
3906 break;
3907#ifndef NDEBUG
3908 Op.dump(&DAG);
3909#endif
3910 llvm_unreachable("Unexpected operation");
3911 }
3912
3913 SDValue Inp = Op.getOperand(i: 0);
3914 MVT InpTy = ty(Op: Inp);
3915 MVT ResTy = ty(Op);
3916
3917 unsigned InpWidth = InpTy.getVectorElementType().getSizeInBits();
3918 unsigned ResWidth = ResTy.getVectorElementType().getSizeInBits();
3919 assert(InpWidth != ResWidth);
3920
3921 if (InpWidth == 2 * ResWidth || ResWidth == 2 * InpWidth)
3922 return Op;
3923
3924 const SDLoc &dl(Op);
3925 unsigned NumElems = InpTy.getVectorNumElements();
3926 assert(NumElems == ResTy.getVectorNumElements());
3927
3928 auto repeatOp = [&](unsigned NewWidth, SDValue Arg) {
3929 MVT Ty = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: NewWidth), NumElements: NumElems);
3930 switch (Opc) {
3931 case HexagonISD::SSAT:
3932 case HexagonISD::USAT:
3933 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, DAG.getValueType(Ty)});
3934 case HexagonISD::TL_EXTEND:
3935 case HexagonISD::TL_TRUNCATE:
3936 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, Op.getOperand(i: 1), Op.getOperand(i: 2)});
3937 default:
3938 llvm_unreachable("Unexpected opcode");
3939 }
3940 };
3941
3942 SDValue S = Inp;
3943 if (InpWidth < ResWidth) {
3944 assert(ResWidth % InpWidth == 0 && isPowerOf2_32(ResWidth / InpWidth));
3945 while (InpWidth * 2 <= ResWidth)
3946 S = repeatOp(InpWidth *= 2, S);
3947 } else {
3948 // InpWidth > ResWidth
3949 assert(InpWidth % ResWidth == 0 && isPowerOf2_32(InpWidth / ResWidth));
3950 while (InpWidth / 2 >= ResWidth)
3951 S = repeatOp(InpWidth /= 2, S);
3952 }
3953 return S;
3954}
3955
3956SDValue
3957HexagonTargetLowering::LegalizeHvxResize(SDValue Op, SelectionDAG &DAG) const {
3958 SDValue Inp0 = Op.getOperand(i: 0);
3959 MVT InpTy = ty(Op: Inp0);
3960 MVT ResTy = ty(Op);
3961 unsigned InpWidth = InpTy.getSizeInBits();
3962 unsigned ResWidth = ResTy.getSizeInBits();
3963 unsigned Opc = Op.getOpcode();
3964
3965 if (shouldWidenToHvx(Ty: InpTy, DAG) || shouldWidenToHvx(Ty: ResTy, DAG)) {
3966 // First, make sure that the narrower type is widened to HVX.
3967 // This may cause the result to be wider than what the legalizer
3968 // expects, so insert EXTRACT_SUBVECTOR to bring it back to the
3969 // desired type.
3970 auto [WInpTy, WResTy] =
3971 InpWidth < ResWidth ? typeWidenToWider(Ty0: typeWidenToHvx(Ty: InpTy), Ty1: ResTy)
3972 : typeWidenToWider(Ty0: InpTy, Ty1: typeWidenToHvx(Ty: ResTy));
3973 SDValue W = appendUndef(Val: Inp0, ResTy: WInpTy, DAG);
3974 SDValue S;
3975 if (Opc == HexagonISD::TL_EXTEND || Opc == HexagonISD::TL_TRUNCATE) {
3976 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: Op.getOperand(i: 1),
3977 N3: Op.getOperand(i: 2));
3978 } else {
3979 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: DAG.getValueType(WResTy));
3980 }
3981 SDValue T = ExpandHvxResizeIntoSteps(Op: S, DAG);
3982 return extractSubvector(Vec: T, SubTy: typeLegalize(Ty: ResTy, DAG), SubIdx: 0, DAG);
3983 } else if (shouldSplitToHvx(Ty: InpWidth < ResWidth ? ResTy : InpTy, DAG)) {
3984 // For multi-step extends/truncates (e.g., i8->i32), expand into
3985 // single-step operations first. Splitting a multi-step TL_EXTEND
3986 // would halve the operand type to a sub-HVX size (e.g., v128i8 ->
3987 // v64i8), creating illegal types that cause issues in the type
3988 // legalizer's map tracking. Single-step operations (e.g., i16->i32)
3989 // are safe to split because their halved operand types remain legal.
3990 SDValue T = ExpandHvxResizeIntoSteps(Op, DAG);
3991 if (T != Op)
3992 return T;
3993 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3994 } else {
3995 assert(isTypeLegal(InpTy) && isTypeLegal(ResTy));
3996 return RemoveTLWrapper(Op, DAG);
3997 }
3998 llvm_unreachable("Unexpected situation");
3999}
4000
4001void
4002HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
4003 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
4004 unsigned Opc = N->getOpcode();
4005 SDValue Op(N, 0);
4006 SDValue Inp0; // Optional first argument.
4007 if (N->getNumOperands() > 0)
4008 Inp0 = Op.getOperand(i: 0);
4009
4010 switch (Opc) {
4011 case ISD::ANY_EXTEND:
4012 case ISD::SIGN_EXTEND:
4013 case ISD::ZERO_EXTEND:
4014 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4015 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4016 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4017 }
4018 break;
4019 case ISD::TRUNCATE:
4020 // Handle truncate to boolean vector when the input is not a
4021 // standard HVX vector type (single or pair). This covers cases
4022 // where the input needs widening (e.g., v64i8 -> v64i1 in
4023 // 128-byte mode) and cases where the result boolean type itself
4024 // needs widening (e.g., v16i32 -> v16i1). When the input is
4025 // already an HVX type, tablegen patterns handle the truncation
4026 // directly (e.g., v64i16 -> v64i1 via V6_vandvrt).
4027 if (ty(Op).getVectorElementType() == MVT::i1 &&
4028 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
4029 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
4030 Results.push_back(Elt: T);
4031 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4032 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4033 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4034 }
4035 break;
4036 case ISD::SETCC:
4037 if (shouldWidenToHvx(Ty: ty(Op: Inp0), DAG)) {
4038 if (SDValue T = WidenHvxSetCC(Op, DAG))
4039 Results.push_back(Elt: T);
4040 }
4041 break;
4042 case ISD::STORE: {
4043 if (shouldWidenToHvx(Ty: ty(Op: cast<StoreSDNode>(Val: N)->getValue()), DAG)) {
4044 SDValue Store = WidenHvxStore(Op, DAG);
4045 Results.push_back(Elt: Store);
4046 }
4047 break;
4048 }
4049 case ISD::MLOAD:
4050 if (isHvxPairTy(Ty: ty(Op))) {
4051 SDValue S = SplitHvxMemOp(Op, DAG);
4052 assert(S->getOpcode() == ISD::MERGE_VALUES);
4053 Results.push_back(Elt: S.getOperand(i: 0));
4054 Results.push_back(Elt: S.getOperand(i: 1));
4055 }
4056 break;
4057 case ISD::MSTORE:
4058 if (isHvxPairTy(Ty: ty(Op: Op->getOperand(Num: 1)))) { // Stored value
4059 SDValue S = SplitHvxMemOp(Op, DAG);
4060 Results.push_back(Elt: S);
4061 }
4062 break;
4063 case ISD::SINT_TO_FP:
4064 case ISD::UINT_TO_FP:
4065 case ISD::FP_TO_SINT:
4066 case ISD::FP_TO_UINT:
4067 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4068 SDValue T = EqualizeFpIntConversion(Op, DAG);
4069 Results.push_back(Elt: T);
4070 }
4071 break;
4072 case HexagonISD::SSAT:
4073 case HexagonISD::USAT:
4074 case HexagonISD::TL_EXTEND:
4075 case HexagonISD::TL_TRUNCATE:
4076 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4077 break;
4078 default:
4079 break;
4080 }
4081}
4082
4083void
4084HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
4085 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
4086 unsigned Opc = N->getOpcode();
4087 SDValue Op(N, 0);
4088 SDValue Inp0; // Optional first argument.
4089 if (N->getNumOperands() > 0)
4090 Inp0 = Op.getOperand(i: 0);
4091
4092 switch (Opc) {
4093 case ISD::ANY_EXTEND:
4094 case ISD::SIGN_EXTEND:
4095 case ISD::ZERO_EXTEND:
4096 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4097 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4098 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4099 }
4100 break;
4101 case ISD::TRUNCATE:
4102 // Handle truncate to boolean vector when the input is not a
4103 // standard HVX vector type. See comment in LowerHvxOperationWrapper.
4104 if (ty(Op).getVectorElementType() == MVT::i1 &&
4105 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
4106 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
4107 Results.push_back(Elt: T);
4108 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4109 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4110 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4111 }
4112 break;
4113 case ISD::SETCC:
4114 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4115 if (SDValue T = WidenHvxSetCC(Op, DAG))
4116 Results.push_back(Elt: T);
4117 }
4118 break;
4119 case ISD::LOAD: {
4120 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4121 SDValue Load = WidenHvxLoad(Op, DAG);
4122 assert(Load->getOpcode() == ISD::MERGE_VALUES);
4123 Results.push_back(Elt: Load.getOperand(i: 0));
4124 Results.push_back(Elt: Load.getOperand(i: 1));
4125 }
4126 break;
4127 }
4128 case ISD::BITCAST:
4129 if (isHvxBoolTy(Ty: ty(Op: Inp0))) {
4130 SDValue C = LowerHvxBitcast(Op, DAG);
4131 Results.push_back(Elt: C);
4132 }
4133 break;
4134 case ISD::FP_TO_SINT:
4135 case ISD::FP_TO_UINT:
4136 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4137 SDValue T = EqualizeFpIntConversion(Op, DAG);
4138 Results.push_back(Elt: T);
4139 }
4140 break;
4141 case HexagonISD::SSAT:
4142 case HexagonISD::USAT:
4143 case HexagonISD::TL_EXTEND:
4144 case HexagonISD::TL_TRUNCATE:
4145 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4146 break;
4147 default:
4148 break;
4149 }
4150}
4151
4152SDValue
4153HexagonTargetLowering::combineTruncateBeforeLegal(SDValue Op,
4154 DAGCombinerInfo &DCI) const {
4155 // Simplify V:v2NiB --(bitcast)--> vNi2B --(truncate)--> vNiB
4156 // to extract-subvector (shuffle V, pick even, pick odd)
4157
4158 assert(Op.getOpcode() == ISD::TRUNCATE);
4159 SelectionDAG &DAG = DCI.DAG;
4160 const SDLoc &dl(Op);
4161
4162 if (Op.getOperand(i: 0).getOpcode() == ISD::BITCAST)
4163 return SDValue();
4164 SDValue Cast = Op.getOperand(i: 0);
4165 SDValue Src = Cast.getOperand(i: 0);
4166
4167 EVT TruncTy = Op.getValueType();
4168 EVT CastTy = Cast.getValueType();
4169 EVT SrcTy = Src.getValueType();
4170 if (SrcTy.isSimple())
4171 return SDValue();
4172 if (SrcTy.getVectorElementType() != TruncTy.getVectorElementType())
4173 return SDValue();
4174 unsigned SrcLen = SrcTy.getVectorNumElements();
4175 unsigned CastLen = CastTy.getVectorNumElements();
4176 if (2 * CastLen != SrcLen)
4177 return SDValue();
4178
4179 SmallVector<int, 128> Mask(SrcLen);
4180 for (int i = 0; i != static_cast<int>(CastLen); ++i) {
4181 Mask[i] = 2 * i;
4182 Mask[i + CastLen] = 2 * i + 1;
4183 }
4184 SDValue Deal =
4185 DAG.getVectorShuffle(VT: SrcTy, dl, N1: Src, N2: DAG.getUNDEF(VT: SrcTy), Mask);
4186 return opSplit(Vec: Deal, dl, DAG).first;
4187}
4188
4189SDValue
4190HexagonTargetLowering::combineConcatOfShuffles(SDValue Op,
4191 SelectionDAG &DAG) const {
4192 // Fold
4193 // concat (shuffle x, y, m1), (shuffle x, y, m2)
4194 // into
4195 // shuffle (concat x, y), undef, m3
4196 if (Op.getNumOperands() != 2)
4197 return SDValue();
4198
4199 const SDLoc &dl(Op);
4200 SDValue V0 = Op.getOperand(i: 0);
4201 SDValue V1 = Op.getOperand(i: 1);
4202
4203 if (V0.getOpcode() != ISD::VECTOR_SHUFFLE)
4204 return SDValue();
4205 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE)
4206 return SDValue();
4207
4208 SetVector<SDValue> Order;
4209 Order.insert(X: V0.getOperand(i: 0));
4210 Order.insert(X: V0.getOperand(i: 1));
4211 Order.insert(X: V1.getOperand(i: 0));
4212 Order.insert(X: V1.getOperand(i: 1));
4213
4214 if (Order.size() > 2)
4215 return SDValue();
4216
4217 // In ISD::VECTOR_SHUFFLE, the types of each input and the type of the
4218 // result must be the same.
4219 EVT InpTy = V0.getValueType();
4220 assert(InpTy.isVector());
4221 unsigned InpLen = InpTy.getVectorNumElements();
4222
4223 SmallVector<int, 128> LongMask;
4224 auto AppendToMask = [&](SDValue Shuffle) {
4225 auto *SV = cast<ShuffleVectorSDNode>(Val: Shuffle.getNode());
4226 ArrayRef<int> Mask = SV->getMask();
4227 SDValue X = Shuffle.getOperand(i: 0);
4228 SDValue Y = Shuffle.getOperand(i: 1);
4229 for (int M : Mask) {
4230 if (M == -1) {
4231 LongMask.push_back(Elt: M);
4232 continue;
4233 }
4234 SDValue Src = static_cast<unsigned>(M) < InpLen ? X : Y;
4235 if (static_cast<unsigned>(M) >= InpLen)
4236 M -= InpLen;
4237
4238 int OutOffset = Order[0] == Src ? 0 : InpLen;
4239 LongMask.push_back(Elt: M + OutOffset);
4240 }
4241 };
4242
4243 AppendToMask(V0);
4244 AppendToMask(V1);
4245
4246 SDValue C0 = Order.front();
4247 SDValue C1 = Order.back(); // Can be same as front
4248 EVT LongTy = InpTy.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4249
4250 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LongTy, Ops: {C0, C1});
4251 return DAG.getVectorShuffle(VT: LongTy, dl, N1: Cat, N2: DAG.getUNDEF(VT: LongTy), Mask: LongMask);
4252}
4253
4254// Reassociate concat(p1, p2, ...) into
4255// concat(concat(p1, ...), concat(pi, ...), ...)
4256// where each inner concat produces a predicate where each bit corresponds
4257// to at most BitBytes bytes.
4258// Concatenating predicates decreases the number of bytes per each predicate
4259// bit.
4260SDValue
4261HexagonTargetLowering::combineConcatOfScalarPreds(SDValue Op, unsigned BitBytes,
4262 SelectionDAG &DAG) const {
4263 const SDLoc &dl(Op);
4264 SmallVector<SDValue> Ops(Op->ops());
4265 MVT ResTy = ty(Op);
4266 MVT InpTy = ty(Op: Ops[0]);
4267 unsigned InpLen = InpTy.getVectorNumElements(); // Scalar predicate
4268 unsigned ResLen = ResTy.getVectorNumElements(); // HVX vector predicate
4269 assert(InpLen <= 8 && "Too long for scalar predicate");
4270 assert(ResLen > 8 && "Too short for HVX vector predicate");
4271
4272 unsigned Bytes = 8 / InpLen; // Bytes-per-bit in input
4273
4274 // Already in the right form?
4275 if (Bytes <= BitBytes)
4276 return Op;
4277
4278 ArrayRef<SDValue> Inputs(Ops);
4279 unsigned SliceLen = Bytes / BitBytes;
4280
4281 SmallVector<SDValue> Cats;
4282 // (8 / BitBytes) is the desired length of the result of the inner concat.
4283 MVT InnerTy = MVT::getVectorVT(VT: MVT::i1, NumElements: 8 / BitBytes);
4284 for (unsigned i = 0; i != ResLen / (8 / BitBytes); ++i) {
4285 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: InnerTy,
4286 Ops: Inputs.slice(N: SliceLen * i, M: SliceLen));
4287 Cats.push_back(Elt: Cat);
4288 }
4289
4290 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Cats);
4291}
4292
4293SDValue HexagonTargetLowering::combineConcatVectorsBeforeLegal(
4294 SDValue Op, DAGCombinerInfo &DCI) const {
4295 MVT ResTy = ty(Op);
4296 MVT ElemTy = ResTy.getVectorElementType();
4297
4298 if (ElemTy != MVT::i1) {
4299 return combineConcatOfShuffles(Op, DAG&: DCI.DAG);
4300 }
4301 return SDValue();
4302}
4303
4304// Create the inner partial reduction MLA that can be efficiently lowered. This
4305// function is used by partial and full reductions.
4306SDValue HexagonTargetLowering::createExtendingPartialReduceMLA(
4307 unsigned Opcode, EVT AccEltType, unsigned AccNumElements, EVT InputType,
4308 const SDValue &A, const SDValue &B, unsigned &RemainingReductionRatio,
4309 const SDLoc &DL, SelectionDAG &DAG) const {
4310 const auto &Subtarget = DAG.getSubtarget<HexagonSubtarget>();
4311 if (!Subtarget.useHVXOps())
4312 return SDValue();
4313
4314 EVT InputEltType = InputType.getVectorElementType();
4315
4316 // Find if an optimized instruction for the sub-reduction is available.
4317 unsigned NativeRatio;
4318 if (AccEltType == MVT::i32 && InputEltType == MVT::i8)
4319 NativeRatio = 4;
4320 else
4321 return SDValue();
4322
4323 // We only handle the case when additional reduction will be needed, i.e.
4324 // input is longer by a larger factor than the result.
4325 ElementCount InputEC = InputType.getVectorElementCount();
4326 if (!InputEC.isKnownMultipleOf(RHS: AccNumElements * NativeRatio))
4327 return SDValue();
4328
4329 unsigned InputNumElements = InputEC.getFixedValue();
4330 RemainingReductionRatio = InputNumElements / (AccNumElements * NativeRatio);
4331 if (RemainingReductionRatio == 1)
4332 return SDValue();
4333
4334 // Create a reduction by the natively supported factor.
4335 EVT IntermediateType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType,
4336 NumElements: InputNumElements / NativeRatio);
4337
4338 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: IntermediateType);
4339 return DAG.getNode(Opcode, DL, VT: IntermediateType, N1: Zero, N2: A, N3: B);
4340}
4341
4342static bool DetectExtendingMultiply(const SDValue &N, EVT ScalarType,
4343 unsigned &Opcode, SDValue &A, SDValue &B) {
4344 SDValue Mul = N;
4345 EVT AccType = Mul.getValueType(); // Vector input type after extension.
4346 if (ScalarType != AccType.getVectorElementType())
4347 return false;
4348 bool swap = false;
4349 if (Mul->getOpcode() != ISD::MUL)
4350 return false;
4351 A = Mul->getOperand(Num: 0);
4352 B = Mul->getOperand(Num: 1);
4353 if (A.getOpcode() == ISD::ZERO_EXTEND) {
4354 if (B.getOpcode() == ISD::ZERO_EXTEND)
4355 Opcode = ISD::PARTIAL_REDUCE_UMLA;
4356 else if (B.getOpcode() == ISD::SIGN_EXTEND) {
4357 swap = true;
4358 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4359 } else
4360 return false;
4361 } else if (A.getOpcode() == ISD::SIGN_EXTEND) {
4362 if (B.getOpcode() == ISD::ZERO_EXTEND)
4363 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4364 else if (B.getOpcode() == ISD::SIGN_EXTEND)
4365 Opcode = ISD::PARTIAL_REDUCE_SMLA;
4366 else
4367 return false;
4368 } else
4369 return false;
4370
4371 // Get multiplication arguments before extension.
4372 A = A->getOperand(Num: 0);
4373 B = B->getOperand(Num: 0);
4374 if (A.getValueType() != B.getValueType())
4375 return false;
4376
4377 if (swap)
4378 std::swap(a&: A, b&: B);
4379
4380 return true;
4381}
4382
4383SDValue HexagonTargetLowering::splitVecReduceAdd(SDNode *N,
4384 SelectionDAG &DAG) const {
4385 if (!Subtarget.useHVXOps())
4386 return SDValue();
4387
4388 EVT ScalarType = N->getValueType(ResNo: 0);
4389 unsigned Opcode;
4390 SDValue A, B;
4391 if (!DetectExtendingMultiply(N: N->getOperand(Num: 0), ScalarType, Opcode, A, B))
4392 return SDValue();
4393
4394 SDLoc DL(N);
4395 unsigned RemainingReductionRatio;
4396 SDValue Partial =
4397 createExtendingPartialReduceMLA(Opcode, AccEltType: ScalarType, AccNumElements: 1, InputType: A.getValueType(),
4398 A, B, RemainingReductionRatio, DL, DAG);
4399 if (!Partial)
4400 return SDValue();
4401
4402 // We could have inserted a trivial MLA and rely on the folding action,
4403 // similar to how vector_partial_reduce_add is lowered to an MLA in
4404 // SelectionDAGBuilder. However, we just replace the final result since we
4405 // have analyzed the input completely.
4406 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: ScalarType, Operand: Partial);
4407}
4408
4409// Shared helper for VECREDUCE_FMIN/FMAX/FMINIMUM/FMAXIMUM on HVX float
4410// vector types. IgnoreNaN=true (FMIN/FMAX): NaN elements are replaced with
4411// the neutral value before the reduction so they don't corrupt the result
4412// even when the hardware pairwise instruction propagates NaN.
4413// IgnoreNaN=false (FMINIMUM/FMAXIMUM): NaN propagates naturally.
4414SDValue HexagonTargetLowering::LowerHvxVecReduceFMinMax(
4415 SDValue Op, unsigned PairwiseOpc, bool IgnoreNaN, SelectionDAG &DAG) const {
4416 SDLoc DL(Op);
4417 SDValue Vec = Op.getOperand(i: 0);
4418 MVT VecTy = ty(Op: Vec);
4419 SDNodeFlags Flags = Op->getFlags();
4420 bool ShouldStripNaN = IgnoreNaN && !Flags.hasNoNaNs();
4421
4422 // Save original input before NaN stripping; needed for the all-NaN fixup.
4423 SDValue OrigVec = Vec;
4424 MVT OrigVecTy = VecTy;
4425
4426 // Replace NaN elements in V with +/-Inf so the tree reduction ignores them.
4427 bool IsMax = (Op.getOpcode() == ISD::VECREDUCE_FMAX);
4428 auto ReplaceNaN = [&](SDValue V, MVT Ty) -> SDValue {
4429 APFloat Neutral = APFloat::getInf(
4430 Sem: Ty.getVectorElementType().getFltSemantics(), /*Negative=*/IsMax);
4431 SDValue NeutralVec = DAG.getConstantFP(Val: Neutral, DL, VT: Ty);
4432 EVT BoolTy = getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: Ty);
4433 SDValue IsNaN = DAG.getSetCC(DL, VT: BoolTy, LHS: V, RHS: V, Cond: ISD::SETUO);
4434 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN, N2: NeutralVec, N3: V);
4435 };
4436
4437 // For a pair vector, strip NaN from each half before the cross-half
4438 // pairwise reduction so a NaN in one half can't corrupt the other.
4439 if (isHvxPairTy(Ty: VecTy)) {
4440 auto [Lo, Hi] = opSplit(Vec, dl: DL, DAG);
4441 MVT SingleTy = ty(Op: Lo);
4442 if (ShouldStripNaN) {
4443 Lo = ReplaceNaN(Lo, SingleTy);
4444 Hi = ReplaceNaN(Hi, SingleTy);
4445 }
4446 Vec = DAG.getNode(Opcode: PairwiseOpc, DL, VT: SingleTy, N1: Lo, N2: Hi, Flags);
4447 VecTy = SingleTy;
4448 } else if (ShouldStripNaN) {
4449 Vec = ReplaceNaN(Vec, VecTy);
4450 }
4451
4452 // Tree reduction using VROR + pairwise op. Each iteration rotates the vector
4453 // by half the remaining element count (in bytes) and takes element-wise
4454 // min/max, halving the active width until element 0 holds the result.
4455 unsigned ElemBytes = VecTy.getScalarSizeInBits() / 8;
4456 unsigned HwLen = Subtarget.getVectorLength();
4457 unsigned NumElems = HwLen / ElemBytes;
4458
4459 SDValue Curr = Vec;
4460 for (unsigned Width = NumElems / 2; Width >= 1; Width /= 2) {
4461 SDValue RotAmt = DAG.getConstant(Val: Width * ElemBytes, DL, VT: MVT::i32);
4462 SDValue Rotated = DAG.getNode(Opcode: HexagonISD::VROR, DL, VT: VecTy, N1: Curr, N2: RotAmt);
4463 Curr = DAG.getNode(Opcode: PairwiseOpc, DL, VT: VecTy, N1: Curr, N2: Rotated, Flags);
4464 }
4465
4466 // Extract element 0 as the scalar result.
4467 MVT ScalarTy = Op.getSimpleValueType();
4468 SDValue Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ScalarTy, N1: Curr,
4469 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4470
4471 // Per llvm.maxnum/minnum semantics, all-NaN input must return NaN. The
4472 // NaN-stripping above replaced every NaN with the neutral value, so an
4473 // all-NaN vector produces the neutral value instead. Fix: if no element
4474 // in the original vector was non-NaN, return NaN.
4475 if (ShouldStripNaN) {
4476 EVT BoolVecTy =
4477 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: OrigVecTy);
4478 // IsOrd[i] == 1 iff OrigVec[i] is not NaN.
4479 SDValue IsOrd = DAG.getSetCC(DL, VT: BoolVecTy, LHS: OrigVec, RHS: OrigVec, Cond: ISD::SETO);
4480 // BoolVecTy is vNi1 with exactly N bits; bitcast to an integer and
4481 // check != 0 to detect whether any element was non-NaN.
4482 unsigned NumBits = BoolVecTy.getSizeInBits();
4483 MVT IntTy = MVT::getIntegerVT(BitWidth: NumBits);
4484 SDValue IsOrdInt = DAG.getBitcast(VT: IntTy, V: IsOrd);
4485 SDValue AnyNonNaN = DAG.getSetCC(DL, VT: MVT::i1, LHS: IsOrdInt,
4486 RHS: DAG.getConstant(Val: 0, DL, VT: IntTy), Cond: ISD::SETNE);
4487 SDValue NaN = DAG.getConstantFP(Val: APFloat::getNaN(Sem: ScalarTy.getFltSemantics()),
4488 DL, VT: ScalarTy);
4489 Result = DAG.getSelect(DL, VT: ScalarTy, Cond: AnyNonNaN, LHS: Result, RHS: NaN);
4490 }
4491
4492 return Result;
4493}
4494
4495SDValue HexagonTargetLowering::LowerHvxVecReduceFMin(SDValue Op,
4496 SelectionDAG &DAG) const {
4497 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMINNUM, /*IgnoreNaN=*/true, DAG);
4498}
4499
4500SDValue HexagonTargetLowering::LowerHvxVecReduceFMax(SDValue Op,
4501 SelectionDAG &DAG) const {
4502 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMAXNUM, /*IgnoreNaN=*/true, DAG);
4503}
4504
4505SDValue
4506HexagonTargetLowering::LowerHvxVecReduceFMinimum(SDValue Op,
4507 SelectionDAG &DAG) const {
4508 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMINIMUM, /*IgnoreNaN=*/false, DAG);
4509}
4510
4511SDValue
4512HexagonTargetLowering::LowerHvxVecReduceFMaximum(SDValue Op,
4513 SelectionDAG &DAG) const {
4514 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMAXIMUM, /*IgnoreNaN=*/false, DAG);
4515}
4516
4517// Lower FMINNUM/FMAXNUM on a single HVX float vector. These ops must ignore
4518// NaN (return the non-NaN operand). Because the hardware vmin/vmax may
4519// propagate NaN, replace NaN in each operand with the neutral value (+/-Inf)
4520// before delegating to FMINIMUM/FMAXIMUM.
4521SDValue HexagonTargetLowering::LowerHvxFMinNum(SDValue Op,
4522 SelectionDAG &DAG) const {
4523 SDLoc DL(Op);
4524 auto A = Op.getOperand(i: 0), B = Op.getOperand(i: 1);
4525 auto Ty = ty(Op);
4526 auto Flags = Op->getFlags();
4527
4528 if (!Flags.hasNoNaNs()) {
4529 auto &Sem = Ty.getVectorElementType().getFltSemantics();
4530 auto PosInf = DAG.getConstantFP(Val: APFloat::getInf(Sem, Negative: false), DL, VT: Ty);
4531 auto BoolTy =
4532 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: Ty);
4533 auto IsNaN_A = DAG.getSetCC(DL, VT: BoolTy, LHS: A, RHS: A, Cond: ISD::SETUO);
4534 auto IsNaN_B = DAG.getSetCC(DL, VT: BoolTy, LHS: B, RHS: B, Cond: ISD::SETUO);
4535 // Per llvm.minnum: if both operands are NaN, return NaN. Replace NaN
4536 // with +Inf so the hardware min ignores single-operand NaN, then restore
4537 // NaN for lanes where both inputs were NaN.
4538 auto BothNaN = DAG.getNode(Opcode: ISD::AND, DL, VT: BoolTy, N1: IsNaN_A, N2: IsNaN_B);
4539 A = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_A, N2: PosInf, N3: A);
4540 B = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_B, N2: PosInf, N3: B);
4541 auto Res = DAG.getNode(Opcode: ISD::FMINIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4542 auto NaN = DAG.getConstantFP(Val: APFloat::getNaN(Sem), DL, VT: Ty);
4543 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: BothNaN, N2: NaN, N3: Res);
4544 }
4545 return DAG.getNode(Opcode: ISD::FMINIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4546}
4547
4548SDValue HexagonTargetLowering::LowerHvxFMaxNum(SDValue Op,
4549 SelectionDAG &DAG) const {
4550 SDLoc DL(Op);
4551 auto A = Op.getOperand(i: 0), B = Op.getOperand(i: 1);
4552 auto Ty = ty(Op);
4553 auto Flags = Op->getFlags();
4554
4555 if (!Flags.hasNoNaNs()) {
4556 auto &Sem = Ty.getVectorElementType().getFltSemantics();
4557 auto NegInf = DAG.getConstantFP(Val: APFloat::getInf(Sem, Negative: true), DL, VT: Ty);
4558 auto BoolTy =
4559 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: Ty);
4560 auto IsNaN_A = DAG.getSetCC(DL, VT: BoolTy, LHS: A, RHS: A, Cond: ISD::SETUO);
4561 auto IsNaN_B = DAG.getSetCC(DL, VT: BoolTy, LHS: B, RHS: B, Cond: ISD::SETUO);
4562 // Per llvm.maxnum: if both operands are NaN, return NaN. Replace NaN
4563 // with -Inf so the hardware max ignores single-operand NaN, then restore
4564 // NaN for lanes where both inputs were NaN.
4565 auto BothNaN = DAG.getNode(Opcode: ISD::AND, DL, VT: BoolTy, N1: IsNaN_A, N2: IsNaN_B);
4566 A = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_A, N2: NegInf, N3: A);
4567 B = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_B, N2: NegInf, N3: B);
4568 auto Res = DAG.getNode(Opcode: ISD::FMAXIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4569 auto NaN = DAG.getConstantFP(Val: APFloat::getNaN(Sem), DL, VT: Ty);
4570 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: BothNaN, N2: NaN, N3: Res);
4571 }
4572 return DAG.getNode(Opcode: ISD::FMAXIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4573}
4574
4575// When possible, separate an MLA reduction with extended operands but
4576// unsupported reduction factor into an extending partial reduction that
4577// can be efficiently lowered, and a follow-up partial reduction.
4578// partial_reduce_mla(a, x, y) ->
4579// partial_reduce_mla(a, partial_reduce_mla(0, x, y), 1)
4580SDValue
4581HexagonTargetLowering::splitExtendingPartialReduceMLA(SDNode *N,
4582 SelectionDAG &DAG) const {
4583 if (!Subtarget.useHVXOps())
4584 return SDValue();
4585
4586 SDValue Acc = N->getOperand(Num: 0);
4587 SDValue A = N->getOperand(Num: 1);
4588 SDValue B = N->getOperand(Num: 2);
4589 if (A.getValueType() != B.getValueType())
4590 return SDValue();
4591
4592 // The types should be declared as custom, but do not split already legal
4593 // operation.
4594 EVT AccType = Acc.getValueType();
4595 EVT InputType = A.getValueType();
4596 if (getPartialReduceMLAAction(Opc: N->getOpcode(), AccVT: AccType, InputVT: InputType) != Custom)
4597 return SDValue();
4598
4599 SDLoc DL(N);
4600 unsigned RemainingReductionRatio;
4601 SDValue Partial = createExtendingPartialReduceMLA(
4602 Opcode: N->getOpcode(), AccEltType: AccType.getVectorElementType(),
4603 AccNumElements: AccType.getVectorNumElements(), InputType, A, B, RemainingReductionRatio,
4604 DL, DAG);
4605 if (!Partial)
4606 return SDValue();
4607 assert(RemainingReductionRatio <= MaxExpandMLA);
4608
4609 // Create the reduction for the remaining ratio.
4610 EVT IntermediateType = Partial->getOperand(Num: 0).getValueType();
4611 SDValue One = DAG.getConstant(Val: 1, DL, VT: IntermediateType);
4612 return DAG.getNode(Opcode: N->getOpcode() == ISD::PARTIAL_REDUCE_UMLA
4613 ? ISD::PARTIAL_REDUCE_UMLA
4614 : ISD::PARTIAL_REDUCE_SUMLA,
4615 DL, VT: AccType, N1: Acc, N2: Partial, N3: One);
4616}
4617
4618SDValue
4619HexagonTargetLowering::LowerHvxPartialReduceMLA(SDValue Op,
4620 SelectionDAG &DAG) const {
4621 const SDLoc &DL(Op);
4622 SDValue Acc = Op.getOperand(i: 0);
4623 SDValue A = Op.getOperand(i: 1);
4624 SDValue B = Op.getOperand(i: 2);
4625
4626 // Split the input vectors into units of one HVX vector length.
4627 unsigned HwVectorSizeInBits = Subtarget.getVectorLength() * 8;
4628
4629 EVT AccType = Acc.getValueType();
4630 EVT AccEltType = AccType.getVectorElementType();
4631 unsigned AccSubvectorNumElements =
4632 HwVectorSizeInBits / AccEltType.getSizeInBits();
4633 EVT AccSubvectorType =
4634 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType, NumElements: AccSubvectorNumElements);
4635
4636 EVT InputType = A.getValueType();
4637 assert(InputType.getSizeInBits() % HwVectorSizeInBits == 0);
4638 EVT InputEltType = InputType.getVectorElementType();
4639 unsigned InputSubvectorNumElements =
4640 HwVectorSizeInBits / InputEltType.getSizeInBits();
4641 EVT InputSubvectorType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InputEltType,
4642 NumElements: InputSubvectorNumElements);
4643
4644 unsigned SubvectorNum = InputType.getFixedSizeInBits() / HwVectorSizeInBits;
4645 SmallVector<SDValue, MaxExpandMLA> Subvectors;
4646
4647 for (unsigned I = 0; I != SubvectorNum; ++I) {
4648 SDValue SubvectorAcc = DAG.getExtractSubvector(DL, VT: AccSubvectorType, Vec: Acc,
4649 Idx: I * AccSubvectorNumElements);
4650 SDValue SubvectorA = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: A,
4651 Idx: I * InputSubvectorNumElements);
4652 SDValue SubvectorB = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: B,
4653 Idx: I * InputSubvectorNumElements);
4654 SDValue SubvectorMLA = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: AccSubvectorType,
4655 N1: SubvectorAcc, N2: SubvectorA, N3: SubvectorB);
4656 Subvectors.push_back(Elt: SubvectorMLA);
4657 }
4658
4659 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: AccType, Ops: Subvectors);
4660}
4661
4662// Lower fcmp oeq on HVX float vectors for architectures before v81, which
4663// lack a dedicated floating-point equality instruction.
4664//
4665// Correct IEEE-754 semantics: oeq(a,b) is true iff a==b and neither is NaN.
4666// We use the available float-GT instruction (V6_vgtsf/V6_vgthf) for the
4667// inequality check and bit manipulation for NaN detection:
4668//
4669// oeq(a, b) = NOT(ogt(a,b) OR ogt(b,a) OR isNaN(a) OR isNaN(b))
4670//
4671// where isNaN(x) = ((int_bits(x) & AbsMask) > NaNThreshold)
4672// f32: AbsMask=0x7FFFFFFF, NaNThreshold=0x7F800000
4673// f16: AbsMask=0x7FFF, NaNThreshold=0x7C00
4674//
4675// This handles +0/-0 correctly because float-GT treats them as equal, so
4676// neither ogt(+0,-0) nor ogt(-0,+0) is ever true.
4677//
4678// Example f32 assembly (no NaNs case):
4679// q0 = vcmp.eq(v0.w, v1.w) // bitwise comparison should just work
4680//
4681// Example f32 assembly (NaN-present case):
4682// q0 = vcmp.gt(v0.sf, v1.sf) // ogt(a,b)
4683// q0 |= vcmp.gt(v1.sf, v0.sf) // |= ogt(b,a)
4684// r0 = ##0x7FFFFFFF
4685// v2 = vsplat(r0) // AbsMask broadcast
4686// r1 = ##0x7F800000
4687// v3 = vsplat(r1) // NaNThresh broadcast
4688// v4 = vand(v0, v2) // int_bits(a) & AbsMask
4689// v5 = vand(v1, v2) // int_bits(b) & AbsMask
4690// q0 |= vcmp.gt(v4.w, v3.w) // |= isNaN(a)
4691// q0 |= vcmp.gt(v5.w, v3.w) // |= isNaN(b)
4692// // q0 now holds AnyFalse; result = XOR(q0, allones) = oeq
4693SDValue HexagonTargetLowering::LowerHvxFpSetoeq(SDValue Op,
4694 SelectionDAG &DAG) const {
4695 auto ResTy = ty(Op);
4696 auto A = Op.getOperand(i: 0), B = Op->getOperand(Num: 1);
4697 MVT FloatTy = ty(Op: A);
4698 MVT ElemTy = FloatTy.getVectorElementType();
4699 bool IsF32 = (ElemTy == MVT::f32);
4700 if (!IsF32) {
4701 assert((ElemTy == MVT::f16));
4702 }
4703 const SDLoc &DL(Op);
4704 MVT IntElemTy = IsF32 ? MVT::i32 : MVT::i16;
4705 MVT IntVecTy = tyVector(Ty: FloatTy, ElemTy: IntElemTy);
4706
4707 // Under nnan semantics NaN cannot appear, so integer equality is both
4708 // correct and cheaper (one instruction vs the float-GT sequence).
4709 bool NoNaN = Op->getFlags().hasNoNaNs();
4710 if (NoNaN) {
4711 SDValue IA = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: A);
4712 SDValue IB = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: B);
4713 return DAG.getSetCC(DL, VT: ResTy, LHS: IA, RHS: IB, Cond: ISD::SETEQ);
4714 }
4715
4716 // Float GT comparisons (IEEE-754: false whenever either operand is NaN).
4717 SDValue QAgtB = DAG.getSetCC(DL, VT: ResTy, LHS: A, RHS: B, Cond: ISD::SETOGT);
4718 SDValue QBgtA = DAG.getSetCC(DL, VT: ResTy, LHS: B, RHS: A, Cond: ISD::SETOGT);
4719
4720 // OR all "false" conditions together, then invert.
4721 SDValue AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: QAgtB, N2: QBgtA);
4722
4723 // Detect NaN by checking whether the unbiased exponent/mantissa field
4724 // exceeds the largest finite value.
4725 // f32: (bits & 0x7FFFFFFF) > 0x7F800000
4726 // f16: (bits & 0x7FFF) > 0x7C00
4727 uint64_t AbsMask = IsF32 ? 0x7FFFFFFFull : 0x7FFFull;
4728 uint64_t NaNThresh = IsF32 ? 0x7F800000ull : 0x7C00ull;
4729
4730 SDValue IA = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: A);
4731 SDValue IB = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: B);
4732 SDValue MaskVec = DAG.getConstant(Val: AbsMask, DL, VT: IntVecTy);
4733 SDValue ThreshVec = DAG.getConstant(Val: NaNThresh, DL, VT: IntVecTy);
4734 SDValue QNanA =
4735 DAG.getSetCC(DL, VT: ResTy, LHS: DAG.getNode(Opcode: ISD::AND, DL, VT: IntVecTy, N1: IA, N2: MaskVec),
4736 RHS: ThreshVec, Cond: ISD::SETGT);
4737 SDValue QNanB =
4738 DAG.getSetCC(DL, VT: ResTy, LHS: DAG.getNode(Opcode: ISD::AND, DL, VT: IntVecTy, N1: IB, N2: MaskVec),
4739 RHS: ThreshVec, Cond: ISD::SETGT);
4740 AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: AnyFalse, N2: QNanA);
4741 AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: AnyFalse, N2: QNanB);
4742
4743 // Result = NOT(<Is A gt B>, <IS B gt A>, <IS A NaN>, <IS B NaN>)
4744 // Use XOR with ones to simulate logical not.
4745 return DAG.getNode(Opcode: ISD::XOR, DL, VT: ResTy, N1: AnyFalse,
4746 N2: DAG.getConstant(Val: 1, DL, VT: ResTy));
4747}
4748
4749SDValue
4750HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
4751 const {
4752 const SDLoc &dl(N);
4753 SelectionDAG &DAG = DCI.DAG;
4754 SDValue Op(N, 0);
4755 unsigned Opc = Op.getOpcode();
4756
4757 SmallVector<SDValue, 4> Ops(N->ops());
4758
4759 if (Opc == ISD::TRUNCATE)
4760 return combineTruncateBeforeLegal(Op, DCI);
4761 if (Opc == ISD::CONCAT_VECTORS)
4762 return combineConcatVectorsBeforeLegal(Op, DCI);
4763
4764 if (DCI.isBeforeLegalizeOps())
4765 return SDValue();
4766
4767 switch (Opc) {
4768 case HexagonISD::V2Q:
4769 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) {
4770 if (const auto *C = dyn_cast<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
4771 return C->isZero() ? DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: ty(Op))
4772 : DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: ty(Op));
4773 }
4774 break;
4775 case HexagonISD::Q2V:
4776 if (Ops[0].getOpcode() == HexagonISD::QTRUE)
4777 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ty(Op),
4778 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
4779 if (Ops[0].getOpcode() == HexagonISD::QFALSE)
4780 return getZero(dl, Ty: ty(Op), DAG);
4781 break;
4782 case HexagonISD::VINSERTW0:
4783 if (isUndef(Op: Ops[1]))
4784 return Ops[0];
4785 break;
4786 case HexagonISD::VROR: {
4787 if (Ops[0].getOpcode() == HexagonISD::VROR) {
4788 SDValue Vec = Ops[0].getOperand(i: 0);
4789 SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(i: 1);
4790 SDValue Rot = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ty(Op: Rot0), Ops: {Rot0, Rot1});
4791 return DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ty(Op), Ops: {Vec, Rot});
4792 }
4793 break;
4794 }
4795 }
4796
4797 return SDValue();
4798}
4799
4800bool
4801HexagonTargetLowering::shouldSplitToHvx(MVT Ty, SelectionDAG &DAG) const {
4802 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4803 return false;
4804 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4805 if (Action == TargetLoweringBase::TypeSplitVector)
4806 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4807 return false;
4808}
4809
4810bool
4811HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const {
4812 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4813 return false;
4814 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4815 if (Action == TargetLoweringBase::TypeWidenVector)
4816 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4817 return false;
4818}
4819
4820bool
4821HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const {
4822 if (!Subtarget.useHVXOps())
4823 return false;
4824 // If the type of any result, or any operand type are HVX vector types,
4825 // this is an HVX operation.
4826 auto IsHvxTy = [this](EVT Ty) {
4827 return Ty.isSimple() && Subtarget.isHVXVectorType(VecTy: Ty.getSimpleVT(), IncludeBool: true);
4828 };
4829 auto IsHvxOp = [this](SDValue Op) {
4830 return Op.getValueType().isSimple() &&
4831 Subtarget.isHVXVectorType(VecTy: ty(Op), IncludeBool: true);
4832 };
4833 if (llvm::any_of(Range: N->values(), P: IsHvxTy) || llvm::any_of(Range: N->ops(), P: IsHvxOp))
4834 return true;
4835
4836 // Check if this could be an HVX operation after type widening.
4837 auto IsWidenedToHvx = [this, &DAG](SDValue Op) {
4838 if (!Op.getValueType().isSimple())
4839 return false;
4840 MVT ValTy = ty(Op);
4841 return ValTy.isVector() && shouldWidenToHvx(Ty: ValTy, DAG);
4842 };
4843
4844 for (int i = 0, e = N->getNumValues(); i != e; ++i) {
4845 if (IsWidenedToHvx(SDValue(N, i)))
4846 return true;
4847 }
4848 return llvm::any_of(Range: N->ops(), P: IsWidenedToHvx);
4849}
4850