1//===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "HexagonISelLowering.h"
10#include "HexagonRegisterInfo.h"
11#include "HexagonSubtarget.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/MemoryLocation.h"
15#include "llvm/CodeGen/MachineBasicBlock.h"
16#include "llvm/CodeGen/MachineFunction.h"
17#include "llvm/CodeGen/MachineInstr.h"
18#include "llvm/CodeGen/MachineOperand.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/TargetInstrInfo.h"
21#include "llvm/IR/IntrinsicsHexagon.h"
22#include "llvm/Support/CommandLine.h"
23#include "llvm/Support/ErrorHandling.h"
24
25#include <algorithm>
26#include <string>
27#include <utility>
28
29using namespace llvm;
30
31static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
32 cl::Hidden, cl::init(Val: 16),
33 cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
34
35static cl::opt<bool>
36 EnableFpFastConvert("hexagon-fp-fast-convert", cl::Hidden, cl::init(Val: false),
37 cl::desc("Enable FP fast conversion routine."));
38
39static cl::opt<bool>
40 EnableXQFFastConvert("hexagon-qf-fast-convert", cl::Hidden, cl::init(Val: false),
41 cl::desc("Enable XQF fast conversion routine."));
42
43static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
44static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
45static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
46static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
47
48static const unsigned MaxExpandMLA = 8;
49
50static std::tuple<unsigned, unsigned, unsigned> getIEEEProperties(MVT Ty) {
51 // For a float scalar type, return (exp-bits, exp-bias, fraction-bits)
52 MVT ElemTy = Ty.getScalarType();
53 switch (ElemTy.SimpleTy) {
54 case MVT::f16:
55 return std::make_tuple(args: 5, args: 15, args: 10);
56 case MVT::f32:
57 return std::make_tuple(args: 8, args: 127, args: 23);
58 case MVT::f64:
59 return std::make_tuple(args: 11, args: 1023, args: 52);
60 default:
61 break;
62 }
63 llvm_unreachable(("Unexpected type: " + EVT(ElemTy).getEVTString()).c_str());
64}
65
66void
67HexagonTargetLowering::initializeHVXLowering() {
68 if (Subtarget.useHVX64BOps()) {
69 addRegisterClass(VT: MVT::v64i8, RC: &Hexagon::HvxVRRegClass);
70 addRegisterClass(VT: MVT::v32i16, RC: &Hexagon::HvxVRRegClass);
71 addRegisterClass(VT: MVT::v16i32, RC: &Hexagon::HvxVRRegClass);
72 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxWRRegClass);
73 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxWRRegClass);
74 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxWRRegClass);
75 // These "short" boolean vector types should be legal because
76 // they will appear as results of vector compares. If they were
77 // not legal, type legalization would try to make them legal
78 // and that would require using operations that do not use or
79 // produce such types. That, in turn, would imply using custom
80 // nodes, which would be unoptimizable by the DAG combiner.
81 // The idea is to rely on target-independent operations as much
82 // as possible.
83 addRegisterClass(VT: MVT::v16i1, RC: &Hexagon::HvxQRRegClass);
84 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
85 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
86 } else if (Subtarget.useHVX128BOps()) {
87 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxVRRegClass);
88 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxVRRegClass);
89 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxVRRegClass);
90 addRegisterClass(VT: MVT::v256i8, RC: &Hexagon::HvxWRRegClass);
91 addRegisterClass(VT: MVT::v128i16, RC: &Hexagon::HvxWRRegClass);
92 addRegisterClass(VT: MVT::v64i32, RC: &Hexagon::HvxWRRegClass);
93 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
94 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
95 addRegisterClass(VT: MVT::v128i1, RC: &Hexagon::HvxQRRegClass);
96 if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) {
97 addRegisterClass(VT: MVT::v32f32, RC: &Hexagon::HvxVRRegClass);
98 addRegisterClass(VT: MVT::v64f16, RC: &Hexagon::HvxVRRegClass);
99 addRegisterClass(VT: MVT::v64f32, RC: &Hexagon::HvxWRRegClass);
100 addRegisterClass(VT: MVT::v128f16, RC: &Hexagon::HvxWRRegClass);
101 }
102 if (Subtarget.useHVXV81Ops()) {
103 addRegisterClass(VT: MVT::v64bf16, RC: &Hexagon::HvxVRRegClass);
104 addRegisterClass(VT: MVT::v128bf16, RC: &Hexagon::HvxWRRegClass);
105 }
106 }
107
108 // Set up operation actions.
109
110 bool Use64b = Subtarget.useHVX64BOps();
111 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
112 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
113 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
114 MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32;
115 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
116
117 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
118 setOperationAction(Op: Opc, VT: FromTy, Action: Promote);
119 AddPromotedToType(Opc, OrigVT: FromTy, DestVT: ToTy);
120 };
121
122 // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
123 // Note: v16i1 -> i16 is handled in type legalization instead of op
124 // legalization.
125 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
126 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
127 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
128 setOperationAction(Op: ISD::BITCAST, VT: MVT::v16i1, Action: Custom);
129 setOperationAction(Op: ISD::BITCAST, VT: MVT::v128i1, Action: Custom);
130 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
131 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteV, Action: Legal);
132 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteW, Action: Legal);
133 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
134
135 if (Subtarget.useHVX128BOps()) {
136 setOperationAction(Op: ISD::BITCAST, VT: MVT::v32i1, Action: Custom);
137 setOperationAction(Op: ISD::BITCAST, VT: MVT::v64i1, Action: Custom);
138 setOperationAction(Op: ISD::STORE, VT: MVT::v32i1, Action: Custom);
139 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i1, Action: Custom);
140 setOperationAction(Op: ISD::STORE, VT: MVT::v64i1, Action: Custom);
141 setOperationAction(Op: ISD::LOAD, VT: MVT::v64i1, Action: Custom);
142 setOperationAction(Op: ISD::STORE, VT: MVT::v128i1, Action: Custom);
143 setOperationAction(Op: ISD::LOAD, VT: MVT::v128i1, Action: Custom);
144 }
145 if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() &&
146 Subtarget.useHVXFloatingPoint()) {
147
148 static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 };
149 static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 };
150
151 for (MVT T : FloatV) {
152 setOperationAction(Op: ISD::FADD, VT: T, Action: Legal);
153 setOperationAction(Op: ISD::FSUB, VT: T, Action: Legal);
154 setOperationAction(Op: ISD::FMUL, VT: T, Action: Legal);
155 setOperationAction(Op: ISD::FMINIMUMNUM, VT: T, Action: Legal);
156 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: T, Action: Legal);
157 setOperationAction(Op: ISD::FMINIMUM, VT: T, Action: Legal);
158 setOperationAction(Op: ISD::FMAXIMUM, VT: T, Action: Legal);
159 setOperationAction(Op: ISD::FMINNUM, VT: T, Action: Custom);
160 setOperationAction(Op: ISD::FMAXNUM, VT: T, Action: Custom);
161 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: T, Action: Custom);
162 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: T, Action: Custom);
163 setOperationAction(Op: ISD::VECREDUCE_FMINIMUM, VT: T, Action: Custom);
164 setOperationAction(Op: ISD::VECREDUCE_FMAXIMUM, VT: T, Action: Custom);
165
166 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
167 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
168
169 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
170 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
171
172 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
173 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
174 // Custom-lower BUILD_VECTOR. The standard (target-independent)
175 // handling of it would convert it to a load, which is not always
176 // the optimal choice.
177 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
178 }
179
180
181 // BUILD_VECTOR with f16 operands cannot be promoted without
182 // promoting the result, so lower the node to vsplat or constant pool
183 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
184 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::f16, Action: Custom);
185
186 // Vector shuffle is always promoted to ByteV and a bitcast to f16 is
187 // generated.
188 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128f16, ByteW);
189 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f16, ByteV);
190 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f32, ByteW);
191 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v32f32, ByteV);
192
193 // For HVX <v81 there is no hardware float-equality instruction; only
194 // float-GT (V6_vgtsf/V6_vgthf) is available. The integer-equality
195 // fallback (V6_veqw/V6_veqh) silently treats NaN as equal to itself
196 // because the bit patterns match. Mark SETCC as Custom for the
197 // single-vector float types so we can synthesise the correct
198 // ordered-equal predicate in LowerHvxFpSetoeq.
199 if (!Subtarget.useHVXV81Ops())
200 for (MVT T : FloatV)
201 setOperationAction(Op: ISD::SETCC, VT: T, Action: Custom);
202
203 if (Subtarget.useHVXV81Ops()) {
204 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128bf16, ByteW);
205 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64bf16, ByteV);
206 setPromoteTo(ISD::SETCC, MVT::v64bf16, MVT::v64f32);
207 setPromoteTo(ISD::FADD, MVT::v64bf16, MVT::v64f32);
208 setPromoteTo(ISD::FSUB, MVT::v64bf16, MVT::v64f32);
209 setPromoteTo(ISD::FMUL, MVT::v64bf16, MVT::v64f32);
210 setPromoteTo(ISD::FMINNUM, MVT::v64bf16, MVT::v64f32);
211 setPromoteTo(ISD::FMAXNUM, MVT::v64bf16, MVT::v64f32);
212
213 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::v64bf16, Action: Legal);
214 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
215 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
216
217 setOperationAction(Op: ISD::LOAD, VT: MVT::v128bf16, Action: Custom);
218 setOperationAction(Op: ISD::STORE, VT: MVT::v128bf16, Action: Custom);
219
220 setOperationAction(Op: ISD::MLOAD, VT: MVT::v64bf16, Action: Custom);
221 setOperationAction(Op: ISD::MSTORE, VT: MVT::v64bf16, Action: Custom);
222 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v64bf16, Action: Custom);
223 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v64bf16, Action: Custom);
224
225 setOperationAction(Op: ISD::MLOAD, VT: MVT::v128bf16, Action: Custom);
226 setOperationAction(Op: ISD::MSTORE, VT: MVT::v128bf16, Action: Custom);
227 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v128bf16, Action: Custom);
228 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v128bf16, Action: Custom);
229
230 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::bf16, Action: Custom);
231 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::bf16, Action: Custom);
232 }
233
234 for (MVT P : FloatW) {
235 setOperationAction(Op: ISD::LOAD, VT: P, Action: Custom);
236 setOperationAction(Op: ISD::STORE, VT: P, Action: Custom);
237 setOperationAction(Op: ISD::FADD, VT: P, Action: Custom);
238 setOperationAction(Op: ISD::FSUB, VT: P, Action: Custom);
239 setOperationAction(Op: ISD::FMUL, VT: P, Action: Custom);
240 setOperationAction(Op: ISD::FMINIMUMNUM, VT: P, Action: Custom);
241 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: P, Action: Custom);
242 setOperationAction(Op: ISD::FMINIMUM, VT: P, Action: Custom);
243 setOperationAction(Op: ISD::FMAXIMUM, VT: P, Action: Custom);
244 setOperationAction(Op: ISD::FMINNUM, VT: P, Action: Custom);
245 setOperationAction(Op: ISD::FMAXNUM, VT: P, Action: Custom);
246 setOperationAction(Op: ISD::VECREDUCE_FMIN, VT: P, Action: Custom);
247 setOperationAction(Op: ISD::VECREDUCE_FMAX, VT: P, Action: Custom);
248 setOperationAction(Op: ISD::VECREDUCE_FMINIMUM, VT: P, Action: Custom);
249 setOperationAction(Op: ISD::VECREDUCE_FMAXIMUM, VT: P, Action: Custom);
250 setOperationAction(Op: ISD::SETCC, VT: P, Action: Custom);
251 setOperationAction(Op: ISD::VSELECT, VT: P, Action: Custom);
252
253 // Custom-lower BUILD_VECTOR. The standard (target-independent)
254 // handling of it would convert it to a load, which is not always
255 // the optimal choice.
256 setOperationAction(Op: ISD::BUILD_VECTOR, VT: P, Action: Custom);
257 // Make concat-vectors custom to handle concats of more than 2 vectors.
258 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: P, Action: Custom);
259
260 setOperationAction(Op: ISD::MLOAD, VT: P, Action: Custom);
261 setOperationAction(Op: ISD::MSTORE, VT: P, Action: Custom);
262 }
263
264 if (Subtarget.useHVXQFloatOps()) {
265 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Custom);
266 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
267 } else if (Subtarget.useHVXIEEEFPOps()) {
268 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Legal);
269 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
270 }
271 }
272
273 for (MVT T : LegalV) {
274 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
275 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
276
277 setOperationAction(Op: ISD::ABS, VT: T, Action: Legal);
278 setOperationAction(Op: ISD::AND, VT: T, Action: Legal);
279 setOperationAction(Op: ISD::OR, VT: T, Action: Legal);
280 setOperationAction(Op: ISD::XOR, VT: T, Action: Legal);
281 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
282 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
283 setOperationAction(Op: ISD::MUL, VT: T, Action: Legal);
284 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Legal);
285 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Legal);
286 setOperationAction(Op: ISD::SELECT, VT: T, Action: Legal);
287 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
288 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
289 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
290 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
291 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
292 if (T != ByteV) {
293 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
294 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
295 setOperationAction(Op: ISD::BSWAP, VT: T, Action: Legal);
296 }
297
298 setOperationAction(Op: ISD::SMIN, VT: T, Action: Legal);
299 setOperationAction(Op: ISD::SMAX, VT: T, Action: Legal);
300 if (T.getScalarType() != MVT::i32) {
301 setOperationAction(Op: ISD::UMIN, VT: T, Action: Legal);
302 setOperationAction(Op: ISD::UMAX, VT: T, Action: Legal);
303 }
304
305 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
306 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
307 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
308 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
309 if (T.getScalarType() != MVT::i32) {
310 setOperationAction(Op: ISD::MULHS, VT: T, Action: Legal);
311 setOperationAction(Op: ISD::MULHU, VT: T, Action: Legal);
312 }
313
314 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
315 // Make concat-vectors custom to handle concats of more than 2 vectors.
316 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
317 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
318 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: T, Action: Custom);
319 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
320 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: T, Action: Custom);
321 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
322 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
323 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
324 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
325 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
326 if (T != ByteV) {
327 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
328 // HVX only has shifts of words and halfwords.
329 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
330 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
331 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
332
333 // Promote all shuffles to operate on vectors of bytes.
334 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
335 }
336
337 if (Subtarget.useHVXFloatingPoint()) {
338 // Same action for both QFloat and IEEE.
339 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
340 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
341 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
342 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
343 }
344
345 setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Expand);
346 setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Expand);
347 setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Expand);
348 setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Expand);
349 setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Expand);
350 setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Expand);
351 setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Expand);
352 }
353
354 for (MVT T : LegalW) {
355 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
356 // independent) handling of it would convert it to a load, which is
357 // not always the optimal choice.
358 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
359 // Make concat-vectors custom to handle concats of more than 2 vectors.
360 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
361
362 // Custom-lower these operations for pairs. Expand them into a concat
363 // of the corresponding operations on individual vectors.
364 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
365 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
366 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
367 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Custom);
368 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
369 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
370 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
371 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Custom);
372
373 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
374 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
375 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
376 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
377 setOperationAction(Op: ISD::ABS, VT: T, Action: Custom);
378 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Custom);
379 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
380 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Custom);
381
382 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
383 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
384 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
385 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
386 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
387 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
388 setOperationAction(Op: ISD::MUL, VT: T, Action: Custom);
389 setOperationAction(Op: ISD::MULHS, VT: T, Action: Custom);
390 setOperationAction(Op: ISD::MULHU, VT: T, Action: Custom);
391 setOperationAction(Op: ISD::AND, VT: T, Action: Custom);
392 setOperationAction(Op: ISD::OR, VT: T, Action: Custom);
393 setOperationAction(Op: ISD::XOR, VT: T, Action: Custom);
394 setOperationAction(Op: ISD::SETCC, VT: T, Action: Custom);
395 setOperationAction(Op: ISD::VSELECT, VT: T, Action: Custom);
396 if (T != ByteW) {
397 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
398 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
399 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
400
401 // Promote all shuffles to operate on vectors of bytes.
402 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
403 }
404 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
405 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
406
407 setOperationAction(Op: ISD::SMIN, VT: T, Action: Custom);
408 setOperationAction(Op: ISD::SMAX, VT: T, Action: Custom);
409 if (T.getScalarType() != MVT::i32) {
410 setOperationAction(Op: ISD::UMIN, VT: T, Action: Custom);
411 setOperationAction(Op: ISD::UMAX, VT: T, Action: Custom);
412 }
413
414 if (Subtarget.useHVXFloatingPoint()) {
415 // Same action for both QFloat and IEEE.
416 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
417 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
418 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
419 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
420 }
421 }
422
423 // Legalize all of these to HexagonISD::[SU]MUL_LOHI.
424 setOperationAction(Op: ISD::MULHS, VT: WordV, Action: Custom); // -> _LOHI
425 setOperationAction(Op: ISD::MULHU, VT: WordV, Action: Custom); // -> _LOHI
426 setOperationAction(Op: ISD::SMUL_LOHI, VT: WordV, Action: Custom);
427 setOperationAction(Op: ISD::UMUL_LOHI, VT: WordV, Action: Custom);
428
429 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v64f16, Action: Expand);
430 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v64f16, Action: Expand);
431 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v64f16, Action: Expand);
432 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v64f16, Action: Expand);
433 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v64f16, Action: Expand);
434 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v64f16, Action: Expand);
435 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v64f16, Action: Expand);
436 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v64f16, Action: Expand);
437 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v64f16, Action: Expand);
438 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v64f16, Action: Expand);
439 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v64f16, Action: Expand);
440 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v64f16, Action: Expand);
441 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v64f16, Action: Expand);
442 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v64f16, Action: Expand);
443
444 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v32f32, Action: Expand);
445 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v32f32, Action: Expand);
446 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v32f32, Action: Expand);
447 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v32f32, Action: Expand);
448 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v32f32, Action: Expand);
449 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v32f32, Action: Expand);
450 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v32f32, Action: Expand);
451 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v32f32, Action: Expand);
452 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v32f32, Action: Expand);
453 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v32f32, Action: Expand);
454 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v32f32, Action: Expand);
455 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v32f32, Action: Expand);
456 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v32f32, Action: Expand);
457 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v32f32, Action: Expand);
458
459 // Boolean vectors.
460
461 for (MVT T : LegalW) {
462 // Boolean types for vector pairs will overlap with the boolean
463 // types for single vectors, e.g.
464 // v64i8 -> v64i1 (single)
465 // v64i16 -> v64i1 (pair)
466 // Set these actions first, and allow the single actions to overwrite
467 // any duplicates.
468 MVT BoolW = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
469 setOperationAction(Op: ISD::SETCC, VT: BoolW, Action: Custom);
470 setOperationAction(Op: ISD::AND, VT: BoolW, Action: Custom);
471 setOperationAction(Op: ISD::OR, VT: BoolW, Action: Custom);
472 setOperationAction(Op: ISD::XOR, VT: BoolW, Action: Custom);
473 // Masked load/store takes a mask that may need splitting.
474 setOperationAction(Op: ISD::MLOAD, VT: BoolW, Action: Custom);
475 setOperationAction(Op: ISD::MSTORE, VT: BoolW, Action: Custom);
476 }
477
478 for (MVT T : LegalV) {
479 MVT BoolV = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
480 setOperationAction(Op: ISD::BUILD_VECTOR, VT: BoolV, Action: Custom);
481 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: BoolV, Action: Custom);
482 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: BoolV, Action: Custom);
483 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: BoolV, Action: Custom);
484 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: BoolV, Action: Custom);
485 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: BoolV, Action: Custom);
486 setOperationAction(Op: ISD::SELECT, VT: BoolV, Action: Custom);
487 setOperationAction(Op: ISD::AND, VT: BoolV, Action: Legal);
488 setOperationAction(Op: ISD::OR, VT: BoolV, Action: Legal);
489 setOperationAction(Op: ISD::XOR, VT: BoolV, Action: Legal);
490 }
491
492 if (Use64b) {
493 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
494 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
495 } else {
496 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
497 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
498 }
499
500 // Handle store widening for short vectors.
501 unsigned HwLen = Subtarget.getVectorLength();
502 for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
503 if (ElemTy == MVT::i1)
504 continue;
505 int ElemWidth = ElemTy.getFixedSizeInBits();
506 int MaxElems = (8*HwLen) / ElemWidth;
507 for (int N = 2; N < MaxElems; N *= 2) {
508 MVT VecTy = MVT::getVectorVT(VT: ElemTy, NumElements: N);
509 auto Action = getPreferredVectorAction(VT: VecTy);
510 if (Action == TargetLoweringBase::TypeWidenVector) {
511 setOperationAction(Op: ISD::LOAD, VT: VecTy, Action: Custom);
512 setOperationAction(Op: ISD::STORE, VT: VecTy, Action: Custom);
513 setOperationAction(Op: ISD::SETCC, VT: VecTy, Action: Custom);
514 setOperationAction(Op: ISD::TRUNCATE, VT: VecTy, Action: Custom);
515 setOperationAction(Op: ISD::ANY_EXTEND, VT: VecTy, Action: Custom);
516 setOperationAction(Op: ISD::SIGN_EXTEND, VT: VecTy, Action: Custom);
517 setOperationAction(Op: ISD::ZERO_EXTEND, VT: VecTy, Action: Custom);
518 if (Subtarget.useHVXFloatingPoint()) {
519 setOperationAction(Op: ISD::FP_TO_SINT, VT: VecTy, Action: Custom);
520 setOperationAction(Op: ISD::FP_TO_UINT, VT: VecTy, Action: Custom);
521 setOperationAction(Op: ISD::SINT_TO_FP, VT: VecTy, Action: Custom);
522 setOperationAction(Op: ISD::UINT_TO_FP, VT: VecTy, Action: Custom);
523 }
524
525 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: N);
526 if (!isTypeLegal(VT: BoolTy))
527 setOperationAction(Op: ISD::SETCC, VT: BoolTy, Action: Custom);
528 }
529 }
530 }
531
532 // Include cases which are not hander earlier
533 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v32i1, Action: Custom);
534 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v64i1, Action: Custom);
535 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v32i1, Action: Custom);
536
537 setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT});
538
539 setTargetDAGCombine({ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA,
540 ISD::PARTIAL_REDUCE_SUMLA});
541
542 // Partial MLA reductions.
543 {
544 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
545 ISD::PARTIAL_REDUCE_UMLA,
546 ISD::PARTIAL_REDUCE_SUMLA};
547
548 auto HvxType = [=](MVT ScalarT, unsigned Factor = 1) {
549 return MVT::getVectorVT(VT: ScalarT, NumElements: Subtarget.getVectorLength() * Factor *
550 8 / ScalarT.getSizeInBits());
551 };
552
553 // Tuple of (Acc element type, input element type, vector pair).
554 // The assumption is both the input and reduction result are of the same
555 // size so the reduction ratio is the same as the ratio of element type
556 // sizes. This may not hold for all available instructions.
557 typedef std::tuple<MVT, MVT, bool> ReductionSignature;
558
559 static const std::vector<ReductionSignature> NativeReductions = {
560 {MVT::i32, MVT::i8, false},
561 };
562
563 for (const auto &R : NativeReductions) {
564
565 MVT AccType = std::get<0>(t: R);
566 MVT InputType = std::get<1>(t: R);
567 unsigned Factor = std::get<2>(t: R) ? 2 : 1;
568
569 // The native size is legal.
570 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: HvxType(AccType), InputVT: HvxType(InputType),
571 Action: Legal);
572
573 // Allow custom partial MLA reductions on larger vectors than legally
574 // supported. These reduction must be declared as Custom (or Legal)
575 // for foldPartialReduceMLAMulOp() to fold the multiply by one pattern
576 // inserted when the partial reduction intrinsic is converted to
577 // PARTIAL_REDUCE_U/S/SUMLA. Otherwise, the Split action will apply
578 // on the original pattern, including the extensions and multiplies,
579 // which will make it impossible to match.
580 // There are two independent ways to extend the
581 // input size: 1. to concatenate the result - output vector is
582 // proportionally extended, 2) to reduce the result - the output vector
583 // size stays the same. We limit allowed combinations so that the total
584 // number of generated reduction instructions is limited by a constant
585 // number. This limit is arbitrary and can be revised. On one hand, it is
586 // convenient to have more choices; on the other hand, there is a
587 // diminishing benefit of very long sequences, which should probably be
588 // written as loops instead.
589 for (unsigned ConcatFactor = 1; ConcatFactor <= MaxExpandMLA;
590 ConcatFactor <<= 1)
591 for (unsigned ReductionFactor = 1; ReductionFactor <= MaxExpandMLA;
592 ReductionFactor <<= 1)
593 if (ConcatFactor * ReductionFactor != 1 &&
594 ConcatFactor * ReductionFactor <= MaxExpandMLA)
595 setPartialReduceMLAAction(
596 Opcodes: MLAOps, AccVT: HvxType(AccType, Factor * ConcatFactor),
597 InputVT: HvxType(InputType, Factor * ConcatFactor * ReductionFactor),
598 Action: Custom);
599 }
600 }
601}
602
603unsigned
604HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
605 // Early exit for invalid input types
606 if (!VecTy.isVector())
607 return ~0u;
608
609 MVT ElemTy = VecTy.getVectorElementType();
610 unsigned VecLen = VecTy.getVectorNumElements();
611 unsigned HwLen = Subtarget.getVectorLength();
612
613 // Split vectors of i1 that exceed byte vector length.
614 if (ElemTy == MVT::i1 && VecLen > HwLen)
615 return TargetLoweringBase::TypeSplitVector;
616
617 ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
618 // For shorter vectors of i1, widen them if any of the corresponding
619 // vectors of integers needs to be widened.
620 if (ElemTy == MVT::i1) {
621 for (MVT T : Tys) {
622 assert(T != MVT::i1);
623 auto A = getPreferredHvxVectorAction(VecTy: MVT::getVectorVT(VT: T, NumElements: VecLen));
624 if (A != ~0u)
625 return A;
626 }
627 return ~0u;
628 }
629
630 // If the size of VecTy is at least half of the vector length,
631 // widen the vector. Note: the threshold was not selected in
632 // any scientific way.
633 if (llvm::is_contained(Range&: Tys, Element: ElemTy)) {
634 unsigned VecWidth = VecTy.getSizeInBits();
635 unsigned HwWidth = 8*HwLen;
636 if (VecWidth > 2*HwWidth)
637 return TargetLoweringBase::TypeSplitVector;
638
639 bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0;
640 if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth)
641 return TargetLoweringBase::TypeWidenVector;
642 if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
643 return TargetLoweringBase::TypeWidenVector;
644 }
645
646 // Defer to default.
647 return ~0u;
648}
649
650unsigned
651HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const {
652 unsigned Opc = Op.getOpcode();
653 switch (Opc) {
654 case HexagonISD::SMUL_LOHI:
655 case HexagonISD::UMUL_LOHI:
656 case HexagonISD::USMUL_LOHI:
657 return TargetLoweringBase::Custom;
658 }
659 return TargetLoweringBase::Legal;
660}
661
662SDValue
663HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
664 const SDLoc &dl, SelectionDAG &DAG) const {
665 SmallVector<SDValue,4> IntOps;
666 IntOps.push_back(Elt: DAG.getConstant(Val: IntId, DL: dl, VT: MVT::i32));
667 append_range(C&: IntOps, R&: Ops);
668 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: ResTy, Ops: IntOps);
669}
670
671MVT
672HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
673 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
674
675 MVT ElemTy = Tys.first.getVectorElementType();
676 return MVT::getVectorVT(VT: ElemTy, NumElements: Tys.first.getVectorNumElements() +
677 Tys.second.getVectorNumElements());
678}
679
680HexagonTargetLowering::TypePair
681HexagonTargetLowering::typeSplit(MVT VecTy) const {
682 assert(VecTy.isVector());
683 unsigned NumElem = VecTy.getVectorNumElements();
684 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
685 MVT HalfTy = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: NumElem/2);
686 return { HalfTy, HalfTy };
687}
688
689MVT
690HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
691 MVT ElemTy = VecTy.getVectorElementType();
692 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() * Factor);
693 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
694}
695
696MVT
697HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
698 MVT ElemTy = VecTy.getVectorElementType();
699 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() / Factor);
700 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
701}
702
703SDValue
704HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
705 SelectionDAG &DAG) const {
706 if (ty(Op: Vec).getVectorElementType() == ElemTy)
707 return Vec;
708 MVT CastTy = tyVector(Ty: Vec.getValueType().getSimpleVT(), ElemTy);
709 return DAG.getBitcast(VT: CastTy, V: Vec);
710}
711
712SDValue
713HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
714 SelectionDAG &DAG) const {
715 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: typeJoin(Tys: ty(Ops)),
716 N1: Ops.first, N2: Ops.second);
717}
718
719HexagonTargetLowering::VectorPair
720HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
721 SelectionDAG &DAG) const {
722 TypePair Tys = typeSplit(VecTy: ty(Op: Vec));
723 if (Vec.getOpcode() == HexagonISD::QCAT)
724 return VectorPair(Vec.getOperand(i: 0), Vec.getOperand(i: 1));
725 return DAG.SplitVector(N: Vec, DL: dl, LoVT: Tys.first, HiVT: Tys.second);
726}
727
728bool
729HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
730 return Subtarget.isHVXVectorType(VecTy: Ty) &&
731 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
732}
733
734bool
735HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
736 return Subtarget.isHVXVectorType(VecTy: Ty) &&
737 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
738}
739
740bool
741HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
742 return Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true) &&
743 Ty.getVectorElementType() == MVT::i1;
744}
745
746bool HexagonTargetLowering::allowsHvxMemoryAccess(
747 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
748 // Bool vectors are excluded by default, but make it explicit to
749 // emphasize that bool vectors cannot be loaded or stored.
750 // Also, disallow double vector stores (to prevent unnecessary
751 // store widening in DAG combiner).
752 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
753 return false;
754 if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
755 return false;
756 if (Fast)
757 *Fast = 1;
758 return true;
759}
760
761bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
762 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
763 if (!Subtarget.isHVXVectorType(VecTy))
764 return false;
765 // XXX Should this be false? vmemu are a bit slower than vmem.
766 if (Fast)
767 *Fast = 1;
768 return true;
769}
770
771void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection(
772 MachineInstr &MI, SDNode *Node) const {
773 unsigned Opc = MI.getOpcode();
774 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
775 MachineBasicBlock &MB = *MI.getParent();
776 MachineFunction &MF = *MB.getParent();
777 MachineRegisterInfo &MRI = MF.getRegInfo();
778 DebugLoc DL = MI.getDebugLoc();
779 auto At = MI.getIterator();
780
781 switch (Opc) {
782 case Hexagon::PS_vsplatib:
783 if (Subtarget.useHVXV62Ops()) {
784 // SplatV = A2_tfrsi #imm
785 // OutV = V6_lvsplatb SplatV
786 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
787 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
788 .add(MO: MI.getOperand(i: 1));
789 Register OutV = MI.getOperand(i: 0).getReg();
790 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
791 .addReg(RegNo: SplatV);
792 } else {
793 // SplatV = A2_tfrsi #imm:#imm:#imm:#imm
794 // OutV = V6_lvsplatw SplatV
795 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
796 const MachineOperand &InpOp = MI.getOperand(i: 1);
797 assert(InpOp.isImm());
798 uint32_t V = InpOp.getImm() & 0xFF;
799 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
800 .addImm(Val: V << 24 | V << 16 | V << 8 | V);
801 Register OutV = MI.getOperand(i: 0).getReg();
802 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
803 }
804 MB.erase(I: At);
805 break;
806 case Hexagon::PS_vsplatrb:
807 if (Subtarget.useHVXV62Ops()) {
808 // OutV = V6_lvsplatb Inp
809 Register OutV = MI.getOperand(i: 0).getReg();
810 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
811 .add(MO: MI.getOperand(i: 1));
812 } else {
813 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
814 const MachineOperand &InpOp = MI.getOperand(i: 1);
815 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::S2_vsplatrb), DestReg: SplatV)
816 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
817 Register OutV = MI.getOperand(i: 0).getReg();
818 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV)
819 .addReg(RegNo: SplatV);
820 }
821 MB.erase(I: At);
822 break;
823 case Hexagon::PS_vsplatih:
824 if (Subtarget.useHVXV62Ops()) {
825 // SplatV = A2_tfrsi #imm
826 // OutV = V6_lvsplath SplatV
827 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
828 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
829 .add(MO: MI.getOperand(i: 1));
830 Register OutV = MI.getOperand(i: 0).getReg();
831 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
832 .addReg(RegNo: SplatV);
833 } else {
834 // SplatV = A2_tfrsi #imm:#imm
835 // OutV = V6_lvsplatw SplatV
836 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
837 const MachineOperand &InpOp = MI.getOperand(i: 1);
838 assert(InpOp.isImm());
839 uint32_t V = InpOp.getImm() & 0xFFFF;
840 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
841 .addImm(Val: V << 16 | V);
842 Register OutV = MI.getOperand(i: 0).getReg();
843 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
844 }
845 MB.erase(I: At);
846 break;
847 case Hexagon::PS_vsplatrh:
848 if (Subtarget.useHVXV62Ops()) {
849 // OutV = V6_lvsplath Inp
850 Register OutV = MI.getOperand(i: 0).getReg();
851 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
852 .add(MO: MI.getOperand(i: 1));
853 } else {
854 // SplatV = A2_combine_ll Inp, Inp
855 // OutV = V6_lvsplatw SplatV
856 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
857 const MachineOperand &InpOp = MI.getOperand(i: 1);
858 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_combine_ll), DestReg: SplatV)
859 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg())
860 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
861 Register OutV = MI.getOperand(i: 0).getReg();
862 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
863 }
864 MB.erase(I: At);
865 break;
866 case Hexagon::PS_vsplatiw:
867 case Hexagon::PS_vsplatrw:
868 if (Opc == Hexagon::PS_vsplatiw) {
869 // SplatV = A2_tfrsi #imm
870 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
871 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
872 .add(MO: MI.getOperand(i: 1));
873 MI.getOperand(i: 1).ChangeToRegister(Reg: SplatV, isDef: false);
874 }
875 // OutV = V6_lvsplatw SplatV/Inp
876 MI.setDesc(TII.get(Opcode: Hexagon::V6_lvsplatw));
877 break;
878 }
879}
880
881SDValue
882HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
883 SelectionDAG &DAG) const {
884 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
885 ElemIdx = DAG.getBitcast(VT: MVT::i32, V: ElemIdx);
886
887 unsigned ElemWidth = ElemTy.getSizeInBits();
888 if (ElemWidth == 8)
889 return ElemIdx;
890
891 unsigned L = Log2_32(Value: ElemWidth/8);
892 const SDLoc &dl(ElemIdx);
893 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32,
894 Ops: {ElemIdx, DAG.getConstant(Val: L, DL: dl, VT: MVT::i32)});
895}
896
897SDValue
898HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
899 SelectionDAG &DAG) const {
900 unsigned ElemWidth = ElemTy.getSizeInBits();
901 assert(ElemWidth >= 8 && ElemWidth <= 32);
902 if (ElemWidth == 32)
903 return Idx;
904
905 if (ty(Op: Idx) != MVT::i32)
906 Idx = DAG.getBitcast(VT: MVT::i32, V: Idx);
907 const SDLoc &dl(Idx);
908 SDValue Mask = DAG.getConstant(Val: 32/ElemWidth - 1, DL: dl, VT: MVT::i32);
909 SDValue SubIdx = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, Ops: {Idx, Mask});
910 return SubIdx;
911}
912
913SDValue
914HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
915 SDValue Op1, ArrayRef<int> Mask,
916 SelectionDAG &DAG) const {
917 MVT OpTy = ty(Op: Op0);
918 assert(OpTy == ty(Op1));
919
920 MVT ElemTy = OpTy.getVectorElementType();
921 if (ElemTy == MVT::i8)
922 return DAG.getVectorShuffle(VT: OpTy, dl, N1: Op0, N2: Op1, Mask);
923 assert(ElemTy.getSizeInBits() >= 8);
924
925 MVT ResTy = tyVector(Ty: OpTy, ElemTy: MVT::i8);
926 unsigned ElemSize = ElemTy.getSizeInBits() / 8;
927
928 SmallVector<int,128> ByteMask;
929 for (int M : Mask) {
930 if (M < 0) {
931 for (unsigned I = 0; I != ElemSize; ++I)
932 ByteMask.push_back(Elt: -1);
933 } else {
934 int NewM = M*ElemSize;
935 for (unsigned I = 0; I != ElemSize; ++I)
936 ByteMask.push_back(Elt: NewM+I);
937 }
938 }
939 assert(ResTy.getVectorNumElements() == ByteMask.size());
940 return DAG.getVectorShuffle(VT: ResTy, dl, N1: opCastElem(Vec: Op0, ElemTy: MVT::i8, DAG),
941 N2: opCastElem(Vec: Op1, ElemTy: MVT::i8, DAG), Mask: ByteMask);
942}
943
944SDValue
945HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
946 const SDLoc &dl, MVT VecTy,
947 SelectionDAG &DAG) const {
948 unsigned VecLen = Values.size();
949 MachineFunction &MF = DAG.getMachineFunction();
950 MVT ElemTy = VecTy.getVectorElementType();
951 unsigned ElemWidth = ElemTy.getSizeInBits();
952 unsigned HwLen = Subtarget.getVectorLength();
953
954 unsigned ElemSize = ElemWidth / 8;
955 assert(ElemSize*VecLen == HwLen);
956 SmallVector<SDValue,32> Words;
957
958 if (VecTy.getVectorElementType() != MVT::i32 &&
959 !(Subtarget.useHVXFloatingPoint() &&
960 VecTy.getVectorElementType() == MVT::f32)) {
961 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
962 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
963 MVT PartVT = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: OpsPerWord);
964 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
965 SDValue W = buildVector32(Elem: Values.slice(N: i, M: OpsPerWord), dl, VecTy: PartVT, DAG);
966 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: W));
967 }
968 } else {
969 for (SDValue V : Values)
970 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V));
971 }
972 auto isSplat = [] (ArrayRef<SDValue> Values, SDValue &SplatV) {
973 unsigned NumValues = Values.size();
974 assert(NumValues > 0);
975 bool IsUndef = true;
976 for (unsigned i = 0; i != NumValues; ++i) {
977 if (Values[i].isUndef())
978 continue;
979 IsUndef = false;
980 if (!SplatV.getNode())
981 SplatV = Values[i];
982 else if (SplatV != Values[i])
983 return false;
984 }
985 if (IsUndef)
986 SplatV = Values[0];
987 return true;
988 };
989
990 unsigned NumWords = Words.size();
991 SDValue SplatV;
992 bool IsSplat = isSplat(Words, SplatV);
993 if (IsSplat && isUndef(Op: SplatV))
994 return DAG.getUNDEF(VT: VecTy);
995 if (IsSplat) {
996 assert(SplatV.getNode());
997 if (isNullConstant(V: SplatV))
998 return getZero(dl, Ty: VecTy, DAG);
999 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
1000 SDValue S = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordTy, Operand: SplatV);
1001 return DAG.getBitcast(VT: VecTy, V: S);
1002 }
1003
1004 // Delay recognizing constant vectors until here, so that we can generate
1005 // a vsplat.
1006 SmallVector<ConstantInt*, 128> Consts(VecLen);
1007 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
1008 if (AllConst) {
1009 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
1010 (Constant**)Consts.end());
1011 Constant *CV = ConstantVector::get(V: Tmp);
1012 Align Alignment(HwLen);
1013 SDValue CP = LowerConstantPool(
1014 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
1015 DAG);
1016 return DAG.getLoad(VT: VecTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
1017 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
1018 }
1019
1020 // A special case is a situation where the vector is built entirely from
1021 // elements extracted from another vector. This could be done via a shuffle
1022 // more efficiently, but typically, the size of the source vector will not
1023 // match the size of the vector being built (which precludes the use of a
1024 // shuffle directly).
1025 // This only handles a single source vector, and the vector being built
1026 // should be of a sub-vector type of the source vector type.
1027 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
1028 SmallVectorImpl<int> &SrcIdx) {
1029 SDValue Vec;
1030 for (SDValue V : Values) {
1031 if (isUndef(Op: V)) {
1032 SrcIdx.push_back(Elt: -1);
1033 continue;
1034 }
1035 if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1036 return false;
1037 // All extracts should come from the same vector.
1038 SDValue T = V.getOperand(i: 0);
1039 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
1040 return false;
1041 Vec = T;
1042 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
1043 if (C == nullptr)
1044 return false;
1045 int I = C->getSExtValue();
1046 assert(I >= 0 && "Negative element index");
1047 SrcIdx.push_back(Elt: I);
1048 }
1049 SrcVec = Vec;
1050 return true;
1051 };
1052
1053 SmallVector<int,128> ExtIdx;
1054 SDValue ExtVec;
1055 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
1056 MVT ExtTy = ty(Op: ExtVec);
1057 unsigned ExtLen = ExtTy.getVectorNumElements();
1058 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
1059 // Construct a new shuffle mask that will produce a vector with the same
1060 // number of elements as the input vector, and such that the vector we
1061 // want will be the initial subvector of it.
1062 SmallVector<int,128> Mask;
1063 BitVector Used(ExtLen);
1064
1065 for (int M : ExtIdx) {
1066 Mask.push_back(Elt: M);
1067 if (M >= 0)
1068 Used.set(M);
1069 }
1070 // Fill the rest of the mask with the unused elements of ExtVec in hopes
1071 // that it will result in a permutation of ExtVec's elements. It's still
1072 // fine if it doesn't (e.g. if undefs are present, or elements are
1073 // repeated), but permutations can always be done efficiently via vdelta
1074 // and vrdelta.
1075 for (unsigned I = 0; I != ExtLen; ++I) {
1076 if (Mask.size() == ExtLen)
1077 break;
1078 if (!Used.test(Idx: I))
1079 Mask.push_back(Elt: I);
1080 }
1081
1082 SDValue S = DAG.getVectorShuffle(VT: ExtTy, dl, N1: ExtVec,
1083 N2: DAG.getUNDEF(VT: ExtTy), Mask);
1084 return ExtLen == VecLen ? S : LoHalf(V: S, DAG);
1085 }
1086 }
1087
1088 // Find most common element to initialize vector with. This is to avoid
1089 // unnecessary vinsert/valign for cases where the same value is present
1090 // many times. Creates a histogram of the vector's elements to find the
1091 // most common element n.
1092 assert(4*Words.size() == Subtarget.getVectorLength());
1093 int VecHist[32];
1094 int n = 0;
1095 for (unsigned i = 0; i != NumWords; ++i) {
1096 VecHist[i] = 0;
1097 if (Words[i].isUndef())
1098 continue;
1099 for (unsigned j = i; j != NumWords; ++j)
1100 if (Words[i] == Words[j])
1101 VecHist[i]++;
1102
1103 if (VecHist[i] > VecHist[n])
1104 n = i;
1105 }
1106
1107 SDValue HalfV = getZero(dl, Ty: VecTy, DAG);
1108 if (VecHist[n] > 1) {
1109 // Always splat at word (i32) granularity so that the SPLAT_VECTOR node
1110 // is selected as PS_vsplatrw (word broadcast) rather than PS_vsplatrb
1111 // (byte broadcast of the low byte only), which would corrupt multi-byte
1112 // element types.
1113 MVT WordVecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
1114 SDValue WordSplat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordVecTy, Operand: Words[n]);
1115 SDValue SplatV = DAG.getBitcast(VT: VecTy, V: WordSplat);
1116 HalfV = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: VecTy,
1117 Ops: {HalfV, SplatV, DAG.getConstant(Val: HwLen/2, DL: dl, VT: MVT::i32)});
1118 }
1119 SDValue HalfV0 = HalfV;
1120 SDValue HalfV1 = HalfV;
1121
1122 // Construct two halves in parallel, then or them together. Rn and Rm count
1123 // number of rotations needed before the next element. One last rotation is
1124 // performed post-loop to position the last element.
1125 int Rn = 0, Rm = 0;
1126 SDValue Sn, Sm;
1127 SDValue N = HalfV0;
1128 SDValue M = HalfV1;
1129 for (unsigned i = 0; i != NumWords/2; ++i) {
1130 // Rotate by element count since last insertion.
1131 if (Words[i] != Words[n] || VecHist[n] <= 1) {
1132 Sn = DAG.getConstant(Val: Rn, DL: dl, VT: MVT::i32);
1133 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1134 N = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1135 Ops: {HalfV0, Words[i]});
1136 Rn = 0;
1137 }
1138 if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) {
1139 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1140 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1141 M = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1142 Ops: {HalfV1, Words[i+NumWords/2]});
1143 Rm = 0;
1144 }
1145 Rn += 4;
1146 Rm += 4;
1147 }
1148 // Perform last rotation.
1149 Sn = DAG.getConstant(Val: Rn+HwLen/2, DL: dl, VT: MVT::i32);
1150 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1151 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1152 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1153
1154 SDValue T0 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV0);
1155 SDValue T1 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV1);
1156
1157 SDValue DstV = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ty(Op: T0), Ops: {T0, T1});
1158
1159 SDValue OutV =
1160 DAG.getBitcast(VT: tyVector(Ty: ty(Op: DstV), ElemTy: VecTy.getVectorElementType()), V: DstV);
1161 return OutV;
1162}
1163
1164SDValue
1165HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
1166 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
1167 MVT PredTy = ty(Op: PredV);
1168 unsigned HwLen = Subtarget.getVectorLength();
1169 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1170
1171 if (Subtarget.isHVXVectorType(VecTy: PredTy, IncludeBool: true)) {
1172 // Move the vector predicate SubV to a vector register, and scale it
1173 // down to match the representation (bytes per type element) that VecV
1174 // uses. The scaling down will pick every 2nd or 4th (every Scale-th
1175 // in general) element and put them at the front of the resulting
1176 // vector. This subvector will then be inserted into the Q2V of VecV.
1177 // To avoid having an operation that generates an illegal type (short
1178 // vector), generate a full size vector.
1179 //
1180 SDValue T = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: PredV);
1181 SmallVector<int,128> Mask(HwLen);
1182 // Scale = BitBytes(PredV) / Given BitBytes.
1183 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
1184 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
1185
1186 for (unsigned i = 0; i != HwLen; ++i) {
1187 unsigned Num = i % Scale;
1188 unsigned Off = i / Scale;
1189 Mask[BlockLen*Num + Off] = i;
1190 }
1191 SDValue S = DAG.getVectorShuffle(VT: ByteTy, dl, N1: T, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1192 if (!ZeroFill)
1193 return S;
1194 // Fill the bytes beyond BlockLen with 0s.
1195 // V6_pred_scalar2 cannot fill the entire predicate, so it only works
1196 // when BlockLen < HwLen.
1197 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1198 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1199 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1200 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1201 SDValue M = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Q);
1202 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ByteTy, N1: S, N2: M);
1203 }
1204
1205 // Make sure that this is a valid scalar predicate.
1206 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
1207
1208 unsigned Bytes = 8 / PredTy.getVectorNumElements();
1209 SmallVector<SDValue,4> Words[2];
1210 unsigned IdxW = 0;
1211
1212 SDValue W0 = isUndef(Op: PredV)
1213 ? DAG.getUNDEF(VT: MVT::i64)
1214 : DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: PredV);
1215 Words[IdxW].push_back(Elt: HiHalf(V: W0, DAG));
1216 Words[IdxW].push_back(Elt: LoHalf(V: W0, DAG));
1217
1218 while (Bytes < BitBytes) {
1219 IdxW ^= 1;
1220 Words[IdxW].clear();
1221
1222 if (Bytes < 4) {
1223 for (const SDValue &W : Words[IdxW ^ 1]) {
1224 SDValue T = expandPredicate(Vec32: W, dl, DAG);
1225 Words[IdxW].push_back(Elt: HiHalf(V: T, DAG));
1226 Words[IdxW].push_back(Elt: LoHalf(V: T, DAG));
1227 }
1228 } else {
1229 for (const SDValue &W : Words[IdxW ^ 1]) {
1230 Words[IdxW].push_back(Elt: W);
1231 Words[IdxW].push_back(Elt: W);
1232 }
1233 }
1234 Bytes *= 2;
1235 }
1236
1237 assert(Bytes == BitBytes);
1238 SDValue Vec = ZeroFill ? getZero(dl, Ty: ByteTy, DAG) : DAG.getUNDEF(VT: ByteTy);
1239 SDValue S4 = DAG.getConstant(Val: HwLen-4, DL: dl, VT: MVT::i32);
1240 for (const SDValue &W : Words[IdxW]) {
1241 Vec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Vec, N2: S4);
1242 Vec = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: ByteTy, N1: Vec, N2: W);
1243 }
1244
1245 return Vec;
1246}
1247
1248SDValue
1249HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
1250 const SDLoc &dl, MVT VecTy,
1251 SelectionDAG &DAG) const {
1252 // Construct a vector V of bytes, such that a comparison V >u 0 would
1253 // produce the required vector predicate.
1254 unsigned VecLen = Values.size();
1255 unsigned HwLen = Subtarget.getVectorLength();
1256 assert(VecLen <= HwLen || VecLen == 8*HwLen);
1257 SmallVector<SDValue,128> Bytes;
1258 bool AllT = true, AllF = true;
1259
1260 auto IsTrue = [] (SDValue V) {
1261 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1262 return !N->isZero();
1263 return false;
1264 };
1265 auto IsFalse = [] (SDValue V) {
1266 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1267 return N->isZero();
1268 return false;
1269 };
1270
1271 if (VecLen <= HwLen) {
1272 // In the hardware, each bit of a vector predicate corresponds to a byte
1273 // of a vector register. Calculate how many bytes does a bit of VecTy
1274 // correspond to.
1275 assert(HwLen % VecLen == 0);
1276 unsigned BitBytes = HwLen / VecLen;
1277 for (SDValue V : Values) {
1278 AllT &= IsTrue(V);
1279 AllF &= IsFalse(V);
1280
1281 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(Op: V, DL: dl, VT: MVT::i8)
1282 : DAG.getUNDEF(VT: MVT::i8);
1283 for (unsigned B = 0; B != BitBytes; ++B)
1284 Bytes.push_back(Elt: Ext);
1285 }
1286 } else {
1287 // There are as many i1 values, as there are bits in a vector register.
1288 // Divide the values into groups of 8 and check that each group consists
1289 // of the same value (ignoring undefs).
1290 for (unsigned I = 0; I != VecLen; I += 8) {
1291 unsigned B = 0;
1292 // Find the first non-undef value in this group.
1293 for (; B != 8; ++B) {
1294 if (!Values[I+B].isUndef())
1295 break;
1296 }
1297 SDValue F = Values[I+B];
1298 AllT &= IsTrue(F);
1299 AllF &= IsFalse(F);
1300
1301 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(Op: F, DL: dl, VT: MVT::i8)
1302 : DAG.getUNDEF(VT: MVT::i8);
1303 Bytes.push_back(Elt: Ext);
1304 // Verify that the rest of values in the group are the same as the
1305 // first.
1306 for (; B != 8; ++B)
1307 assert(Values[I+B].isUndef() || Values[I+B] == F);
1308 }
1309 }
1310
1311 if (AllT)
1312 return DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: VecTy);
1313 if (AllF)
1314 return DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: VecTy);
1315
1316 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1317 SDValue ByteVec = buildHvxVectorReg(Values: Bytes, dl, VecTy: ByteTy, DAG);
1318 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1319}
1320
1321SDValue
1322HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
1323 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1324 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1325
1326 unsigned ElemWidth = ElemTy.getSizeInBits();
1327 assert(ElemWidth >= 8 && ElemWidth <= 32);
1328 (void)ElemWidth;
1329
1330 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1331 SDValue ExWord = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1332 Ops: {VecV, ByteIdx});
1333 if (ElemTy == MVT::i32)
1334 return ExWord;
1335
1336 // Have an extracted word, need to extract the smaller element out of it.
1337 // 1. Extract the bits of (the original) IdxV that correspond to the index
1338 // of the desired element in the 32-bit word.
1339 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1340 // 2. Extract the element from the word.
1341 SDValue ExVec = DAG.getBitcast(VT: tyVector(Ty: ty(Op: ExWord), ElemTy), V: ExWord);
1342 return extractVector(VecV: ExVec, IdxV: SubIdx, dl, ValTy: ElemTy, ResTy: MVT::i32, DAG);
1343}
1344
1345SDValue
1346HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
1347 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1348 // Implement other return types if necessary.
1349 assert(ResTy == MVT::i1);
1350
1351 unsigned HwLen = Subtarget.getVectorLength();
1352 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1353 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1354
1355 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1356 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1357 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1358
1359 SDValue ExtB = extractHvxElementReg(VecV: ByteVec, IdxV, dl, ResTy: MVT::i32, DAG);
1360 SDValue Zero = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1361 return getInstr(MachineOpc: Hexagon::C2_cmpgtui, dl, Ty: MVT::i1, Ops: {ExtB, Zero}, DAG);
1362}
1363
1364SDValue
1365HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
1366 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1367 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1368
1369 unsigned ElemWidth = ElemTy.getSizeInBits();
1370 assert(ElemWidth >= 8 && ElemWidth <= 32);
1371 (void)ElemWidth;
1372
1373 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
1374 SDValue ByteIdxV) {
1375 MVT VecTy = ty(Op: VecV);
1376 unsigned HwLen = Subtarget.getVectorLength();
1377 SDValue MaskV =
1378 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
1379 Ops: {ByteIdxV, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)});
1380 SDValue RotV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {VecV, MaskV});
1381 SDValue InsV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy, Ops: {RotV, ValV});
1382 SDValue SubV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1383 Ops: {DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32), MaskV});
1384 SDValue TorV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {InsV, SubV});
1385 return TorV;
1386 };
1387
1388 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1389 if (ElemTy == MVT::i32)
1390 return InsertWord(VecV, ValV, ByteIdx);
1391
1392 // If this is not inserting a 32-bit word, convert it into such a thing.
1393 // 1. Extract the existing word from the target vector.
1394 SDValue WordIdx = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
1395 Ops: {ByteIdx, DAG.getConstant(Val: 2, DL: dl, VT: MVT::i32)});
1396 SDValue Ext = extractHvxElementReg(VecV: opCastElem(Vec: VecV, ElemTy: MVT::i32, DAG), IdxV: WordIdx,
1397 dl, ResTy: MVT::i32, DAG);
1398
1399 // 2. Treating the extracted word as a 32-bit vector, insert the given
1400 // value into it.
1401 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1402 MVT SubVecTy = tyVector(Ty: ty(Op: Ext), ElemTy);
1403 SDValue Ins = insertVector(VecV: DAG.getBitcast(VT: SubVecTy, V: Ext),
1404 ValV, IdxV: SubIdx, dl, ValTy: ElemTy, DAG);
1405
1406 // 3. Insert the 32-bit word back into the original vector.
1407 return InsertWord(VecV, Ins, ByteIdx);
1408}
1409
1410SDValue
1411HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
1412 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1413 unsigned HwLen = Subtarget.getVectorLength();
1414 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1415 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1416
1417 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1418 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1419 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1420 ValV = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i32, Operand: ValV);
1421
1422 SDValue InsV = insertHvxElementReg(VecV: ByteVec, IdxV, ValV, dl, DAG);
1423 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ty(Op: VecV), Operand: InsV);
1424}
1425
1426SDValue
1427HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV,
1428 SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1429 MVT VecTy = ty(Op: VecV);
1430 unsigned HwLen = Subtarget.getVectorLength();
1431 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1432 MVT ElemTy = VecTy.getVectorElementType();
1433 unsigned ElemWidth = ElemTy.getSizeInBits();
1434
1435 // If the source vector is a vector pair, get the single vector containing
1436 // the subvector of interest. The subvector will never overlap two single
1437 // vectors.
1438 if (isHvxPairTy(Ty: VecTy)) {
1439 unsigned SubIdx = Hexagon::vsub_lo;
1440 if (Idx * ElemWidth >= 8 * HwLen) {
1441 SubIdx = Hexagon::vsub_hi;
1442 Idx -= VecTy.getVectorNumElements() / 2;
1443 }
1444
1445 VecTy = typeSplit(VecTy).first;
1446 VecV = DAG.getTargetExtractSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV);
1447 if (VecTy == ResTy)
1448 return VecV;
1449 }
1450
1451 // The only meaningful subvectors of a single HVX vector are those that
1452 // fit in a scalar register.
1453 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
1454
1455 MVT WordTy = tyVector(Ty: VecTy, ElemTy: MVT::i32);
1456 SDValue WordVec = DAG.getBitcast(VT: WordTy, V: VecV);
1457 unsigned WordIdx = (Idx*ElemWidth) / 32;
1458
1459 SDValue W0Idx = DAG.getConstant(Val: WordIdx, DL: dl, VT: MVT::i32);
1460 SDValue W0 = extractHvxElementReg(VecV: WordVec, IdxV: W0Idx, dl, ResTy: MVT::i32, DAG);
1461 if (ResTy.getSizeInBits() == 32)
1462 return DAG.getBitcast(VT: ResTy, V: W0);
1463
1464 SDValue W1Idx = DAG.getConstant(Val: WordIdx+1, DL: dl, VT: MVT::i32);
1465 SDValue W1 = extractHvxElementReg(VecV: WordVec, IdxV: W1Idx, dl, ResTy: MVT::i32, DAG);
1466 SDValue WW = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::i64, DAG);
1467 return DAG.getBitcast(VT: ResTy, V: WW);
1468}
1469
1470SDValue
1471HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
1472 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1473 MVT VecTy = ty(Op: VecV);
1474 unsigned HwLen = Subtarget.getVectorLength();
1475 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1476 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1477 // IdxV is required to be a constant.
1478 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1479
1480 unsigned ResLen = ResTy.getVectorNumElements();
1481 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1482 unsigned Offset = Idx * BitBytes;
1483 SDValue Undef = DAG.getUNDEF(VT: ByteTy);
1484 SmallVector<int,128> Mask;
1485
1486 if (Subtarget.isHVXVectorType(VecTy: ResTy, IncludeBool: true)) {
1487 // Converting between two vector predicates. Since the result is shorter
1488 // than the source, it will correspond to a vector predicate with the
1489 // relevant bits replicated. The replication count is the ratio of the
1490 // source and target vector lengths.
1491 unsigned Rep = VecTy.getVectorNumElements() / ResLen;
1492 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
1493 for (unsigned i = 0; i != HwLen/Rep; ++i) {
1494 for (unsigned j = 0; j != Rep; ++j)
1495 Mask.push_back(Elt: i + Offset);
1496 }
1497 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1498 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: ShuffV);
1499 }
1500
1501 // Converting between a vector predicate and a scalar predicate. In the
1502 // vector predicate, a group of BitBytes bits will correspond to a single
1503 // i1 element of the source vector type. Those bits will all have the same
1504 // value. The same will be true for ByteVec, where each byte corresponds
1505 // to a bit in the vector predicate.
1506 // The algorithm is to traverse the ByteVec, going over the i1 values from
1507 // the source vector, and generate the corresponding representation in an
1508 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
1509 // elements so that the interesting 8 bytes will be in the low end of the
1510 // vector.
1511 unsigned Rep = 8 / ResLen;
1512 // Make sure the output fill the entire vector register, so repeat the
1513 // 8-byte groups as many times as necessary.
1514 for (unsigned r = 0; r != HwLen / 8; ++r) {
1515 // This will generate the indexes of the 8 interesting bytes.
1516 for (unsigned i = 0; i != ResLen; ++i) {
1517 for (unsigned j = 0; j != Rep; ++j)
1518 Mask.push_back(Elt: Offset + i*BitBytes);
1519 }
1520 }
1521
1522 SDValue Zero = getZero(dl, Ty: MVT::i32, DAG);
1523 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1524 // Combine the two low words from ShuffV into a v8i8, and byte-compare
1525 // them against 0.
1526 SDValue W0 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32, Ops: {ShuffV, Zero});
1527 SDValue W1 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1528 Ops: {ShuffV, DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32)});
1529 SDValue Vec64 = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::v8i8, DAG);
1530 return getInstr(MachineOpc: Hexagon::A4_vcmpbgtui, dl, Ty: ResTy,
1531 Ops: {Vec64, DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32)}, DAG);
1532}
1533
1534SDValue
1535HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
1536 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1537 MVT VecTy = ty(Op: VecV);
1538 MVT SubTy = ty(Op: SubV);
1539 unsigned HwLen = Subtarget.getVectorLength();
1540 MVT ElemTy = VecTy.getVectorElementType();
1541 unsigned ElemWidth = ElemTy.getSizeInBits();
1542
1543 bool IsPair = isHvxPairTy(Ty: VecTy);
1544 MVT SingleTy = MVT::getVectorVT(VT: ElemTy, NumElements: (8*HwLen)/ElemWidth);
1545 // The two single vectors that VecV consists of, if it's a pair.
1546 SDValue V0, V1;
1547 SDValue SingleV = VecV;
1548 SDValue PickHi;
1549
1550 if (IsPair) {
1551 V0 = LoHalf(V: VecV, DAG);
1552 V1 = HiHalf(V: VecV, DAG);
1553
1554 SDValue HalfV = DAG.getConstant(Val: SingleTy.getVectorNumElements(),
1555 DL: dl, VT: MVT::i32);
1556 PickHi = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: IdxV, RHS: HalfV, Cond: ISD::SETUGT);
1557 if (isHvxSingleTy(Ty: SubTy)) {
1558 if (const auto *CN = dyn_cast<const ConstantSDNode>(Val: IdxV.getNode())) {
1559 unsigned Idx = CN->getZExtValue();
1560 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
1561 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
1562 return DAG.getTargetInsertSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV, Subreg: SubV);
1563 }
1564 // If IdxV is not a constant, generate the two variants: with the
1565 // SubV as the high and as the low subregister, and select the right
1566 // pair based on the IdxV.
1567 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SubV, V1});
1568 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SubV});
1569 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1570 }
1571 // The subvector being inserted must be entirely contained in one of
1572 // the vectors V0 or V1. Set SingleV to the correct one, and update
1573 // IdxV to be the index relative to the beginning of that vector.
1574 SDValue S = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: IdxV, N2: HalfV);
1575 IdxV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: PickHi, N2: S, N3: IdxV);
1576 SingleV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: SingleTy, N1: PickHi, N2: V1, N3: V0);
1577 }
1578
1579 // The only meaningful subvectors of a single HVX vector are those that
1580 // fit in a scalar register.
1581 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
1582 // Convert IdxV to be index in bytes.
1583 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1584 if (!IdxN || !IdxN->isZero()) {
1585 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1586 N2: DAG.getConstant(Val: ElemWidth/8, DL: dl, VT: MVT::i32));
1587 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: IdxV);
1588 }
1589 // When inserting a single word, the rotation back to the original position
1590 // would be by HwLen-Idx, but if two words are inserted, it will need to be
1591 // by (HwLen-4)-Idx.
1592 unsigned RolBase = HwLen;
1593 if (SubTy.getSizeInBits() == 32) {
1594 SDValue V = DAG.getBitcast(VT: MVT::i32, V: SubV);
1595 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: V);
1596 } else {
1597 SDValue V = DAG.getBitcast(VT: MVT::i64, V: SubV);
1598 SDValue R0 = LoHalf(V, DAG);
1599 SDValue R1 = HiHalf(V, DAG);
1600 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R0);
1601 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV,
1602 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
1603 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R1);
1604 RolBase = HwLen-4;
1605 }
1606 // If the vector wasn't ror'ed, don't ror it back.
1607 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1608 SDValue RolV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1609 N1: DAG.getConstant(Val: RolBase, DL: dl, VT: MVT::i32), N2: IdxV);
1610 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: RolV);
1611 }
1612
1613 if (IsPair) {
1614 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SingleV, V1});
1615 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SingleV});
1616 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1617 }
1618 return SingleV;
1619}
1620
1621SDValue
1622HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1623 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1624 MVT VecTy = ty(Op: VecV);
1625 MVT SubTy = ty(Op: SubV);
1626 assert(Subtarget.isHVXVectorType(VecTy, true));
1627 // VecV is an HVX vector predicate. SubV may be either an HVX vector
1628 // predicate as well, or it can be a scalar predicate.
1629
1630 unsigned VecLen = VecTy.getVectorNumElements();
1631 unsigned HwLen = Subtarget.getVectorLength();
1632 assert(HwLen % VecLen == 0 && "Unexpected vector type");
1633
1634 unsigned Scale = VecLen / SubTy.getVectorNumElements();
1635 unsigned BitBytes = HwLen / VecLen;
1636 unsigned BlockLen = HwLen / Scale;
1637
1638 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1639 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1640 SDValue ByteSub = createHvxPrefixPred(PredV: SubV, dl, BitBytes, ZeroFill: false, DAG);
1641 SDValue ByteIdx;
1642
1643 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1644 if (!IdxN || !IdxN->isZero()) {
1645 ByteIdx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1646 N2: DAG.getConstant(Val: BitBytes, DL: dl, VT: MVT::i32));
1647 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteIdx);
1648 }
1649
1650 // ByteVec is the target vector VecV rotated in such a way that the
1651 // subvector should be inserted at index 0. Generate a predicate mask
1652 // and use vmux to do the insertion.
1653 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1654 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1655 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1656 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1657 ByteVec = getInstr(MachineOpc: Hexagon::V6_vmux, dl, Ty: ByteTy, Ops: {Q, ByteSub, ByteVec}, DAG);
1658 // Rotate ByteVec back, and convert to a vector predicate.
1659 if (!IdxN || !IdxN->isZero()) {
1660 SDValue HwLenV = DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32);
1661 SDValue ByteXdi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: HwLenV, N2: ByteIdx);
1662 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteXdi);
1663 }
1664 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1665}
1666
1667SDValue
1668HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1669 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1670 // Sign- and any-extending of a vector predicate to a vector register is
1671 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1672 // a vector of 1s (where the 1s are of type matching the vector type).
1673 assert(Subtarget.isHVXVectorType(ResTy));
1674 if (!ZeroExt)
1675 return DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ResTy, Operand: VecV);
1676
1677 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1678 SDValue True = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
1679 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1680 SDValue False = getZero(dl, Ty: ResTy, DAG);
1681 return DAG.getSelect(DL: dl, VT: ResTy, Cond: VecV, LHS: True, RHS: False);
1682}
1683
1684SDValue
1685HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1686 MVT ResTy, SelectionDAG &DAG) const {
1687 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1688 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1689 // vector register. The remaining bits of the vector register are
1690 // unspecified.
1691
1692 MachineFunction &MF = DAG.getMachineFunction();
1693 unsigned HwLen = Subtarget.getVectorLength();
1694 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1695 MVT PredTy = ty(Op: VecQ);
1696 unsigned PredLen = PredTy.getVectorNumElements();
1697 assert(HwLen % PredLen == 0);
1698 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8*HwLen/PredLen), NumElements: PredLen);
1699
1700 Type *Int8Ty = Type::getInt8Ty(C&: *DAG.getContext());
1701 SmallVector<Constant*, 128> Tmp;
1702 // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
1703 // These are bytes with the LSB rotated left with respect to their index.
1704 for (unsigned i = 0; i != HwLen/8; ++i) {
1705 for (unsigned j = 0; j != 8; ++j)
1706 Tmp.push_back(Elt: ConstantInt::get(Ty: Int8Ty, V: 1ull << j));
1707 }
1708 Constant *CV = ConstantVector::get(V: Tmp);
1709 Align Alignment(HwLen);
1710 SDValue CP = LowerConstantPool(
1711 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
1712 DAG);
1713 SDValue Bytes =
1714 DAG.getLoad(VT: ByteTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
1715 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
1716
1717 // Select the bytes that correspond to true bits in the vector predicate.
1718 SDValue Sel = DAG.getSelect(DL: dl, VT: VecTy, Cond: VecQ, LHS: DAG.getBitcast(VT: VecTy, V: Bytes),
1719 RHS: getZero(dl, Ty: VecTy, DAG));
1720 // Calculate the OR of all bytes in each group of 8. That will compress
1721 // all the individual bits into a single byte.
1722 // First, OR groups of 4, via vrmpy with 0x01010101.
1723 SDValue All1 =
1724 DAG.getSplatBuildVector(VT: MVT::v4i8, DL: dl, Op: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1725 SDValue Vrmpy = getInstr(MachineOpc: Hexagon::V6_vrmpyub, dl, Ty: ByteTy, Ops: {Sel, All1}, DAG);
1726 // Then rotate the accumulated vector by 4 bytes, and do the final OR.
1727 SDValue Rot = getInstr(MachineOpc: Hexagon::V6_valignbi, dl, Ty: ByteTy,
1728 Ops: {Vrmpy, Vrmpy, DAG.getTargetConstant(Val: 4, DL: dl, VT: MVT::i32)}, DAG);
1729 SDValue Vor = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, Ops: {Vrmpy, Rot});
1730
1731 // Pick every 8th byte and coalesce them at the beginning of the output.
1732 // For symmetry, coalesce every 1+8th byte after that, then every 2+8th
1733 // byte and so on.
1734 SmallVector<int,128> Mask;
1735 for (unsigned i = 0; i != HwLen; ++i)
1736 Mask.push_back(Elt: (8*i) % HwLen + i/(HwLen/8));
1737 SDValue Collect =
1738 DAG.getVectorShuffle(VT: ByteTy, dl, N1: Vor, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1739 return DAG.getBitcast(VT: ResTy, V: Collect);
1740}
1741
1742SDValue
1743HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed,
1744 const SDLoc &dl, SelectionDAG &DAG) const {
1745 // Take a vector and resize the element type to match the given type.
1746 MVT InpTy = ty(Op: VecV);
1747 if (InpTy == ResTy)
1748 return VecV;
1749
1750 unsigned InpWidth = InpTy.getSizeInBits();
1751 unsigned ResWidth = ResTy.getSizeInBits();
1752
1753 if (InpTy.isFloatingPoint()) {
1754 return InpWidth < ResWidth
1755 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: ResTy, Operand: VecV)
1756 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ResTy, N1: VecV,
1757 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
1758 }
1759
1760 assert(InpTy.isInteger());
1761
1762 if (InpWidth < ResWidth) {
1763 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1764 return DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ResTy, Operand: VecV);
1765 } else {
1766 unsigned NarOpc = Signed ? HexagonISD::SSAT : HexagonISD::USAT;
1767 return DAG.getNode(Opcode: NarOpc, DL: dl, VT: ResTy, N1: VecV, N2: DAG.getValueType(ResTy));
1768 }
1769}
1770
1771SDValue
1772HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx,
1773 SelectionDAG &DAG) const {
1774 assert(ty(Vec).getSizeInBits() % SubTy.getSizeInBits() == 0);
1775
1776 const SDLoc &dl(Vec);
1777 unsigned ElemIdx = SubIdx * SubTy.getVectorNumElements();
1778 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubTy,
1779 Ops: {Vec, DAG.getConstant(Val: ElemIdx, DL: dl, VT: MVT::i32)});
1780}
1781
1782SDValue
1783HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1784 const {
1785 const SDLoc &dl(Op);
1786 MVT VecTy = ty(Op);
1787
1788 unsigned Size = Op.getNumOperands();
1789 SmallVector<SDValue,128> Ops;
1790 for (unsigned i = 0; i != Size; ++i)
1791 Ops.push_back(Elt: Op.getOperand(i));
1792
1793 if (VecTy.getVectorElementType() == MVT::i1)
1794 return buildHvxVectorPred(Values: Ops, dl, VecTy, DAG);
1795
1796 // In case of MVT::f16 BUILD_VECTOR, since MVT::f16 is
1797 // not a legal type, just bitcast the node to use i16
1798 // types and bitcast the result back to f16
1799 if (VecTy.getVectorElementType() == MVT::f16 ||
1800 VecTy.getVectorElementType() == MVT::bf16) {
1801 SmallVector<SDValue, 64> NewOps;
1802 for (unsigned i = 0; i != Size; i++)
1803 NewOps.push_back(Elt: DAG.getBitcast(VT: MVT::i16, V: Ops[i]));
1804
1805 SDValue T0 =
1806 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: tyVector(Ty: VecTy, ElemTy: MVT::i16), Ops: NewOps);
1807 return DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: VecTy.getVectorElementType()), V: T0);
1808 }
1809
1810 // First, split the BUILD_VECTOR for vector pairs. We could generate
1811 // some pairs directly (via splat), but splats should be generated
1812 // by the combiner prior to getting here.
1813 if (VecTy.getSizeInBits() == 16 * Subtarget.getVectorLength()) {
1814 ArrayRef<SDValue> A(Ops);
1815 MVT SingleTy = typeSplit(VecTy).first;
1816 SDValue V0 = buildHvxVectorReg(Values: A.take_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1817 SDValue V1 = buildHvxVectorReg(Values: A.drop_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1818 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, N1: V0, N2: V1);
1819 }
1820
1821 return buildHvxVectorReg(Values: Ops, dl, VecTy, DAG);
1822}
1823
1824SDValue
1825HexagonTargetLowering::LowerHvxSplatVector(SDValue Op, SelectionDAG &DAG)
1826 const {
1827 const SDLoc &dl(Op);
1828 MVT VecTy = ty(Op);
1829 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
1830
1831 if (ArgTy == MVT::f16 || ArgTy == MVT::bf16) {
1832 MVT SplatTy = MVT::getVectorVT(VT: MVT::i16, NumElements: VecTy.getVectorNumElements());
1833 SDValue ToInt16 = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: 0));
1834 SDValue ToInt32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: ToInt16);
1835 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: SplatTy, Operand: ToInt32);
1836 return DAG.getBitcast(VT: VecTy, V: Splat);
1837 }
1838
1839 return SDValue();
1840}
1841
1842SDValue
1843HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1844 const {
1845 // Vector concatenation of two integer (non-bool) vectors does not need
1846 // special lowering. Custom-lower concats of bool vectors and expand
1847 // concats of more than 2 vectors.
1848 MVT VecTy = ty(Op);
1849 const SDLoc &dl(Op);
1850 unsigned NumOp = Op.getNumOperands();
1851 if (VecTy.getVectorElementType() != MVT::i1) {
1852 if (NumOp == 2)
1853 return Op;
1854 // Expand the other cases into a build-vector.
1855 SmallVector<SDValue,8> Elems;
1856 for (SDValue V : Op.getNode()->ops())
1857 DAG.ExtractVectorElements(Op: V, Args&: Elems);
1858 // A vector of i16 will be broken up into a build_vector of i16's.
1859 // This is a problem, since at the time of operation legalization,
1860 // all operations are expected to be type-legalized, and i16 is not
1861 // a legal type. If any of the extracted elements is not of a valid
1862 // type, sign-extend it to a valid one.
1863 for (SDValue &V : Elems) {
1864 MVT Ty = ty(Op: V);
1865 if (!isTypeLegal(VT: Ty)) {
1866 MVT NTy = typeLegalize(Ty, DAG);
1867 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1868 V = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NTy,
1869 N1: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NTy,
1870 N1: V.getOperand(i: 0), N2: V.getOperand(i: 1)),
1871 N2: DAG.getValueType(Ty));
1872 continue;
1873 }
1874 // A few less complicated cases.
1875 switch (V.getOpcode()) {
1876 case ISD::Constant:
1877 V = DAG.getSExtOrTrunc(Op: V, DL: dl, VT: NTy);
1878 break;
1879 case ISD::UNDEF:
1880 V = DAG.getUNDEF(VT: NTy);
1881 break;
1882 case ISD::TRUNCATE:
1883 V = V.getOperand(i: 0);
1884 break;
1885 default:
1886 llvm_unreachable("Unexpected vector element");
1887 }
1888 }
1889 }
1890 return DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Elems);
1891 }
1892
1893 assert(VecTy.getVectorElementType() == MVT::i1);
1894 unsigned HwLen = Subtarget.getVectorLength();
1895 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1896
1897 SDValue Op0 = Op.getOperand(i: 0);
1898
1899 // If the operands are HVX types (i.e. not scalar predicates), then
1900 // defer the concatenation, and create QCAT instead.
1901 if (Subtarget.isHVXVectorType(VecTy: ty(Op: Op0), IncludeBool: true)) {
1902 if (NumOp == 2)
1903 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: Op0, N2: Op.getOperand(i: 1));
1904
1905 ArrayRef<SDUse> U(Op.getNode()->ops());
1906 SmallVector<SDValue, 4> SV(U);
1907 ArrayRef<SDValue> Ops(SV);
1908
1909 MVT HalfTy = typeSplit(VecTy).first;
1910 SDValue V0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1911 Ops: Ops.take_front(N: NumOp/2));
1912 SDValue V1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1913 Ops: Ops.take_back(N: NumOp/2));
1914 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: V0, N2: V1);
1915 }
1916
1917 // Count how many bytes (in a vector register) each bit in VecTy
1918 // corresponds to.
1919 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1920
1921 // Make sure that createHvxPrefixPred will only ever need to expand
1922 // the predicate, i.e. bytes-per-bit in the input is not greater than
1923 // the target bytes-per-bit in the result.
1924 SDValue Combined = combineConcatOfScalarPreds(Op, BitBytes, DAG);
1925 SmallVector<SDValue,8> Prefixes;
1926 for (SDValue V : Combined.getNode()->op_values()) {
1927 SDValue P = createHvxPrefixPred(PredV: V, dl, BitBytes, ZeroFill: true, DAG);
1928 Prefixes.push_back(Elt: P);
1929 }
1930
1931 unsigned InpLen = ty(Op: Combined.getOperand(i: 0)).getVectorNumElements();
1932 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1933 SDValue S = DAG.getConstant(Val: HwLen - InpLen*BitBytes, DL: dl, VT: MVT::i32);
1934 SDValue Res = getZero(dl, Ty: ByteTy, DAG);
1935 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1936 Res = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Res, N2: S);
1937 Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, N1: Res, N2: Prefixes[e-i-1]);
1938 }
1939 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: Res);
1940}
1941
1942SDValue
1943HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1944 const {
1945 // Change the type of the extracted element to i32.
1946 SDValue VecV = Op.getOperand(i: 0);
1947 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1948 const SDLoc &dl(Op);
1949 SDValue IdxV = Op.getOperand(i: 1);
1950 if (ElemTy == MVT::i1)
1951 return extractHvxElementPred(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1952
1953 return extractHvxElementReg(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1954}
1955
1956SDValue
1957HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1958 const {
1959 const SDLoc &dl(Op);
1960 SDValue VecV = Op.getOperand(i: 0);
1961 SDValue ValV = Op.getOperand(i: 1);
1962 SDValue IdxV = Op.getOperand(i: 2);
1963 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1964 if (ElemTy == MVT::i1)
1965 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1966
1967 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1968}
1969
1970SDValue
1971HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1972 const {
1973 SDValue SrcV = Op.getOperand(i: 0);
1974 MVT SrcTy = ty(Op: SrcV);
1975 MVT DstTy = ty(Op);
1976 SDValue IdxV = Op.getOperand(i: 1);
1977 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1978 assert(Idx % DstTy.getVectorNumElements() == 0);
1979 (void)Idx;
1980 const SDLoc &dl(Op);
1981
1982 MVT ElemTy = SrcTy.getVectorElementType();
1983 if (ElemTy == MVT::i1)
1984 return extractHvxSubvectorPred(VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1985
1986 return extractHvxSubvectorReg(OrigOp: Op, VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1987}
1988
1989SDValue
1990HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1991 const {
1992 // Idx does not need to be a constant.
1993 SDValue VecV = Op.getOperand(i: 0);
1994 SDValue ValV = Op.getOperand(i: 1);
1995 SDValue IdxV = Op.getOperand(i: 2);
1996
1997 const SDLoc &dl(Op);
1998 MVT VecTy = ty(Op: VecV);
1999 MVT ElemTy = VecTy.getVectorElementType();
2000 if (ElemTy == MVT::i1)
2001 return insertHvxSubvectorPred(VecV, SubV: ValV, IdxV, dl, DAG);
2002
2003 return insertHvxSubvectorReg(VecV, SubV: ValV, IdxV, dl, DAG);
2004}
2005
2006SDValue
2007HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
2008 // Lower any-extends of boolean vectors to sign-extends, since they
2009 // translate directly to Q2V. Zero-extending could also be done equally
2010 // fast, but Q2V is used/recognized in more places.
2011 // For all other vectors, use zero-extend.
2012 MVT ResTy = ty(Op);
2013 SDValue InpV = Op.getOperand(i: 0);
2014 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2015 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2016 return LowerHvxSignExt(Op, DAG);
2017 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: ResTy, Operand: InpV);
2018}
2019
2020SDValue
2021HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
2022 MVT ResTy = ty(Op);
2023 SDValue InpV = Op.getOperand(i: 0);
2024 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2025 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2026 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: false, DAG);
2027 return Op;
2028}
2029
2030SDValue
2031HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
2032 MVT ResTy = ty(Op);
2033 SDValue InpV = Op.getOperand(i: 0);
2034 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2035 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2036 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: true, DAG);
2037 return Op;
2038}
2039
2040SDValue
2041HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
2042 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
2043 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
2044 const SDLoc &dl(Op);
2045 MVT ResTy = ty(Op);
2046 SDValue InpV = Op.getOperand(i: 0);
2047 assert(ResTy == ty(InpV));
2048
2049 // Calculate the vectors of 1 and bitwidth(x).
2050 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2051 unsigned ElemWidth = ElemTy.getSizeInBits();
2052
2053 SDValue Vec1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2054 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
2055 SDValue VecW = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2056 Operand: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2057 SDValue VecN1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2058 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
2059
2060 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
2061 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
2062 // it separately in custom combine or selection).
2063 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy,
2064 Ops: {DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {InpV, VecN1}),
2065 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {InpV, Vec1})});
2066 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy,
2067 Ops: {VecW, DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: ResTy, Operand: A)});
2068}
2069
2070SDValue
2071HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
2072 const SDLoc &dl(Op);
2073 MVT ResTy = ty(Op);
2074 assert(ResTy.getVectorElementType() == MVT::i32);
2075
2076 SDValue Vs = Op.getOperand(i: 0);
2077 SDValue Vt = Op.getOperand(i: 1);
2078
2079 SDVTList ResTys = DAG.getVTList(VT1: ResTy, VT2: ResTy);
2080 unsigned Opc = Op.getOpcode();
2081
2082 // On HVX v62+ producing the full product is cheap, so legalize MULH to LOHI.
2083 if (Opc == ISD::MULHU)
2084 return DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2085 if (Opc == ISD::MULHS)
2086 return DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2087
2088#ifndef NDEBUG
2089 Op.dump(&DAG);
2090#endif
2091 llvm_unreachable("Unexpected mulh operation");
2092}
2093
2094SDValue
2095HexagonTargetLowering::LowerHvxMulLoHi(SDValue Op, SelectionDAG &DAG) const {
2096 const SDLoc &dl(Op);
2097 unsigned Opc = Op.getOpcode();
2098 SDValue Vu = Op.getOperand(i: 0);
2099 SDValue Vv = Op.getOperand(i: 1);
2100
2101 // If the HI part is not used, convert it to a regular MUL.
2102 if (auto HiVal = Op.getValue(R: 1); HiVal.use_empty()) {
2103 // Need to preserve the types and the number of values.
2104 SDValue Hi = DAG.getUNDEF(VT: ty(Op: HiVal));
2105 SDValue Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: ty(Op), Ops: {Vu, Vv});
2106 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2107 }
2108
2109 bool SignedVu = Opc == HexagonISD::SMUL_LOHI;
2110 bool SignedVv = Opc == HexagonISD::SMUL_LOHI || Opc == HexagonISD::USMUL_LOHI;
2111
2112 // Legal on HVX v62+, but lower it here because patterns can't handle multi-
2113 // valued nodes.
2114 if (Subtarget.useHVXV62Ops())
2115 return emitHvxMulLoHiV62(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2116
2117 if (Opc == HexagonISD::SMUL_LOHI) {
2118 // Direct MULHS expansion is cheaper than doing the whole SMUL_LOHI,
2119 // for other signedness LOHI is cheaper.
2120 if (auto LoVal = Op.getValue(R: 0); LoVal.use_empty()) {
2121 SDValue Hi = emitHvxMulHsV60(A: Vu, B: Vv, dl, DAG);
2122 SDValue Lo = DAG.getUNDEF(VT: ty(Op: LoVal));
2123 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2124 }
2125 }
2126
2127 return emitHvxMulLoHiV60(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2128}
2129
2130SDValue
2131HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
2132 SDValue Val = Op.getOperand(i: 0);
2133 MVT ResTy = ty(Op);
2134 MVT ValTy = ty(Op: Val);
2135 const SDLoc &dl(Op);
2136
2137 if (isHvxBoolTy(Ty: ValTy) && ResTy.isScalarInteger()) {
2138 unsigned HwLen = Subtarget.getVectorLength();
2139 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
2140
2141 // When the predicate is shorter than the predicate register, each boolean
2142 // is represented by multiple consecutive bits in the input register.
2143 // Condense the bits so each boolean is represented by one bit. This only
2144 // handles 2x and 4x compaction ratios.
2145 unsigned PredLen = ValTy.getVectorNumElements();
2146 if (PredLen < HwLen) {
2147 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2148 Val = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Val);
2149 if (HwLen > PredLen * 2) {
2150 assert(HwLen == PredLen * 4);
2151 PredLen *= 2;
2152 Val = getInstr(MachineOpc: Hexagon::V6_vdealh, dl, Ty: ByteTy, Ops: Val, DAG);
2153 }
2154 if (HwLen > PredLen) {
2155 assert(HwLen == PredLen * 2);
2156 Val = getInstr(MachineOpc: Hexagon::V6_vdealb, dl, Ty: ByteTy, Ops: Val, DAG);
2157 }
2158 Val = DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ValTy, Operand: Val);
2159 }
2160
2161 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2162 unsigned BitWidth = ResTy.getSizeInBits();
2163
2164 if (BitWidth < 64) {
2165 SDValue W0 = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
2166 dl, ResTy: MVT::i32, DAG);
2167 if (BitWidth == 32)
2168 return W0;
2169 assert(BitWidth < 32u);
2170 return DAG.getZExtOrTrunc(Op: W0, DL: dl, VT: ResTy);
2171 }
2172
2173 // The result is >= 64 bits. The only options are 64 or 128.
2174 assert(BitWidth == 64 || BitWidth == 128);
2175 SmallVector<SDValue,4> Words;
2176 for (unsigned i = 0; i != BitWidth/32; ++i) {
2177 SDValue W = extractHvxElementReg(
2178 VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl, ResTy: MVT::i32, DAG);
2179 Words.push_back(Elt: W);
2180 }
2181 SmallVector<SDValue,2> Combines;
2182 assert(Words.size() % 2 == 0);
2183 for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
2184 SDValue C = getCombine(Hi: Words[i+1], Lo: Words[i], dl, ResTy: MVT::i64, DAG);
2185 Combines.push_back(Elt: C);
2186 }
2187
2188 if (BitWidth == 64)
2189 return Combines[0];
2190
2191 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: ResTy, Ops: Combines);
2192 }
2193
2194 // Handle bitcast from i32, v2i16, and v4i8 to v32i1.
2195 // Splat the input into a 32-element i32 vector, then AND each element
2196 // with a unique bitmask to isolate individual bits.
2197 auto bitcastI32ToV32I1 = [&](SDValue Val32) {
2198 assert(Val32.getValueType().getSizeInBits() == 32 &&
2199 "Input must be 32 bits");
2200 MVT VecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: 32);
2201 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Val32);
2202 SmallVector<SDValue, 32> Mask;
2203 for (unsigned i = 0; i < 32; ++i)
2204 Mask.push_back(Elt: DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32));
2205
2206 SDValue MaskVec = DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Mask);
2207 SDValue Anded = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: VecTy, N1: Splat, N2: MaskVec);
2208 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: MVT::v32i1, Operand: Anded);
2209 };
2210 // === Case: v32i1 ===
2211 if (ResTy == MVT::v32i1 &&
2212 (ValTy == MVT::i32 || ValTy == MVT::v2i16 || ValTy == MVT::v4i8) &&
2213 Subtarget.useHVX128BOps()) {
2214 SDValue Val32 = Val;
2215 if (ValTy == MVT::v2i16 || ValTy == MVT::v4i8)
2216 Val32 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Val);
2217 return bitcastI32ToV32I1(Val32);
2218 }
2219 // === Case: v64i1 ===
2220 if (ResTy == MVT::v64i1 && ValTy == MVT::i64 && Subtarget.useHVX128BOps()) {
2221 // Split i64 into lo/hi 32-bit halves.
2222 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Val);
2223 SDValue HiShifted = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Val,
2224 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64));
2225 SDValue Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: HiShifted);
2226
2227 // Reuse the same 32-bit logic twice.
2228 SDValue LoRes = bitcastI32ToV32I1(Lo);
2229 SDValue HiRes = bitcastI32ToV32I1(Hi);
2230
2231 // Concatenate into a v64i1 predicate.
2232 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v64i1, N1: LoRes, N2: HiRes);
2233 }
2234
2235 if (isHvxBoolTy(Ty: ResTy) && ValTy.isScalarInteger()) {
2236 // Handle bitcast from i128 -> v128i1 and i64 -> v64i1.
2237 unsigned BitWidth = ValTy.getSizeInBits();
2238 unsigned HwLen = Subtarget.getVectorLength();
2239 assert(BitWidth == HwLen);
2240
2241 MVT ValAsVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: BitWidth / 8);
2242 SDValue ValAsVec = DAG.getBitcast(VT: ValAsVecTy, V: Val);
2243 // Splat each byte of Val 8 times.
2244 // Bytes = [(b0)x8, (b1)x8, ...., (b15)x8]
2245 // where b0, b1,..., b15 are least to most significant bytes of I.
2246 SmallVector<SDValue, 128> Bytes;
2247 // Tmp: 0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80, 0x01,0x02,0x04,0x08,...
2248 // These are bytes with the LSB rotated left with respect to their index.
2249 SmallVector<SDValue, 128> Tmp;
2250 for (unsigned I = 0; I != HwLen / 8; ++I) {
2251 SDValue Idx = DAG.getConstant(Val: I, DL: dl, VT: MVT::i32);
2252 SDValue Byte =
2253 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i8, N1: ValAsVec, N2: Idx);
2254 for (unsigned J = 0; J != 8; ++J) {
2255 Bytes.push_back(Elt: Byte);
2256 Tmp.push_back(Elt: DAG.getConstant(Val: 1ull << J, DL: dl, VT: MVT::i8));
2257 }
2258 }
2259
2260 MVT ConstantVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2261 SDValue ConstantVec = DAG.getBuildVector(VT: ConstantVecTy, DL: dl, Ops: Tmp);
2262 SDValue I2V = buildHvxVectorReg(Values: Bytes, dl, VecTy: ConstantVecTy, DAG);
2263
2264 // Each Byte in the I2V will be set iff corresponding bit is set in Val.
2265 I2V = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ConstantVecTy, Ops: {I2V, ConstantVec});
2266 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: I2V);
2267 }
2268
2269 return Op;
2270}
2271
2272SDValue HexagonTargetLowering::LowerHvxStore(SDValue Op,
2273 SelectionDAG &DAG) const {
2274 const SDLoc &dl(Op);
2275 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
2276 SDValue Val = SN->getValue();
2277 MVT ValTy = ty(Op: Val);
2278
2279 // Check if this is a store of an HVX bool vector (predicate)
2280 if (!isHvxBoolTy(Ty: ValTy))
2281 return SDValue();
2282
2283 unsigned NumBits = ValTy.getVectorNumElements();
2284 MachineMemOperand *MMO = SN->getMemOperand();
2285
2286 // Check alignment requirements based on predicate size
2287 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2288 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2289 return SDValue();
2290
2291 unsigned HwLen = Subtarget.getVectorLength();
2292 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
2293
2294 // Compress the predicate into a vector register
2295 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2296
2297 // Extract words from the compressed vector
2298 SmallVector<SDValue, 4> Words;
2299 for (unsigned i = 0; i != NumBits / 32; ++i) {
2300 SDValue W = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl,
2301 ResTy: MVT::i32, DAG);
2302 Words.push_back(Elt: W);
2303 }
2304
2305 SDValue Chain = SN->getChain();
2306 SDValue BasePtr = SN->getBasePtr();
2307 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2308
2309 if (NumBits == 32)
2310 return DAG.getStore(Chain, dl, Val: Words[0], Ptr: BasePtr, PtrInfo,
2311 Alignment: MMO->getBaseAlign());
2312
2313 if (NumBits == 64) {
2314 SDValue W64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2315 return DAG.getStore(Chain, dl, Val: W64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2316 }
2317
2318 if (NumBits == 128) {
2319 SDValue Lo64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2320 SDValue Hi64 = getCombine(Hi: Words[3], Lo: Words[2], dl, ResTy: MVT::i64, DAG);
2321
2322 Chain =
2323 DAG.getStore(Chain, dl, Val: Lo64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2324
2325 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2326 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2327 return DAG.getStore(Chain, dl, Val: Hi64, Ptr: Ptr8, PtrInfo: PtrInfo.getWithOffset(O: 8),
2328 Alignment: Align(8));
2329 }
2330
2331 return SDValue();
2332}
2333
2334SDValue HexagonTargetLowering::LowerHvxLoad(SDValue Op,
2335 SelectionDAG &DAG) const {
2336 const SDLoc &dl(Op);
2337 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
2338 MVT ResTy = ty(Op);
2339
2340 // Check if this is a load of an HVX bool vector (predicate)
2341 if (!isHvxBoolTy(Ty: ResTy))
2342 return SDValue();
2343
2344 unsigned NumBits = ResTy.getVectorNumElements();
2345 MachineMemOperand *MMO = LN->getMemOperand();
2346
2347 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2348 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2349 return SDValue();
2350
2351 SDValue Chain = LN->getChain();
2352 SDValue BasePtr = LN->getBasePtr();
2353 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2354
2355 if (NumBits == 32) {
2356 SDValue W32 =
2357 DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2358 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v32i1, Operand: W32);
2359 SDValue Ops[] = {Pred, W32.getValue(R: 1)};
2360 return DAG.getMergeValues(Ops, dl);
2361 }
2362
2363 if (NumBits == 64) {
2364 SDValue W64 =
2365 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2366 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: W64);
2367 SDValue Ops[] = {Pred, W64.getValue(R: 1)};
2368 return DAG.getMergeValues(Ops, dl);
2369 }
2370
2371 if (NumBits == 128) {
2372 SDValue Lo64 =
2373 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2374 Chain = Lo64.getValue(R: 1);
2375
2376 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2377 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2378 SDValue Hi64 = DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: Ptr8,
2379 PtrInfo: PtrInfo.getWithOffset(O: 8), Alignment: Align(8));
2380
2381 SDValue LoPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Lo64);
2382 SDValue HiPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Hi64);
2383 SDValue Pred =
2384 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v128i1, N1: LoPred, N2: HiPred);
2385
2386 SDValue Ops[] = {Pred, Hi64.getValue(R: 1)};
2387 return DAG.getMergeValues(Ops, dl);
2388 }
2389
2390 return SDValue();
2391}
2392
2393SDValue
2394HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
2395 // Sign- and zero-extends are legal.
2396 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
2397 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL: SDLoc(Op), VT: ty(Op),
2398 Operand: Op.getOperand(i: 0));
2399}
2400
2401SDValue
2402HexagonTargetLowering::LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const {
2403 MVT ResTy = ty(Op);
2404 if (ResTy.getVectorElementType() != MVT::i1)
2405 return Op;
2406
2407 const SDLoc &dl(Op);
2408 unsigned HwLen = Subtarget.getVectorLength();
2409 unsigned VecLen = ResTy.getVectorNumElements();
2410 assert(HwLen % VecLen == 0);
2411 unsigned ElemSize = HwLen / VecLen;
2412
2413 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemSize * 8), NumElements: VecLen);
2414 SDValue S =
2415 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: Op.getOperand(i: 0),
2416 N2: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 1)),
2417 N3: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 2)));
2418 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: S);
2419}
2420
2421SDValue
2422HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
2423 if (SDValue S = getVectorShiftByInt(Op, DAG))
2424 return S;
2425 return Op;
2426}
2427
2428SDValue
2429HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op,
2430 SelectionDAG &DAG) const {
2431 unsigned Opc = Op.getOpcode();
2432 assert(Opc == ISD::FSHL || Opc == ISD::FSHR);
2433
2434 // Make sure the shift amount is within the range of the bitwidth
2435 // of the element type.
2436 SDValue A = Op.getOperand(i: 0);
2437 SDValue B = Op.getOperand(i: 1);
2438 SDValue S = Op.getOperand(i: 2);
2439
2440 MVT InpTy = ty(Op: A);
2441 MVT ElemTy = InpTy.getVectorElementType();
2442
2443 const SDLoc &dl(Op);
2444 unsigned ElemWidth = ElemTy.getSizeInBits();
2445 bool IsLeft = Opc == ISD::FSHL;
2446
2447 // The expansion into regular shifts produces worse code for i8 and for
2448 // right shift of i32 on v65+.
2449 bool UseShifts = ElemTy != MVT::i8;
2450 if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32)
2451 UseShifts = false;
2452
2453 if (SDValue SplatV = getSplatValue(Op: S, DAG); SplatV && UseShifts) {
2454 // If this is a funnel shift by a scalar, lower it into regular shifts.
2455 SDValue Mask = DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: MVT::i32);
2456 SDValue ModS =
2457 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
2458 Ops: {DAG.getZExtOrTrunc(Op: SplatV, DL: dl, VT: MVT::i32), Mask});
2459 SDValue NegS =
2460 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2461 Ops: {DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32), ModS});
2462 SDValue IsZero =
2463 DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ModS, RHS: getZero(dl, Ty: MVT::i32, DAG), Cond: ISD::SETEQ);
2464 // FSHL A, B => A << | B >>n
2465 // FSHR A, B => A <<n | B >>
2466 SDValue Part1 =
2467 DAG.getNode(Opcode: HexagonISD::VASL, DL: dl, VT: InpTy, Ops: {A, IsLeft ? ModS : NegS});
2468 SDValue Part2 =
2469 DAG.getNode(Opcode: HexagonISD::VLSR, DL: dl, VT: InpTy, Ops: {B, IsLeft ? NegS : ModS});
2470 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Part1, Part2});
2471 // If the shift amount was 0, pick A or B, depending on the direction.
2472 // The opposite shift will also be by 0, so the "Or" will be incorrect.
2473 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InpTy, Ops: {IsZero, (IsLeft ? A : B), Or});
2474 }
2475
2476 SDValue Mask = DAG.getSplatBuildVector(
2477 VT: InpTy, DL: dl, Op: DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: ElemTy));
2478
2479 unsigned MOpc = Opc == ISD::FSHL ? HexagonISD::MFSHL : HexagonISD::MFSHR;
2480 return DAG.getNode(Opcode: MOpc, DL: dl, VT: ty(Op),
2481 Ops: {A, B, DAG.getNode(Opcode: ISD::AND, DL: dl, VT: InpTy, Ops: {S, Mask})});
2482}
2483
2484SDValue
2485HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
2486 const SDLoc &dl(Op);
2487 unsigned IntNo = Op.getConstantOperandVal(i: 0);
2488 SmallVector<SDValue> Ops(Op->ops());
2489
2490 auto Swap = [&](SDValue P) {
2491 return DAG.getMergeValues(Ops: {P.getValue(R: 1), P.getValue(R: 0)}, dl);
2492 };
2493
2494 const SDLoc &DL(Op);
2495 switch (IntNo) {
2496 // Translate IEEE intrinsic to XQF.
2497 // In v79, the instruction v0.h = vcvt(v0.hf) is not an exact conversion.
2498 // The bit exactness is mimicked using a library call.
2499 case Intrinsic::hexagon_V6_vcvt_b_hf_128B:
2500 case Intrinsic::hexagon_V6_vcvt_h_hf_128B:
2501 case Intrinsic::hexagon_V6_vcvt_ub_hf_128B:
2502 case Intrinsic::hexagon_V6_vcvt_uh_hf_128B: {
2503 MachineFunction &MF = DAG.getMachineFunction();
2504 auto &HST = MF.getSubtarget<HexagonSubtarget>();
2505 // Generate the call only on v79 only.
2506 if (!HST.useHVXV79OpsOnly())
2507 return Op;
2508
2509 ArgListTy Args;
2510 EVT ArgVT;
2511 for (unsigned i = 1; i < Op.getNumOperands(); ++i) {
2512 SDValue Arg = Op.getOperand(i);
2513 ArgVT = Arg.getValueType();
2514 assert(ArgVT.isVector() &&
2515 "Expecting a vector argument type for V6_vcvt_h_hf instrinsic.");
2516 ArgListEntry Entry(Arg, ArgVT.getTypeForEVT(Context&: *DAG.getContext()));
2517 Args.push_back(x: Entry);
2518 }
2519 Type *RetTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2520
2521 bool LongCalls = HST.useLongCalls();
2522 unsigned Flag = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
2523
2524 const char *Name = nullptr;
2525 switch (IntNo) {
2526 case Intrinsic::hexagon_V6_vcvt_b_hf_128B: {
2527 Name = EnableXQFFastConvert ? "__qf_fast_convert_hf_to_b_rne"
2528 : "__qf_convert_hf_to_b_rne";
2529 break;
2530 }
2531 case Intrinsic::hexagon_V6_vcvt_h_hf_128B: {
2532 Name = EnableXQFFastConvert ? "__qf_fast_convert_hf_to_h_rne"
2533 : "__qf_convert_hf_to_h_rne";
2534 break;
2535 }
2536 case Intrinsic::hexagon_V6_vcvt_ub_hf_128B: {
2537 Name = EnableXQFFastConvert ? "__qf_fast_convert_hf_to_ub_rne"
2538 : "__qf_convert_hf_to_ub_rne";
2539 break;
2540 }
2541 case Intrinsic::hexagon_V6_vcvt_uh_hf_128B: {
2542 Name = EnableXQFFastConvert ? "__qf_fast_convert_hf_to_uh_rne"
2543 : "__qf_convert_hf_to_uh_rne";
2544 break;
2545 }
2546 default:
2547 report_fatal_error(reason: "Unhandled intrinsic '" +
2548 Intrinsic::getName(id: static_cast<Intrinsic::ID>(IntNo)) +
2549 "' in HexagonTargetLowering::LowerHvxIntrinsic");
2550 }
2551
2552 SDValue Callee = DAG.getTargetExternalSymbol(
2553 Sym: Name, VT: getPointerTy(DL: DAG.getDataLayout()), TargetFlags: Flag);
2554
2555 TargetLowering::CallLoweringInfo CLI(DAG);
2556 CLI.setDebugLoc(DL)
2557 .setChain(DAG.getEntryNode())
2558 .setLibCallee(CC: CallingConv::Fast, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args));
2559 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2560 return CallResult.first;
2561 }
2562 case Intrinsic::hexagon_V6_pred_typecast:
2563 case Intrinsic::hexagon_V6_pred_typecast_128B: {
2564 MVT ResTy = ty(Op), InpTy = ty(Op: Ops[1]);
2565 if (isHvxBoolTy(Ty: ResTy) && isHvxBoolTy(Ty: InpTy)) {
2566 if (ResTy == InpTy)
2567 return Ops[1];
2568 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: ResTy, Operand: Ops[1]);
2569 }
2570 break;
2571 }
2572 case Intrinsic::hexagon_V6_vmpyss_parts:
2573 case Intrinsic::hexagon_V6_vmpyss_parts_128B:
2574 return Swap(DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2575 Ops: {Ops[1], Ops[2]}));
2576 case Intrinsic::hexagon_V6_vmpyuu_parts:
2577 case Intrinsic::hexagon_V6_vmpyuu_parts_128B:
2578 return Swap(DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2579 Ops: {Ops[1], Ops[2]}));
2580 case Intrinsic::hexagon_V6_vmpyus_parts:
2581 case Intrinsic::hexagon_V6_vmpyus_parts_128B: {
2582 return Swap(DAG.getNode(Opcode: HexagonISD::USMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2583 Ops: {Ops[1], Ops[2]}));
2584 }
2585 } // switch
2586
2587 return Op;
2588}
2589
2590SDValue
2591HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const {
2592 const SDLoc &dl(Op);
2593 unsigned HwLen = Subtarget.getVectorLength();
2594 MachineFunction &MF = DAG.getMachineFunction();
2595 auto *MaskN = cast<MaskedLoadStoreSDNode>(Val: Op.getNode());
2596 SDValue Mask = MaskN->getMask();
2597 SDValue Chain = MaskN->getChain();
2598 SDValue Base = MaskN->getBasePtr();
2599 auto *MemOp = MF.getMachineMemOperand(MMO: MaskN->getMemOperand(), Offset: 0, Size: HwLen);
2600
2601 unsigned Opc = Op->getOpcode();
2602 assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE);
2603
2604 if (Opc == ISD::MLOAD) {
2605 MVT ValTy = ty(Op);
2606 SDValue Load = DAG.getLoad(VT: ValTy, dl, Chain, Ptr: Base, MMO: MemOp);
2607 SDValue Thru = cast<MaskedLoadSDNode>(Val: MaskN)->getPassThru();
2608 if (isUndef(Op: Thru))
2609 return Load;
2610 SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: Mask, N2: Load, N3: Thru);
2611 return DAG.getMergeValues(Ops: {VSel, Load.getValue(R: 1)}, dl);
2612 }
2613
2614 // MSTORE
2615 // HVX only has aligned masked stores.
2616
2617 // TODO: Fold negations of the mask into the store.
2618 unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
2619 SDValue Value = cast<MaskedStoreSDNode>(Val: MaskN)->getValue();
2620 SDValue Offset0 = DAG.getTargetConstant(Val: 0, DL: dl, VT: ty(Op: Base));
2621
2622 if (MaskN->getAlign().value() % HwLen == 0) {
2623 SDValue Store = getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2624 Ops: {Mask, Base, Offset0, Value, Chain}, DAG);
2625 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Store.getNode()), NewMemRefs: {MemOp});
2626 return Store;
2627 }
2628
2629 // Unaligned case.
2630 auto StoreAlign = [&](SDValue V, SDValue A) {
2631 SDValue Z = getZero(dl, Ty: ty(Op: V), DAG);
2632 // TODO: use funnel shifts?
2633 // vlalign(Vu,Vv,Rt) rotates the pair Vu:Vv left by Rt and takes the
2634 // upper half.
2635 SDValue LoV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {V, Z, A}, DAG);
2636 SDValue HiV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {Z, V, A}, DAG);
2637 return std::make_pair(x&: LoV, y&: HiV);
2638 };
2639
2640 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2641 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
2642 SDValue MaskV = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Mask);
2643 VectorPair Tmp = StoreAlign(MaskV, Base);
2644 VectorPair MaskU = {DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.first),
2645 DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.second)};
2646 VectorPair ValueU = StoreAlign(Value, Base);
2647
2648 SDValue Offset1 = DAG.getTargetConstant(Val: HwLen, DL: dl, VT: MVT::i32);
2649 SDValue StoreLo =
2650 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2651 Ops: {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
2652 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreLo.getNode()), NewMemRefs: {MemOp});
2653
2654 // If the store fits within one HwLen-aligned block, the high half's predicate
2655 // is always all-zeros and the vmem(Base+HwLen) can be elided entirely.
2656 // Proof: addr % StoreAlign == 0 and StoreMemSize <= StoreAlign implies
2657 // addr % HwLen <= HwLen - StoreAlign, so addr % HwLen + StoreMemSize
2658 // <= HwLen.
2659 // Without this guard, Hexagon v73+ probes the TLB for vmem(Base+HwLen) even
2660 // when the predicate is all-zeros, causing a TLBMISS if that page is
2661 // unmapped.
2662 uint64_t StoreMemSize = MaskN->getMemoryVT().getStoreSize().getFixedValue();
2663 if (StoreMemSize <= MaskN->getAlign().value())
2664 return StoreLo;
2665
2666 SDValue StoreHi =
2667 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2668 Ops: {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
2669 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreHi.getNode()), NewMemRefs: {MemOp});
2670 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: {StoreLo, StoreHi});
2671}
2672
2673SDValue HexagonTargetLowering::LowerHvxFpExtend(SDValue Op,
2674 SelectionDAG &DAG) const {
2675 // This conversion only applies to QFloat. IEEE extension from f16 to f32
2676 // is legal (done via a pattern).
2677 assert(Subtarget.useHVXQFloatOps());
2678
2679 assert(Op->getOpcode() == ISD::FP_EXTEND);
2680
2681 MVT VecTy = ty(Op);
2682 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
2683 const SDLoc &dl(Op);
2684
2685 if (ArgTy == MVT::v64bf16) {
2686 MVT HalfTy = typeSplit(VecTy).first;
2687 SDValue BF16Vec = Op.getOperand(i: 0);
2688 SDValue Zeroes =
2689 getInstr(MachineOpc: Hexagon::V6_vxor, dl, Ty: HalfTy, Ops: {BF16Vec, BF16Vec}, DAG);
2690 // Interleave zero vector with the bf16 vector, with zeroes in the lower
2691 // half of each 32 bit lane, effectively extending the bf16 values to fp32
2692 // values.
2693 SDValue ShuffVec =
2694 getInstr(MachineOpc: Hexagon::V6_vshufoeh, dl, Ty: VecTy, Ops: {BF16Vec, Zeroes}, DAG);
2695 VectorPair VecPair = opSplit(Vec: ShuffVec, dl, DAG);
2696 SDValue Result = getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2697 Ops: {VecPair.second, VecPair.first,
2698 DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)},
2699 DAG);
2700 return Result;
2701 }
2702
2703 assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16);
2704
2705 SDValue F16Vec = Op.getOperand(i: 0);
2706
2707 APFloat FloatVal = APFloat(1.0f);
2708 bool Ignored;
2709 FloatVal.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
2710 SDValue Fp16Ones = DAG.getConstantFP(Val: FloatVal, DL: dl, VT: ArgTy);
2711 SDValue VmpyVec =
2712 getInstr(MachineOpc: Hexagon::V6_vmpy_qf32_hf, dl, Ty: VecTy, Ops: {F16Vec, Fp16Ones}, DAG);
2713
2714 MVT HalfTy = typeSplit(VecTy).first;
2715 VectorPair Pair = opSplit(Vec: VmpyVec, dl, DAG);
2716 SDValue LoVec =
2717 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.first}, DAG);
2718 SDValue HiVec =
2719 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.second}, DAG);
2720
2721 SDValue ShuffVec =
2722 getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2723 Ops: {HiVec, LoVec, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)}, DAG);
2724
2725 return ShuffVec;
2726}
2727
2728SDValue
2729HexagonTargetLowering::LowerHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
2730 // Catch invalid conversion ops (just in case).
2731 assert(Op.getOpcode() == ISD::FP_TO_SINT ||
2732 Op.getOpcode() == ISD::FP_TO_UINT);
2733
2734 MVT ResTy = ty(Op);
2735 MVT FpTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2736 MVT IntTy = ResTy.getVectorElementType();
2737
2738 if (Subtarget.useHVXIEEEFPOps()) {
2739 // There are only conversions from f16.
2740 if (FpTy == MVT::f16) {
2741 // Other int types aren't legal in HVX, so we shouldn't see them here.
2742 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2743 // Conversions to i8 and i16 are legal.
2744 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2745 return Op;
2746 }
2747 }
2748
2749 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2750 return EqualizeFpIntConversion(Op, DAG);
2751
2752 return ExpandHvxFpToInt(Op, DAG);
2753}
2754
2755// For vector type v32i1 uint_to_fp/sint_to_fp to v32f32:
2756// R1 = #1, R2 holds the v32i1 param
2757// V1 = vsplat(R1)
2758// V2 = vsplat(R2)
2759// Q0 = vand(V1,R1)
2760// V0.w=prefixsum(Q0)
2761// V0.w=vsub(V0.w,V1.w)
2762// V2.w = vlsr(V2.w,V0.w)
2763// V2 = vand(V2,V1)
2764// V2.sf = V2.w
2765SDValue HexagonTargetLowering::LowerHvxPred32ToFp(SDValue PredOp,
2766 SelectionDAG &DAG) const {
2767
2768 MVT ResTy = ty(Op: PredOp);
2769 const SDLoc &dl(PredOp);
2770
2771 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2772 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2773 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2774 Op1: SDValue(RegConst, 0));
2775 SDNode *PredTransfer =
2776 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2777 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2778 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2779 Op1: SDValue(PredTransfer, 0));
2780 SDNode *SplatParam = DAG.getMachineNode(
2781 Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2782 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: PredOp.getOperand(i: 0)));
2783 SDNode *Vsub =
2784 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2785 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2786 SDNode *IndexShift =
2787 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2788 Op1: SDValue(SplatParam, 0), Op2: SDValue(Vsub, 0));
2789 SDNode *MaskOff =
2790 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2791 Op1: SDValue(IndexShift, 0), Op2: SDValue(SplatConst, 0));
2792 SDNode *Convert = DAG.getMachineNode(Opcode: Hexagon::V6_vconv_sf_w, dl, VT: ResTy,
2793 Op1: SDValue(MaskOff, 0));
2794 return SDValue(Convert, 0);
2795}
2796
2797// For vector type v64i1 uint_to_fo to v64f16:
2798// i64 R32 = bitcast v64i1 R3:2 (R3:2 holds v64i1)
2799// R3 = subreg_high (R32)
2800// R2 = subreg_low (R32)
2801// R1 = #1
2802// V1 = vsplat(R1)
2803// V2 = vsplat(R2)
2804// V3 = vsplat(R3)
2805// Q0 = vand(V1,R1)
2806// V0.w=prefixsum(Q0)
2807// V0.w=vsub(V0.w,V1.w)
2808// V2.w = vlsr(V2.w,V0.w)
2809// V3.w = vlsr(V3.w,V0.w)
2810// V2 = vand(V2,V1)
2811// V3 = vand(V3,V1)
2812// V2.h = vpacke(V3.w,V2.w)
2813// V2.hf = V2.h
2814SDValue HexagonTargetLowering::LowerHvxPred64ToFp(SDValue PredOp,
2815 SelectionDAG &DAG) const {
2816
2817 MVT ResTy = ty(Op: PredOp);
2818 const SDLoc &dl(PredOp);
2819
2820 SDValue Inp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: PredOp.getOperand(i: 0));
2821 // Get the hi and lo regs
2822 SDValue HiReg =
2823 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_hi, DL: dl, VT: MVT::i32, Operand: Inp);
2824 SDValue LoReg =
2825 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_lo, DL: dl, VT: MVT::i32, Operand: Inp);
2826 // Get constant #1 and splat into vector V1
2827 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2828 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2829 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2830 Op1: SDValue(RegConst, 0));
2831 // Splat the hi and lo args
2832 SDNode *SplatHi =
2833 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2834 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: HiReg));
2835 SDNode *SplatLo =
2836 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2837 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: LoReg));
2838 // vand between splatted const and const
2839 SDNode *PredTransfer =
2840 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2841 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2842 // Get the prefixsum
2843 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2844 Op1: SDValue(PredTransfer, 0));
2845 // Get the vsub
2846 SDNode *Vsub =
2847 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2848 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2849 // Get vlsr for hi and lo
2850 SDNode *IndexShift_hi =
2851 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2852 Op1: SDValue(SplatHi, 0), Op2: SDValue(Vsub, 0));
2853 SDNode *IndexShift_lo =
2854 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2855 Op1: SDValue(SplatLo, 0), Op2: SDValue(Vsub, 0));
2856 // Get vand of hi and lo
2857 SDNode *MaskOff_hi =
2858 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2859 Op1: SDValue(IndexShift_hi, 0), Op2: SDValue(SplatConst, 0));
2860 SDNode *MaskOff_lo =
2861 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2862 Op1: SDValue(IndexShift_lo, 0), Op2: SDValue(SplatConst, 0));
2863 // Pack them
2864 SDNode *Pack =
2865 DAG.getMachineNode(Opcode: Hexagon::V6_vpackeh, dl, VT: MVT::v64i16,
2866 Op1: SDValue(MaskOff_hi, 0), Op2: SDValue(MaskOff_lo, 0));
2867 SDNode *Convert =
2868 DAG.getMachineNode(Opcode: Hexagon::V6_vconv_hf_h, dl, VT: ResTy, Op1: SDValue(Pack, 0));
2869 return SDValue(Convert, 0);
2870}
2871
2872SDValue
2873HexagonTargetLowering::LowerHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
2874 // Catch invalid conversion ops (just in case).
2875 assert(Op.getOpcode() == ISD::SINT_TO_FP ||
2876 Op.getOpcode() == ISD::UINT_TO_FP);
2877
2878 MVT ResTy = ty(Op);
2879 MVT IntTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2880 MVT FpTy = ResTy.getVectorElementType();
2881
2882 if (Op.getOpcode() == ISD::UINT_TO_FP || Op.getOpcode() == ISD::SINT_TO_FP) {
2883 if (ResTy == MVT::v32f32 && ty(Op: Op.getOperand(i: 0)) == MVT::v32i1)
2884 return LowerHvxPred32ToFp(PredOp: Op, DAG);
2885 if (ResTy == MVT::v64f16 && ty(Op: Op.getOperand(i: 0)) == MVT::v64i1)
2886 return LowerHvxPred64ToFp(PredOp: Op, DAG);
2887 }
2888
2889 if (Subtarget.useHVXIEEEFPOps()) {
2890 // There are only conversions to f16.
2891 if (FpTy == MVT::f16) {
2892 // Other int types aren't legal in HVX, so we shouldn't see them here.
2893 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2894 // i8, i16 -> f16 is legal.
2895 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2896 return Op;
2897 }
2898 }
2899
2900 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2901 return EqualizeFpIntConversion(Op, DAG);
2902
2903 return ExpandHvxIntToFp(Op, DAG);
2904}
2905
2906HexagonTargetLowering::TypePair
2907HexagonTargetLowering::typeExtendToWider(MVT Ty0, MVT Ty1) const {
2908 // Compare the widths of elements of the two types, and extend the narrower
2909 // type to match the with of the wider type. For vector types, apply this
2910 // to the element type.
2911 assert(Ty0.isVector() == Ty1.isVector());
2912
2913 MVT ElemTy0 = Ty0.getScalarType();
2914 MVT ElemTy1 = Ty1.getScalarType();
2915
2916 unsigned Width0 = ElemTy0.getSizeInBits();
2917 unsigned Width1 = ElemTy1.getSizeInBits();
2918 unsigned MaxWidth = std::max(a: Width0, b: Width1);
2919
2920 auto getScalarWithWidth = [](MVT ScalarTy, unsigned Width) {
2921 if (ScalarTy.isInteger())
2922 return MVT::getIntegerVT(BitWidth: Width);
2923 assert(ScalarTy.isFloatingPoint());
2924 return MVT::getFloatingPointVT(BitWidth: Width);
2925 };
2926
2927 MVT WideETy0 = getScalarWithWidth(ElemTy0, MaxWidth);
2928 MVT WideETy1 = getScalarWithWidth(ElemTy1, MaxWidth);
2929
2930 if (!Ty0.isVector()) {
2931 // Both types are scalars.
2932 return {WideETy0, WideETy1};
2933 }
2934
2935 // Vector types.
2936 unsigned NumElem = Ty0.getVectorNumElements();
2937 assert(NumElem == Ty1.getVectorNumElements());
2938
2939 return {MVT::getVectorVT(VT: WideETy0, NumElements: NumElem),
2940 MVT::getVectorVT(VT: WideETy1, NumElements: NumElem)};
2941}
2942
2943HexagonTargetLowering::TypePair
2944HexagonTargetLowering::typeWidenToWider(MVT Ty0, MVT Ty1) const {
2945 // Compare the numbers of elements of two vector types, and widen the
2946 // narrower one to match the number of elements in the wider one.
2947 assert(Ty0.isVector() && Ty1.isVector());
2948
2949 unsigned Len0 = Ty0.getVectorNumElements();
2950 unsigned Len1 = Ty1.getVectorNumElements();
2951 if (Len0 == Len1)
2952 return {Ty0, Ty1};
2953
2954 unsigned MaxLen = std::max(a: Len0, b: Len1);
2955 return {MVT::getVectorVT(VT: Ty0.getVectorElementType(), NumElements: MaxLen),
2956 MVT::getVectorVT(VT: Ty1.getVectorElementType(), NumElements: MaxLen)};
2957}
2958
2959MVT
2960HexagonTargetLowering::typeLegalize(MVT Ty, SelectionDAG &DAG) const {
2961 EVT LegalTy = getTypeToTransformTo(Context&: *DAG.getContext(), VT: Ty);
2962 assert(LegalTy.isSimple());
2963 return LegalTy.getSimpleVT();
2964}
2965
2966MVT
2967HexagonTargetLowering::typeWidenToHvx(MVT Ty) const {
2968 unsigned HwWidth = 8 * Subtarget.getVectorLength();
2969 assert(Ty.getSizeInBits() <= HwWidth);
2970 if (Ty.getSizeInBits() == HwWidth)
2971 return Ty;
2972
2973 MVT ElemTy = Ty.getScalarType();
2974 return MVT::getVectorVT(VT: ElemTy, NumElements: HwWidth / ElemTy.getSizeInBits());
2975}
2976
2977HexagonTargetLowering::VectorPair
2978HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B,
2979 const SDLoc &dl, bool Signed, SelectionDAG &DAG) const {
2980 // Compute A+B, return {A+B, O}, where O = vector predicate indicating
2981 // whether an overflow has occurred.
2982 MVT ResTy = ty(Op: A);
2983 assert(ResTy == ty(B));
2984 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: ResTy.getVectorNumElements());
2985
2986 if (!Signed) {
2987 // V62+ has V6_vaddcarry, but it requires input predicate, so it doesn't
2988 // save any instructions.
2989 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2990 SDValue Ovf = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Add, RHS: A, Cond: ISD::SETULT);
2991 return {Add, Ovf};
2992 }
2993
2994 // Signed overflow has happened, if:
2995 // (A, B have the same sign) and (A+B has a different sign from either)
2996 // i.e. (~A xor B) & ((A+B) xor B), then check the sign bit
2997 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2998 SDValue NotA =
2999 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {A, DAG.getAllOnesConstant(DL: dl, VT: ResTy)});
3000 SDValue Xor0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {NotA, B});
3001 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {Add, B});
3002 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy, Ops: {Xor0, Xor1});
3003 SDValue MSB =
3004 DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: getZero(dl, Ty: ResTy, DAG), Cond: ISD::SETLT);
3005 return {Add, MSB};
3006}
3007
3008HexagonTargetLowering::VectorPair
3009HexagonTargetLowering::emitHvxShiftRightRnd(SDValue Val, unsigned Amt,
3010 bool Signed, SelectionDAG &DAG) const {
3011 // Shift Val right by Amt bits, round the result to the nearest integer,
3012 // tie-break by rounding halves to even integer.
3013
3014 const SDLoc &dl(Val);
3015 MVT ValTy = ty(Op: Val);
3016
3017 // This should also work for signed integers.
3018 //
3019 // uint tmp0 = inp + ((1 << (Amt-1)) - 1);
3020 // bool ovf = (inp > tmp0);
3021 // uint rup = inp & (1 << (Amt+1));
3022 //
3023 // uint tmp1 = inp >> (Amt-1); // tmp1 == tmp2 iff
3024 // uint tmp2 = tmp0 >> (Amt-1); // the Amt-1 lower bits were all 0
3025 // uint tmp3 = tmp2 + rup;
3026 // uint frac = (tmp1 != tmp2) ? tmp2 >> 1 : tmp3 >> 1;
3027 unsigned ElemWidth = ValTy.getVectorElementType().getSizeInBits();
3028 MVT ElemTy = MVT::getIntegerVT(BitWidth: ElemWidth);
3029 MVT IntTy = tyVector(Ty: ValTy, ElemTy);
3030 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: IntTy.getVectorNumElements());
3031 unsigned ShRight = Signed ? ISD::SRA : ISD::SRL;
3032
3033 SDValue Inp = DAG.getBitcast(VT: IntTy, V: Val);
3034 SDValue LowBits = DAG.getConstant(Val: (1ull << (Amt - 1)) - 1, DL: dl, VT: IntTy);
3035
3036 SDValue AmtP1 = DAG.getConstant(Val: 1ull << Amt, DL: dl, VT: IntTy);
3037 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntTy, Ops: {Inp, AmtP1});
3038 SDValue Zero = getZero(dl, Ty: IntTy, DAG);
3039 SDValue Bit = DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: Zero, Cond: ISD::SETNE);
3040 SDValue Rup = DAG.getZExtOrTrunc(Op: Bit, DL: dl, VT: IntTy);
3041 auto [Tmp0, Ovf] = emitHvxAddWithOverflow(A: Inp, B: LowBits, dl, Signed, DAG);
3042
3043 SDValue AmtM1 = DAG.getConstant(Val: Amt - 1, DL: dl, VT: IntTy);
3044 SDValue Tmp1 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Inp, N2: AmtM1);
3045 SDValue Tmp2 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Tmp0, N2: AmtM1);
3046 SDValue Tmp3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntTy, N1: Tmp2, N2: Rup);
3047
3048 SDValue Eq = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Tmp1, RHS: Tmp2, Cond: ISD::SETEQ);
3049 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntTy);
3050 SDValue Tmp4 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp2, One});
3051 SDValue Tmp5 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp3, One});
3052 SDValue Mux = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: IntTy, Ops: {Eq, Tmp5, Tmp4});
3053 return {Mux, Ovf};
3054}
3055
3056SDValue
3057HexagonTargetLowering::emitHvxMulHsV60(SDValue A, SDValue B, const SDLoc &dl,
3058 SelectionDAG &DAG) const {
3059 MVT VecTy = ty(Op: A);
3060 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3061 assert(VecTy.getVectorElementType() == MVT::i32);
3062
3063 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
3064
3065 // mulhs(A,B) =
3066 // = [(Hi(A)*2^16 + Lo(A)) *s (Hi(B)*2^16 + Lo(B))] >> 32
3067 // = [Hi(A)*2^16 *s Hi(B)*2^16 + Hi(A) *su Lo(B)*2^16
3068 // + Lo(A) *us (Hi(B)*2^16 + Lo(B))] >> 32
3069 // = [Hi(A) *s Hi(B)*2^32 + Hi(A) *su Lo(B)*2^16 + Lo(A) *us B] >> 32
3070 // The low half of Lo(A)*Lo(B) will be discarded (it's not added to
3071 // anything, so it cannot produce any carry over to higher bits),
3072 // so everything in [] can be shifted by 16 without loss of precision.
3073 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + Lo(A)*B >> 16] >> 16
3074 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + V6_vmpyewuh(A,B)] >> 16
3075 // The final additions need to make sure to properly maintain any carry-
3076 // out bits.
3077 //
3078 // Hi(B) Lo(B)
3079 // Hi(A) Lo(A)
3080 // --------------
3081 // Lo(B)*Lo(A) | T0 = V6_vmpyewuh(B,A) does this,
3082 // Hi(B)*Lo(A) | + dropping the low 16 bits
3083 // Hi(A)*Lo(B) | T2
3084 // Hi(B)*Hi(A)
3085
3086 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh, dl, Ty: VecTy, Ops: {B, A}, DAG);
3087 // T1 = get Hi(A) into low halves.
3088 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {A, S16}, DAG);
3089 // P0 = interleaved T1.h*B.uh (full precision product)
3090 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyhus, dl, Ty: PairTy, Ops: {T1, B}, DAG);
3091 // T2 = T1.even(h) * B.even(uh), i.e. Hi(A)*Lo(B)
3092 SDValue T2 = LoHalf(V: P0, DAG);
3093 // We need to add T0+T2, recording the carry-out, which will be 1<<16
3094 // added to the final sum.
3095 // P1 = interleaved even/odd 32-bit (unsigned) sums of 16-bit halves
3096 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3097 // P2 = interleaved even/odd 32-bit (signed) sums of 16-bit halves
3098 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vaddhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3099 // T3 = full-precision(T0+T2) >> 16
3100 // The low halves are added-unsigned, the high ones are added-signed.
3101 SDValue T3 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3102 Ops: {HiHalf(V: P2, DAG), LoHalf(V: P1, DAG), S16}, DAG);
3103 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {B, S16}, DAG);
3104 // P3 = interleaved Hi(B)*Hi(A) (full precision),
3105 // which is now Lo(T1)*Lo(T4), so we want to keep the even product.
3106 SDValue P3 = getInstr(MachineOpc: Hexagon::V6_vmpyhv, dl, Ty: PairTy, Ops: {T1, T4}, DAG);
3107 SDValue T5 = LoHalf(V: P3, DAG);
3108 // Add:
3109 SDValue T6 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {T3, T5});
3110 return T6;
3111}
3112
3113SDValue
3114HexagonTargetLowering::emitHvxMulLoHiV60(SDValue A, bool SignedA, SDValue B,
3115 bool SignedB, const SDLoc &dl,
3116 SelectionDAG &DAG) const {
3117 MVT VecTy = ty(Op: A);
3118 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3119 assert(VecTy.getVectorElementType() == MVT::i32);
3120
3121 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
3122
3123 if (SignedA && !SignedB) {
3124 // Make A:unsigned, B:signed.
3125 std::swap(a&: A, b&: B);
3126 std::swap(a&: SignedA, b&: SignedB);
3127 }
3128
3129 // Do halfword-wise multiplications for unsigned*unsigned product, then
3130 // add corrections for signed and unsigned*signed.
3131
3132 SDValue Lo, Hi;
3133
3134 // P0:lo = (uu) products of low halves of A and B,
3135 // P0:hi = (uu) products of high halves.
3136 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, B}, DAG);
3137
3138 // Swap low/high halves in B
3139 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_lvsplatw, dl, Ty: VecTy,
3140 Ops: {DAG.getConstant(Val: 0x02020202, DL: dl, VT: MVT::i32)}, DAG);
3141 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vdelta, dl, Ty: VecTy, Ops: {B, T0}, DAG);
3142 // P1 = products of even/odd halfwords.
3143 // P1:lo = (uu) products of even(A.uh) * odd(B.uh)
3144 // P1:hi = (uu) products of odd(A.uh) * even(B.uh)
3145 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, T1}, DAG);
3146
3147 // P2:lo = low halves of P1:lo + P1:hi,
3148 // P2:hi = high halves of P1:lo + P1:hi.
3149 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy,
3150 Ops: {HiHalf(V: P1, DAG), LoHalf(V: P1, DAG)}, DAG);
3151 // Still need to add the high halves of P0:lo to P2:lo
3152 SDValue T2 =
3153 getInstr(MachineOpc: Hexagon::V6_vlsrw, dl, Ty: VecTy, Ops: {LoHalf(V: P0, DAG), S16}, DAG);
3154 SDValue T3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {LoHalf(V: P2, DAG), T2});
3155
3156 // The high halves of T3 will contribute to the HI part of LOHI.
3157 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3158 Ops: {HiHalf(V: P2, DAG), T3, S16}, DAG);
3159
3160 // The low halves of P2 need to be added to high halves of the LO part.
3161 Lo = getInstr(MachineOpc: Hexagon::V6_vaslw_acc, dl, Ty: VecTy,
3162 Ops: {LoHalf(V: P0, DAG), LoHalf(V: P2, DAG), S16}, DAG);
3163 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {HiHalf(V: P0, DAG), T4});
3164
3165 if (SignedA) {
3166 assert(SignedB && "Signed A and unsigned B should have been inverted");
3167
3168 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3169 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3170 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3171 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3172 SDValue X0 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: VecTy, Ops: {Q0, B, Zero});
3173 SDValue X1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, X0, A}, DAG);
3174 Hi = getInstr(MachineOpc: Hexagon::V6_vsubw, dl, Ty: VecTy, Ops: {Hi, X1}, DAG);
3175 } else if (SignedB) {
3176 // Same correction as for mulhus:
3177 // mulhus(A.uw,B.w) = mulhu(A.uw,B.uw) - (A.w if B < 0)
3178 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3179 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3180 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3181 Hi = getInstr(MachineOpc: Hexagon::V6_vsubwq, dl, Ty: VecTy, Ops: {Q1, Hi, A}, DAG);
3182 } else {
3183 assert(!SignedA && !SignedB);
3184 }
3185
3186 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3187}
3188
3189SDValue
3190HexagonTargetLowering::emitHvxMulLoHiV62(SDValue A, bool SignedA,
3191 SDValue B, bool SignedB,
3192 const SDLoc &dl,
3193 SelectionDAG &DAG) const {
3194 MVT VecTy = ty(Op: A);
3195 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3196 assert(VecTy.getVectorElementType() == MVT::i32);
3197
3198 if (SignedA && !SignedB) {
3199 // Make A:unsigned, B:signed.
3200 std::swap(a&: A, b&: B);
3201 std::swap(a&: SignedA, b&: SignedB);
3202 }
3203
3204 // Do S*S first, then make corrections for U*S or U*U if needed.
3205 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh_64, dl, Ty: PairTy, Ops: {A, B}, DAG);
3206 SDValue P1 =
3207 getInstr(MachineOpc: Hexagon::V6_vmpyowh_64_acc, dl, Ty: PairTy, Ops: {P0, A, B}, DAG);
3208 SDValue Lo = LoHalf(V: P1, DAG);
3209 SDValue Hi = HiHalf(V: P1, DAG);
3210
3211 if (!SignedB) {
3212 assert(!SignedA && "Signed A and unsigned B should have been inverted");
3213 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3214 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3215
3216 // Mulhu(X, Y) = Mulhs(X, Y) + (X, if Y < 0) + (Y, if X < 0).
3217 // def: Pat<(VecI32 (mulhu HVI32:$A, HVI32:$B)),
3218 // (V6_vaddw (HiHalf (Muls64O $A, $B)),
3219 // (V6_vaddwq (V6_vgtw (V6_vd0), $B),
3220 // (V6_vandvqv (V6_vgtw (V6_vd0), $A), $B),
3221 // $A))>;
3222 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3223 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3224 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vandvqv, dl, Ty: VecTy, Ops: {Q0, B}, DAG);
3225 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, T0, A}, DAG);
3226 Hi = getInstr(MachineOpc: Hexagon::V6_vaddw, dl, Ty: VecTy, Ops: {Hi, T1}, DAG);
3227 } else if (!SignedA) {
3228 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3229 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3230
3231 // Mulhus(unsigned X, signed Y) = Mulhs(X, Y) + (Y, if X < 0).
3232 // def: Pat<(VecI32 (HexagonMULHUS HVI32:$A, HVI32:$B)),
3233 // (V6_vaddwq (V6_vgtw (V6_vd0), $A),
3234 // (HiHalf (Muls64O $A, $B)),
3235 // $B)>;
3236 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3237 Hi = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q0, Hi, B}, DAG);
3238 }
3239
3240 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3241}
3242
3243SDValue
3244HexagonTargetLowering::EqualizeFpIntConversion(SDValue Op, SelectionDAG &DAG)
3245 const {
3246 // Rewrite conversion between integer and floating-point in such a way that
3247 // the integer type is extended/narrowed to match the bitwidth of the
3248 // floating-point type, combined with additional integer-integer extensions
3249 // or narrowings to match the original input/result types.
3250 // E.g. f32 -> i8 ==> f32 -> i32 -> i8
3251 //
3252 // The input/result types are not required to be legal, but if they are
3253 // legal, this function should not introduce illegal types.
3254
3255 unsigned Opc = Op.getOpcode();
3256 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT ||
3257 Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3258
3259 SDValue Inp = Op.getOperand(i: 0);
3260 MVT InpTy = ty(Op: Inp);
3261 MVT ResTy = ty(Op);
3262
3263 if (InpTy == ResTy)
3264 return Op;
3265
3266 const SDLoc &dl(Op);
3267 bool Signed = Opc == ISD::FP_TO_SINT || Opc == ISD::SINT_TO_FP;
3268
3269 auto [WInpTy, WResTy] = typeExtendToWider(Ty0: InpTy, Ty1: ResTy);
3270 SDValue WInp = resizeToWidth(VecV: Inp, ResTy: WInpTy, Signed, dl, DAG);
3271 SDValue Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: WResTy, Operand: WInp);
3272 SDValue Res = resizeToWidth(VecV: Conv, ResTy, Signed, dl, DAG);
3273 return Res;
3274}
3275
3276SDValue
3277HexagonTargetLowering::ExpandHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
3278 unsigned Opc = Op.getOpcode();
3279 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT);
3280
3281 const SDLoc &dl(Op);
3282 SDValue Op0 = Op.getOperand(i: 0);
3283 MVT InpTy = ty(Op: Op0);
3284 MVT ResTy = ty(Op);
3285 assert(InpTy.changeTypeToInteger() == ResTy);
3286
3287 // At this point this is an experiment under a flag.
3288 // In arch before V81 the rounding mode is towards nearest value.
3289 // The C/C++ standard requires rounding towards zero:
3290 // C (C99 and later): ISO/IEC 9899:2018 (C18), section 6.3.1.4 — "When a
3291 // finite value of real floating type is converted to an integer type, the
3292 // fractional part is discarded (i.e., the value is truncated toward zero)."
3293 // C++: ISO/IEC 14882:2020 (C++20), section 7.3.7 — "A prvalue of a
3294 // floating-point type can be converted to a prvalue of an integer type. The
3295 // conversion truncates; that is, the fractional part is discarded."
3296 if (InpTy == MVT::v64f16) {
3297 if (Subtarget.useHVXV81Ops()) {
3298 // This is c/c++ compliant
3299 SDValue ConvVec =
3300 getInstr(MachineOpc: Hexagon::V6_vconv_h_hf_rnd, dl, Ty: ResTy, Ops: {Op0}, DAG);
3301 return ConvVec;
3302 } else if (EnableFpFastConvert) {
3303 // Vd32.h=Vu32.hf same as Q6_Vh_equals_Vhf
3304 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_h_hf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3305 return ConvVec;
3306 }
3307 } else if (EnableFpFastConvert && InpTy == MVT::v32f32) {
3308 // Vd32.w=Vu32.sf same as Q6_Vw_equals_Vsf
3309 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_w_sf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3310 return ConvVec;
3311 }
3312
3313 // int32_t conv_f32_to_i32(uint32_t inp) {
3314 // // s | exp8 | frac23
3315 //
3316 // int neg = (int32_t)inp < 0;
3317 //
3318 // // "expm1" is the actual exponent minus 1: instead of "bias", subtract
3319 // // "bias+1". When the encoded exp is "all-1" (i.e. inf/nan), this will
3320 // // produce a large positive "expm1", which will result in max u/int.
3321 // // In all IEEE formats, bias is the largest positive number that can be
3322 // // represented in bias-width bits (i.e. 011..1).
3323 // int32_t expm1 = (inp << 1) - 0x80000000;
3324 // expm1 >>= 24;
3325 //
3326 // // Always insert the "implicit 1". Subnormal numbers will become 0
3327 // // regardless.
3328 // uint32_t frac = (inp << 8) | 0x80000000;
3329 //
3330 // // "frac" is the fraction part represented as Q1.31. If it was
3331 // // interpreted as uint32_t, it would be the fraction part multiplied
3332 // // by 2^31.
3333 //
3334 // // Calculate the amount of right shift, since shifting further to the
3335 // // left would lose significant bits. Limit it to 32, because we want
3336 // // shifts by 32+ to produce 0, whereas V6_vlsrwv treats the shift
3337 // // amount as a 6-bit signed value (so 33 is same as -31, i.e. shift
3338 // // left by 31). "rsh" can be negative.
3339 // int32_t rsh = min(31 - (expm1 + 1), 32);
3340 //
3341 // frac >>= rsh; // rsh == 32 will produce 0
3342 //
3343 // // Everything up to this point is the same for conversion to signed
3344 // // unsigned integer.
3345 //
3346 // if (neg) // Only for signed int
3347 // frac = -frac; //
3348 // if (rsh <= 0 && neg) // bound = neg ? 0x80000000 : 0x7fffffff
3349 // frac = 0x80000000; // frac = rsh <= 0 ? bound : frac
3350 // if (rsh <= 0 && !neg) //
3351 // frac = 0x7fffffff; //
3352 //
3353 // if (neg) // Only for unsigned int
3354 // frac = 0; //
3355 // if (rsh < 0 && !neg) // frac = rsh < 0 ? 0x7fffffff : frac;
3356 // frac = 0x7fffffff; // frac = neg ? 0 : frac;
3357 //
3358 // return frac;
3359 // }
3360
3361 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: ResTy.getVectorElementCount());
3362
3363 // Zero = V6_vd0();
3364 // Neg = V6_vgtw(Zero, Inp);
3365 // One = V6_lvsplatw(1);
3366 // M80 = V6_lvsplatw(0x80000000);
3367 // Exp00 = V6_vaslwv(Inp, One);
3368 // Exp01 = V6_vsubw(Exp00, M80);
3369 // ExpM1 = V6_vasrw(Exp01, 24);
3370 // Frc00 = V6_vaslw(Inp, 8);
3371 // Frc01 = V6_vor(Frc00, M80);
3372 // Rsh00 = V6_vsubw(V6_lvsplatw(30), ExpM1);
3373 // Rsh01 = V6_vminw(Rsh00, V6_lvsplatw(32));
3374 // Frc02 = V6_vlsrwv(Frc01, Rsh01);
3375
3376 // if signed int:
3377 // Bnd = V6_vmux(Neg, M80, V6_lvsplatw(0x7fffffff))
3378 // Pos = V6_vgtw(Rsh01, Zero);
3379 // Frc13 = V6_vsubw(Zero, Frc02);
3380 // Frc14 = V6_vmux(Neg, Frc13, Frc02);
3381 // Int = V6_vmux(Pos, Frc14, Bnd);
3382 //
3383 // if unsigned int:
3384 // Rsn = V6_vgtw(Zero, Rsh01)
3385 // Frc23 = V6_vmux(Rsn, V6_lvsplatw(0x7fffffff), Frc02)
3386 // Int = V6_vmux(Neg, Zero, Frc23)
3387
3388 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: InpTy);
3389 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3390 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias));
3391
3392 SDValue Inp = DAG.getBitcast(VT: ResTy, V: Op0);
3393 SDValue Zero = getZero(dl, Ty: ResTy, DAG);
3394 SDValue Neg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Inp, RHS: Zero, Cond: ISD::SETLT);
3395 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: ResTy);
3396 SDValue M7F = DAG.getConstant(Val: (1ull << (ElemWidth - 1)) - 1, DL: dl, VT: ResTy);
3397 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResTy);
3398 SDValue Exp00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, One});
3399 SDValue Exp01 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Exp00, M80});
3400 SDValue MNE = DAG.getConstant(Val: ElemWidth - ExpWidth, DL: dl, VT: ResTy);
3401 SDValue ExpM1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: ResTy, Ops: {Exp01, MNE});
3402
3403 SDValue ExpW = DAG.getConstant(Val: ExpWidth, DL: dl, VT: ResTy);
3404 SDValue Frc00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, ExpW});
3405 SDValue Frc01 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ResTy, Ops: {Frc00, M80});
3406
3407 SDValue MN2 = DAG.getConstant(Val: ElemWidth - 2, DL: dl, VT: ResTy);
3408 SDValue Rsh00 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {MN2, ExpM1});
3409 SDValue MW = DAG.getConstant(Val: ElemWidth, DL: dl, VT: ResTy);
3410 SDValue Rsh01 = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ResTy, Ops: {Rsh00, MW});
3411 SDValue Frc02 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ResTy, Ops: {Frc01, Rsh01});
3412
3413 SDValue Int;
3414
3415 if (Opc == ISD::FP_TO_SINT) {
3416 SDValue Bnd = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, M80, M7F});
3417 SDValue Pos = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETGT);
3418 SDValue Frc13 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Zero, Frc02});
3419 SDValue Frc14 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, Frc13, Frc02});
3420 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Pos, Frc14, Bnd});
3421 } else {
3422 assert(Opc == ISD::FP_TO_UINT);
3423 SDValue Rsn = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETLT);
3424 SDValue Frc23 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Rsn, N2: M7F, N3: Frc02);
3425 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Neg, N2: Zero, N3: Frc23);
3426 }
3427
3428 return Int;
3429}
3430
3431SDValue
3432HexagonTargetLowering::ExpandHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
3433 unsigned Opc = Op.getOpcode();
3434 assert(Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3435
3436 const SDLoc &dl(Op);
3437 SDValue Op0 = Op.getOperand(i: 0);
3438 MVT InpTy = ty(Op: Op0);
3439 MVT ResTy = ty(Op);
3440 assert(ResTy.changeTypeToInteger() == InpTy);
3441
3442 // uint32_t vnoc1_rnd(int32_t w) {
3443 // int32_t iszero = w == 0;
3444 // int32_t isneg = w < 0;
3445 // uint32_t u = __builtin_HEXAGON_A2_abs(w);
3446 //
3447 // uint32_t norm_left = __builtin_HEXAGON_S2_cl0(u) + 1;
3448 // uint32_t frac0 = (uint64_t)u << norm_left;
3449 //
3450 // // Rounding:
3451 // uint32_t frac1 = frac0 + ((1 << 8) - 1);
3452 // uint32_t renorm = (frac0 > frac1);
3453 // uint32_t rup = (int)(frac0 << 22) < 0;
3454 //
3455 // uint32_t frac2 = frac0 >> 8;
3456 // uint32_t frac3 = frac1 >> 8;
3457 // uint32_t frac = (frac2 != frac3) ? frac3 >> 1 : (frac3 + rup) >> 1;
3458 //
3459 // int32_t exp = 32 - norm_left + renorm + 127;
3460 // exp <<= 23;
3461 //
3462 // uint32_t sign = 0x80000000 * isneg;
3463 // uint32_t f = sign | exp | frac;
3464 // return iszero ? 0 : f;
3465 // }
3466
3467 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: InpTy.getVectorElementCount());
3468 bool Signed = Opc == ISD::SINT_TO_FP;
3469
3470 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: ResTy);
3471 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3472
3473 SDValue Zero = getZero(dl, Ty: InpTy, DAG);
3474 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: InpTy);
3475 SDValue IsZero = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ);
3476 SDValue Abs = Signed ? DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: InpTy, Operand: Op0) : Op0;
3477 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: InpTy, Operand: Abs);
3478 SDValue NLeft = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Clz, One});
3479 SDValue Frac0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy, Ops: {Abs, NLeft});
3480
3481 auto [Frac, Ovf] = emitHvxShiftRightRnd(Val: Frac0, Amt: ExpWidth + 1, Signed: false, DAG);
3482 if (Signed) {
3483 SDValue IsNeg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETLT);
3484 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: InpTy);
3485 SDValue Sign = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsNeg, M80, Zero});
3486 Frac = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Sign, Frac});
3487 }
3488
3489 SDValue Rnrm = DAG.getZExtOrTrunc(Op: Ovf, DL: dl, VT: InpTy);
3490 SDValue Exp0 = DAG.getConstant(Val: ElemWidth + ExpBias, DL: dl, VT: InpTy);
3491 SDValue Exp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Rnrm, Exp0});
3492 SDValue Exp2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: InpTy, Ops: {Exp1, NLeft});
3493 SDValue Exp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy,
3494 Ops: {Exp2, DAG.getConstant(Val: FracWidth, DL: dl, VT: InpTy)});
3495 SDValue Flt0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Frac, Exp3});
3496 SDValue Flt1 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsZero, Zero, Flt0});
3497 SDValue Flt = DAG.getBitcast(VT: ResTy, V: Flt1);
3498
3499 return Flt;
3500}
3501
3502SDValue
3503HexagonTargetLowering::CreateTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3504 unsigned Opc = Op.getOpcode();
3505 unsigned TLOpc;
3506 switch (Opc) {
3507 case ISD::ANY_EXTEND:
3508 case ISD::SIGN_EXTEND:
3509 case ISD::ZERO_EXTEND:
3510 TLOpc = HexagonISD::TL_EXTEND;
3511 break;
3512 case ISD::TRUNCATE:
3513 TLOpc = HexagonISD::TL_TRUNCATE;
3514 break;
3515#ifndef NDEBUG
3516 Op.dump(&DAG);
3517#endif
3518 llvm_unreachable("Unexpected operator");
3519 }
3520
3521 const SDLoc &dl(Op);
3522 return DAG.getNode(Opcode: TLOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0),
3523 N2: DAG.getUNDEF(VT: MVT::i128), // illegal type
3524 N3: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
3525}
3526
3527SDValue
3528HexagonTargetLowering::RemoveTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3529 assert(Op.getOpcode() == HexagonISD::TL_EXTEND ||
3530 Op.getOpcode() == HexagonISD::TL_TRUNCATE);
3531 unsigned Opc = Op.getConstantOperandVal(i: 2);
3532 return DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: ty(Op), Operand: Op.getOperand(i: 0));
3533}
3534
3535HexagonTargetLowering::VectorPair
3536HexagonTargetLowering::SplitVectorOp(SDValue Op, SelectionDAG &DAG) const {
3537 assert(!Op.isMachineOpcode());
3538 SmallVector<SDValue, 2> OpsL, OpsH;
3539 const SDLoc &dl(Op);
3540
3541 auto SplitVTNode = [&DAG, this](const VTSDNode *N) {
3542 MVT Ty = typeSplit(VecTy: N->getVT().getSimpleVT()).first;
3543 SDValue TV = DAG.getValueType(Ty);
3544 return std::make_pair(x&: TV, y&: TV);
3545 };
3546
3547 for (SDValue A : Op.getNode()->ops()) {
3548 auto [Lo, Hi] =
3549 ty(Op: A).isVector() ? opSplit(Vec: A, dl, DAG) : std::make_pair(x&: A, y&: A);
3550 // Special case for type operand.
3551 switch (Op.getOpcode()) {
3552 case ISD::SIGN_EXTEND_INREG:
3553 case HexagonISD::SSAT:
3554 case HexagonISD::USAT:
3555 if (const auto *N = dyn_cast<const VTSDNode>(Val: A.getNode()))
3556 std::tie(args&: Lo, args&: Hi) = SplitVTNode(N);
3557 break;
3558 }
3559 OpsL.push_back(Elt: Lo);
3560 OpsH.push_back(Elt: Hi);
3561 }
3562
3563 MVT ResTy = ty(Op);
3564 MVT HalfTy = typeSplit(VecTy: ResTy).first;
3565 SDValue L = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsL);
3566 SDValue H = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsH);
3567 return {L, H};
3568}
3569
3570SDValue
3571HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
3572 auto *MemN = cast<MemSDNode>(Val: Op.getNode());
3573 unsigned MemOpc = MemN->getOpcode();
3574 EVT MemTy = MemN->getMemoryVT();
3575
3576 if ((MemOpc == ISD::STORE || MemOpc == ISD::LOAD) &&
3577 (!MemTy.isSimple() || !isHvxPairTy(Ty: MemTy.getSimpleVT())))
3578 return Op;
3579
3580 EVT ValueType;
3581 if (MemOpc == ISD::STORE)
3582 ValueType = ty(Op: cast<StoreSDNode>(Val&: Op)->getValue());
3583 else if (MemOpc == ISD::MSTORE)
3584 ValueType = ty(Op: cast<MaskedStoreSDNode>(Val&: Op)->getValue());
3585 else // ISD::LOAD, ISD::MLOAD.
3586 ValueType = MemN->getValueType(ResNo: 0);
3587
3588 EVT LoVT, HiVT;
3589 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: ValueType);
3590
3591 EVT LoMemVT, HiMemVT;
3592 bool HiIsEmpty = false;
3593 std::tie(args&: LoMemVT, args&: HiMemVT) =
3594 DAG.GetDependentSplitDestVTs(VT: MemTy, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
3595
3596 uint64_t LoSize = LoMemVT.getSizeInBits().getFixedValue() / 8;
3597 uint64_t HiSize = HiMemVT.getSizeInBits().getFixedValue() / 8;
3598
3599 const SDLoc &dl(Op);
3600 SDValue Chain = MemN->getChain();
3601 SDValue Base0 = MemN->getBasePtr();
3602 SDValue Base1 =
3603 DAG.getMemBasePlusOffset(Base: Base0, Offset: TypeSize::getFixed(ExactSize: LoSize), DL: dl);
3604
3605 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
3606 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
3607 MachineFunction &MF = DAG.getMachineFunction();
3608 auto MemSize = [=](uint64_t Size) {
3609 return (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
3610 ? (uint64_t)MemoryLocation::UnknownSize
3611 : Size;
3612 };
3613 // MOp1 will not be used if HiIsEmpty for masked loads and stores (MLOAD and
3614 // MSTORE). Non-masked loads and store are always of double-vector size (see
3615 // isHvxPairTy() check above).
3616 MOp0 = MF.getMachineMemOperand(MMO, Offset: 0, Size: MemSize(LoSize));
3617 MOp1 = MF.getMachineMemOperand(MMO, Offset: LoSize, Size: MemSize(HiSize));
3618 }
3619
3620 if (MemOpc == ISD::LOAD) {
3621 assert(cast<LoadSDNode>(Op)->isUnindexed());
3622 SDValue Load0 = DAG.getLoad(VT: LoVT, dl, Chain, Ptr: Base0, MMO: MOp0);
3623 SDValue Load1 = DAG.getLoad(VT: HiVT, dl, Chain, Ptr: Base1, MMO: MOp1);
3624 return DAG.getMergeValues(
3625 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: Load0,
3626 N2: Load1),
3627 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Load0.getValue(R: 1),
3628 N2: Load1.getValue(R: 1))},
3629 dl);
3630 }
3631 if (MemOpc == ISD::STORE) {
3632 assert(cast<StoreSDNode>(Op)->isUnindexed());
3633 VectorPair Vals = opSplit(Vec: cast<StoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3634 SDValue Store0 = DAG.getStore(Chain, dl, Val: Vals.first, Ptr: Base0, MMO: MOp0);
3635 SDValue Store1 = DAG.getStore(Chain, dl, Val: Vals.second, Ptr: Base1, MMO: MOp1);
3636 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store0, N2: Store1);
3637 }
3638
3639 assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE);
3640
3641 auto MaskN = cast<MaskedLoadStoreSDNode>(Val&: Op);
3642 assert(MaskN->isUnindexed());
3643 VectorPair Masks = opSplit(Vec: MaskN->getMask(), dl, DAG);
3644 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3645
3646 if (MemOpc == ISD::MLOAD) {
3647 VectorPair Thru =
3648 opSplit(Vec: cast<MaskedLoadSDNode>(Val&: Op)->getPassThru(), dl, DAG);
3649 SDValue MLoad0 = DAG.getMaskedLoad(VT: LoVT, dl, Chain, Base: Base0, Offset,
3650 Mask: Masks.first, Src0: Thru.first, MemVT: LoMemVT, MMO: MOp0,
3651 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3652
3653 // The hi masked load has zero storage size. We therefore simply set it to
3654 // the low masked load and rely on subsequent removal from the chain as it
3655 // is unused. See DAGTypeLegalizer::SplitVecRes_MLOAD() for the same logic.
3656 SDValue MLoad1 =
3657 HiIsEmpty ? MLoad0
3658 : DAG.getMaskedLoad(VT: HiVT, dl, Chain, Base: Base1, Offset,
3659 Mask: Masks.second, Src0: Thru.second, MemVT: HiMemVT, MMO: MOp1,
3660 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3661 return DAG.getMergeValues(
3662 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: MLoad0,
3663 N2: MLoad1),
3664 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MLoad0.getValue(R: 1),
3665 N2: MLoad1.getValue(R: 1))},
3666 dl);
3667 }
3668 if (MemOpc == ISD::MSTORE) {
3669 VectorPair Vals = opSplit(Vec: cast<MaskedStoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3670 SDValue MStore0 =
3671 DAG.getMaskedStore(Chain, dl, Val: Vals.first, Base: Base0, Offset, Mask: Masks.first,
3672 MemVT: LoMemVT, MMO: MOp0, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3673 if (HiIsEmpty)
3674 return MStore0;
3675 SDValue MStore1 =
3676 DAG.getMaskedStore(Chain, dl, Val: Vals.second, Base: Base1, Offset, Mask: Masks.second,
3677 MemVT: HiMemVT, MMO: MOp1, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3678 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MStore0, N2: MStore1);
3679 }
3680
3681 std::string Name = "Unexpected operation: " + Op->getOperationName(G: &DAG);
3682 llvm_unreachable(Name.c_str());
3683}
3684
3685SDValue
3686HexagonTargetLowering::WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const {
3687 const SDLoc &dl(Op);
3688 auto *LoadN = cast<LoadSDNode>(Val: Op.getNode());
3689 assert(LoadN->isUnindexed() && "Not widening indexed loads yet");
3690 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3691 "Not widening loads of i1 yet");
3692
3693 SDValue Chain = LoadN->getChain();
3694 SDValue Base = LoadN->getBasePtr();
3695 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3696
3697 MVT ResTy = ty(Op);
3698 unsigned HwLen = Subtarget.getVectorLength();
3699 unsigned ResLen = ResTy.getStoreSize();
3700 assert(ResLen < HwLen && "vsetq(v1) prerequisite");
3701
3702 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3703 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3704 Ops: {DAG.getConstant(Val: ResLen, DL: dl, VT: MVT::i32)}, DAG);
3705
3706 MVT LoadTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
3707 MachineFunction &MF = DAG.getMachineFunction();
3708 auto *MemOp = MF.getMachineMemOperand(MMO: LoadN->getMemOperand(), Offset: 0, Size: HwLen);
3709
3710 SDValue Load = DAG.getMaskedLoad(VT: LoadTy, dl, Chain, Base, Offset, Mask,
3711 Src0: DAG.getUNDEF(VT: LoadTy), MemVT: LoadTy, MMO: MemOp,
3712 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3713 SDValue Value = opCastElem(Vec: Load, ElemTy: ResTy.getVectorElementType(), DAG);
3714 return DAG.getMergeValues(Ops: {Value, Load.getValue(R: 1)}, dl);
3715}
3716
3717SDValue
3718HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const {
3719 const SDLoc &dl(Op);
3720 auto *StoreN = cast<StoreSDNode>(Val: Op.getNode());
3721 assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
3722 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3723 "Not widening stores of i1 yet");
3724
3725 SDValue Chain = StoreN->getChain();
3726 SDValue Base = StoreN->getBasePtr();
3727 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3728
3729 SDValue Value = opCastElem(Vec: StoreN->getValue(), ElemTy: MVT::i8, DAG);
3730 MVT ValueTy = ty(Op: Value);
3731 unsigned ValueLen = ValueTy.getVectorNumElements();
3732 unsigned HwLen = Subtarget.getVectorLength();
3733 assert(isPowerOf2_32(ValueLen));
3734
3735 for (unsigned Len = ValueLen; Len < HwLen; ) {
3736 Value = opJoin(Ops: {Value, DAG.getUNDEF(VT: ty(Op: Value))}, dl, DAG);
3737 Len = ty(Op: Value).getVectorNumElements(); // This is Len *= 2
3738 }
3739 assert(ty(Value).getVectorNumElements() == HwLen); // Paranoia
3740
3741 assert(ValueLen < HwLen && "vsetq(v1) prerequisite");
3742 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3743 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3744 Ops: {DAG.getConstant(Val: ValueLen, DL: dl, VT: MVT::i32)}, DAG);
3745 MachineFunction &MF = DAG.getMachineFunction();
3746 auto *MemOp = MF.getMachineMemOperand(MMO: StoreN->getMemOperand(), Offset: 0, Size: HwLen);
3747 return DAG.getMaskedStore(Chain, dl, Val: Value, Base, Offset, Mask,
3748 MemVT: StoreN->getMemoryVT(), MMO: MemOp, AM: ISD::UNINDEXED, IsTruncating: false,
3749 IsCompressing: false);
3750}
3751
3752SDValue
3753HexagonTargetLowering::WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const {
3754 const SDLoc &dl(Op);
3755 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
3756 MVT ElemTy = ty(Op: Op0).getVectorElementType();
3757 unsigned HwLen = Subtarget.getVectorLength();
3758
3759 unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits();
3760 assert(WideOpLen * ElemTy.getSizeInBits() == 8 * HwLen);
3761 MVT WideOpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideOpLen);
3762 if (!Subtarget.isHVXVectorType(VecTy: WideOpTy, IncludeBool: true))
3763 return SDValue();
3764
3765 SDValue WideOp0 = appendUndef(Val: Op0, ResTy: WideOpTy, DAG);
3766 SDValue WideOp1 = appendUndef(Val: Op1, ResTy: WideOpTy, DAG);
3767 EVT ResTy =
3768 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: WideOpTy);
3769 SDValue SetCC = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: ResTy,
3770 Ops: {WideOp0, WideOp1, Op.getOperand(i: 2)});
3771
3772 EVT RetTy = typeLegalize(Ty: ty(Op), DAG);
3773 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3774 Ops: {SetCC, getZero(dl, Ty: MVT::i32, DAG)});
3775}
3776
3777SDValue HexagonTargetLowering::WidenHvxTruncateToBool(SDValue Op,
3778 SelectionDAG &DAG) const {
3779 // Handle truncation to boolean vector where the result boolean type
3780 // needs widening (e.g., v16i32 -> v16i1 where v16i1 is not a standard
3781 // HVX predicate type, or v16i8 -> v16i1 in 128-byte mode).
3782 // Widen the input to HVX width, perform the truncate to the widened
3783 // boolean type, then extract the result.
3784 const SDLoc &dl(Op);
3785 SDValue Inp = Op.getOperand(i: 0);
3786 MVT InpTy = ty(Op: Inp);
3787 MVT ResTy = ty(Op);
3788
3789 assert(ResTy.getVectorElementType() == MVT::i1 &&
3790 "Expected boolean result type");
3791
3792 MVT ElemTy = InpTy.getVectorElementType();
3793 unsigned HwLen = Subtarget.getVectorLength();
3794
3795 // Calculate the widened input type that fills the HVX register.
3796 unsigned WideLen = (8 * HwLen) / ElemTy.getSizeInBits();
3797 MVT WideInpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideLen);
3798 if (!Subtarget.isHVXVectorType(VecTy: WideInpTy, IncludeBool: false))
3799 return SDValue();
3800
3801 // Widen the input to HVX width.
3802 SDValue WideInp = appendUndef(Val: Inp, ResTy: WideInpTy, DAG);
3803
3804 // Perform the truncate to widened boolean type.
3805 MVT WideBoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: WideLen);
3806 SDValue WideTrunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: WideBoolTy, Operand: WideInp);
3807
3808 // Extract the result.
3809 EVT RetTy = typeLegalize(Ty: ResTy, DAG);
3810 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3811 Ops: {WideTrunc, getZero(dl, Ty: MVT::i32, DAG)});
3812}
3813
3814SDValue
3815HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
3816 unsigned Opc = Op.getOpcode();
3817 bool IsPairOp = isHvxPairTy(Ty: ty(Op)) ||
3818 llvm::any_of(Range: Op.getNode()->ops(), P: [this] (SDValue V) {
3819 return isHvxPairTy(Ty: ty(Op: V));
3820 });
3821
3822 if (IsPairOp) {
3823 switch (Opc) {
3824 default:
3825 break;
3826 case ISD::LOAD:
3827 case ISD::STORE:
3828 case ISD::MLOAD:
3829 case ISD::MSTORE:
3830 return SplitHvxMemOp(Op, DAG);
3831 case ISD::SINT_TO_FP:
3832 case ISD::UINT_TO_FP:
3833 case ISD::FP_TO_SINT:
3834 case ISD::FP_TO_UINT:
3835 if (ty(Op).getSizeInBits() == ty(Op: Op.getOperand(i: 0)).getSizeInBits())
3836 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3837 break;
3838 case ISD::ABS:
3839 case ISD::CTPOP:
3840 case ISD::CTLZ:
3841 case ISD::CTTZ:
3842 case ISD::MUL:
3843 case ISD::FADD:
3844 case ISD::FSUB:
3845 case ISD::FMUL:
3846 case ISD::FMINIMUMNUM:
3847 case ISD::FMAXIMUMNUM:
3848 case ISD::FMINIMUM:
3849 case ISD::FMAXIMUM:
3850 case ISD::FMINNUM:
3851 case ISD::FMAXNUM:
3852 case ISD::MULHS:
3853 case ISD::MULHU:
3854 case ISD::AND:
3855 case ISD::OR:
3856 case ISD::XOR:
3857 case ISD::SRA:
3858 case ISD::SHL:
3859 case ISD::SRL:
3860 case ISD::FSHL:
3861 case ISD::FSHR:
3862 case ISD::SMIN:
3863 case ISD::SMAX:
3864 case ISD::UMIN:
3865 case ISD::UMAX:
3866 case ISD::SETCC:
3867 case ISD::VSELECT:
3868 case ISD::SIGN_EXTEND_INREG:
3869 case ISD::SPLAT_VECTOR:
3870 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3871 case ISD::SIGN_EXTEND:
3872 case ISD::ZERO_EXTEND:
3873 // In general, sign- and zero-extends can't be split and still
3874 // be legal. The only exception is extending bool vectors.
3875 if (ty(Op: Op.getOperand(i: 0)).getVectorElementType() == MVT::i1)
3876 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3877 break;
3878 }
3879 }
3880
3881 switch (Opc) {
3882 default:
3883 break;
3884 // clang-format off
3885 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
3886 case ISD::SPLAT_VECTOR: return LowerHvxSplatVector(Op, DAG);
3887 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
3888 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
3889 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
3890 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
3891 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
3892 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
3893 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
3894 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
3895 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
3896 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
3897 case ISD::SELECT: return LowerHvxSelect(Op, DAG);
3898 case ISD::SRA:
3899 case ISD::SHL:
3900 case ISD::SRL: return LowerHvxShift(Op, DAG);
3901 case ISD::FSHL:
3902 case ISD::FSHR: return LowerHvxFunnelShift(Op, DAG);
3903 case ISD::MULHS:
3904 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
3905 case ISD::SMUL_LOHI:
3906 case ISD::UMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3907 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
3908 case ISD::SETCC: {
3909 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
3910 if (CC == ISD::SETOEQ &&
3911 ty(Op: Op.getOperand(i: 0)).getScalarType().isFloatingPoint())
3912 return LowerHvxFpSetoeq(Op, DAG);
3913 return Op;
3914 }
3915 case ISD::INTRINSIC_VOID: return Op;
3916 case ISD::INTRINSIC_W_CHAIN: return Op;
3917 case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG);
3918 case ISD::MLOAD:
3919 case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG);
3920 // Unaligned loads will be handled by the default lowering.
3921 case ISD::LOAD: return LowerHvxLoad(Op, DAG);
3922 case ISD::STORE: return LowerHvxStore(Op, DAG);
3923 case ISD::FP_EXTEND: return LowerHvxFpExtend(Op, DAG);
3924 case ISD::FP_TO_SINT:
3925 case ISD::FP_TO_UINT: return LowerHvxFpToInt(Op, DAG);
3926 case ISD::SINT_TO_FP:
3927 case ISD::UINT_TO_FP: return LowerHvxIntToFp(Op, DAG);
3928
3929 // Special nodes:
3930 case HexagonISD::SMUL_LOHI:
3931 case HexagonISD::UMUL_LOHI:
3932 case HexagonISD::USMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3933
3934 case ISD::PARTIAL_REDUCE_SMLA:
3935 case ISD::PARTIAL_REDUCE_UMLA:
3936 case ISD::PARTIAL_REDUCE_SUMLA:
3937 return LowerHvxPartialReduceMLA(Op, DAG);
3938 case ISD::VECREDUCE_FMIN:
3939 return LowerHvxVecReduceFMin(Op, DAG);
3940 case ISD::VECREDUCE_FMAX:
3941 return LowerHvxVecReduceFMax(Op, DAG);
3942 case ISD::VECREDUCE_FMINIMUM:
3943 return LowerHvxVecReduceFMinimum(Op, DAG);
3944 case ISD::VECREDUCE_FMAXIMUM:
3945 return LowerHvxVecReduceFMaximum(Op, DAG);
3946 case ISD::FMINNUM:
3947 return LowerHvxFMinNum(Op, DAG);
3948 case ISD::FMAXNUM:
3949 return LowerHvxFMaxNum(Op, DAG);
3950 // clang-format on
3951 }
3952#ifndef NDEBUG
3953 Op.dumpr(&DAG);
3954#endif
3955 llvm_unreachable("Unhandled HVX operation");
3956}
3957
3958SDValue
3959HexagonTargetLowering::ExpandHvxResizeIntoSteps(SDValue Op, SelectionDAG &DAG)
3960 const {
3961 // Rewrite the extension/truncation/saturation op into steps where each
3962 // step changes the type widths by a factor of 2.
3963 // E.g. i8 -> i16 remains unchanged, but i8 -> i32 ==> i8 -> i16 -> i32.
3964 //
3965 // Some of the vector types in Op may not be legal.
3966
3967 unsigned Opc = Op.getOpcode();
3968 switch (Opc) {
3969 case HexagonISD::SSAT:
3970 case HexagonISD::USAT:
3971 case HexagonISD::TL_EXTEND:
3972 case HexagonISD::TL_TRUNCATE:
3973 break;
3974 case ISD::ANY_EXTEND:
3975 case ISD::ZERO_EXTEND:
3976 case ISD::SIGN_EXTEND:
3977 case ISD::TRUNCATE:
3978 llvm_unreachable("ISD:: ops will be auto-folded");
3979 break;
3980#ifndef NDEBUG
3981 Op.dump(&DAG);
3982#endif
3983 llvm_unreachable("Unexpected operation");
3984 }
3985
3986 SDValue Inp = Op.getOperand(i: 0);
3987 MVT InpTy = ty(Op: Inp);
3988 MVT ResTy = ty(Op);
3989
3990 unsigned InpWidth = InpTy.getVectorElementType().getSizeInBits();
3991 unsigned ResWidth = ResTy.getVectorElementType().getSizeInBits();
3992 assert(InpWidth != ResWidth);
3993
3994 if (InpWidth == 2 * ResWidth || ResWidth == 2 * InpWidth)
3995 return Op;
3996
3997 const SDLoc &dl(Op);
3998 unsigned NumElems = InpTy.getVectorNumElements();
3999 assert(NumElems == ResTy.getVectorNumElements());
4000
4001 auto repeatOp = [&](unsigned NewWidth, SDValue Arg) {
4002 MVT Ty = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: NewWidth), NumElements: NumElems);
4003 switch (Opc) {
4004 case HexagonISD::SSAT:
4005 case HexagonISD::USAT:
4006 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, DAG.getValueType(Ty)});
4007 case HexagonISD::TL_EXTEND:
4008 case HexagonISD::TL_TRUNCATE:
4009 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, Op.getOperand(i: 1), Op.getOperand(i: 2)});
4010 default:
4011 llvm_unreachable("Unexpected opcode");
4012 }
4013 };
4014
4015 SDValue S = Inp;
4016 if (InpWidth < ResWidth) {
4017 assert(ResWidth % InpWidth == 0 && isPowerOf2_32(ResWidth / InpWidth));
4018 while (InpWidth * 2 <= ResWidth)
4019 S = repeatOp(InpWidth *= 2, S);
4020 } else {
4021 // InpWidth > ResWidth
4022 assert(InpWidth % ResWidth == 0 && isPowerOf2_32(InpWidth / ResWidth));
4023 while (InpWidth / 2 >= ResWidth)
4024 S = repeatOp(InpWidth /= 2, S);
4025 }
4026 return S;
4027}
4028
4029SDValue
4030HexagonTargetLowering::LegalizeHvxResize(SDValue Op, SelectionDAG &DAG) const {
4031 SDValue Inp0 = Op.getOperand(i: 0);
4032 MVT InpTy = ty(Op: Inp0);
4033 MVT ResTy = ty(Op);
4034 unsigned InpWidth = InpTy.getSizeInBits();
4035 unsigned ResWidth = ResTy.getSizeInBits();
4036 unsigned Opc = Op.getOpcode();
4037
4038 if (shouldWidenToHvx(Ty: InpTy, DAG) || shouldWidenToHvx(Ty: ResTy, DAG)) {
4039 // First, make sure that the narrower type is widened to HVX.
4040 // This may cause the result to be wider than what the legalizer
4041 // expects, so insert EXTRACT_SUBVECTOR to bring it back to the
4042 // desired type.
4043 auto [WInpTy, WResTy] =
4044 InpWidth < ResWidth ? typeWidenToWider(Ty0: typeWidenToHvx(Ty: InpTy), Ty1: ResTy)
4045 : typeWidenToWider(Ty0: InpTy, Ty1: typeWidenToHvx(Ty: ResTy));
4046 SDValue W = appendUndef(Val: Inp0, ResTy: WInpTy, DAG);
4047 SDValue S;
4048 if (Opc == HexagonISD::TL_EXTEND || Opc == HexagonISD::TL_TRUNCATE) {
4049 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: Op.getOperand(i: 1),
4050 N3: Op.getOperand(i: 2));
4051 } else {
4052 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: DAG.getValueType(WResTy));
4053 }
4054 SDValue T = ExpandHvxResizeIntoSteps(Op: S, DAG);
4055 return extractSubvector(Vec: T, SubTy: typeLegalize(Ty: ResTy, DAG), SubIdx: 0, DAG);
4056 } else if (shouldSplitToHvx(Ty: InpWidth < ResWidth ? ResTy : InpTy, DAG)) {
4057 // For multi-step extends/truncates (e.g., i8->i32), expand into
4058 // single-step operations first. Splitting a multi-step TL_EXTEND
4059 // would halve the operand type to a sub-HVX size (e.g., v128i8 ->
4060 // v64i8), creating illegal types that cause issues in the type
4061 // legalizer's map tracking. Single-step operations (e.g., i16->i32)
4062 // are safe to split because their halved operand types remain legal.
4063 SDValue T = ExpandHvxResizeIntoSteps(Op, DAG);
4064 if (T != Op)
4065 return T;
4066 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
4067 } else {
4068 assert(isTypeLegal(InpTy) && isTypeLegal(ResTy));
4069 return RemoveTLWrapper(Op, DAG);
4070 }
4071 llvm_unreachable("Unexpected situation");
4072}
4073
4074void
4075HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
4076 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
4077 unsigned Opc = N->getOpcode();
4078 SDValue Op(N, 0);
4079 SDValue Inp0; // Optional first argument.
4080 if (N->getNumOperands() > 0)
4081 Inp0 = Op.getOperand(i: 0);
4082
4083 switch (Opc) {
4084 case ISD::ANY_EXTEND:
4085 case ISD::SIGN_EXTEND:
4086 case ISD::ZERO_EXTEND:
4087 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4088 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4089 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4090 }
4091 break;
4092 case ISD::TRUNCATE:
4093 // Handle truncate to boolean vector when the input is not a
4094 // standard HVX vector type (single or pair). This covers cases
4095 // where the input needs widening (e.g., v64i8 -> v64i1 in
4096 // 128-byte mode) and cases where the result boolean type itself
4097 // needs widening (e.g., v16i32 -> v16i1). When the input is
4098 // already an HVX type, tablegen patterns handle the truncation
4099 // directly (e.g., v64i16 -> v64i1 via V6_vandvrt).
4100 if (ty(Op).getVectorElementType() == MVT::i1 &&
4101 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
4102 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
4103 Results.push_back(Elt: T);
4104 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4105 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4106 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4107 }
4108 break;
4109 case ISD::SETCC:
4110 if (shouldWidenToHvx(Ty: ty(Op: Inp0), DAG)) {
4111 if (SDValue T = WidenHvxSetCC(Op, DAG))
4112 Results.push_back(Elt: T);
4113 }
4114 break;
4115 case ISD::STORE: {
4116 if (shouldWidenToHvx(Ty: ty(Op: cast<StoreSDNode>(Val: N)->getValue()), DAG)) {
4117 SDValue Store = WidenHvxStore(Op, DAG);
4118 Results.push_back(Elt: Store);
4119 }
4120 break;
4121 }
4122 case ISD::MLOAD:
4123 if (isHvxPairTy(Ty: ty(Op))) {
4124 SDValue S = SplitHvxMemOp(Op, DAG);
4125 assert(S->getOpcode() == ISD::MERGE_VALUES);
4126 Results.push_back(Elt: S.getOperand(i: 0));
4127 Results.push_back(Elt: S.getOperand(i: 1));
4128 }
4129 break;
4130 case ISD::MSTORE:
4131 if (isHvxPairTy(Ty: ty(Op: Op->getOperand(Num: 1)))) { // Stored value
4132 SDValue S = SplitHvxMemOp(Op, DAG);
4133 Results.push_back(Elt: S);
4134 }
4135 break;
4136 case ISD::SINT_TO_FP:
4137 case ISD::UINT_TO_FP:
4138 case ISD::FP_TO_SINT:
4139 case ISD::FP_TO_UINT:
4140 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4141 SDValue T = EqualizeFpIntConversion(Op, DAG);
4142 Results.push_back(Elt: T);
4143 }
4144 break;
4145 case HexagonISD::SSAT:
4146 case HexagonISD::USAT:
4147 case HexagonISD::TL_EXTEND:
4148 case HexagonISD::TL_TRUNCATE:
4149 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4150 break;
4151 default:
4152 break;
4153 }
4154}
4155
4156void
4157HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
4158 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
4159 unsigned Opc = N->getOpcode();
4160 SDValue Op(N, 0);
4161 SDValue Inp0; // Optional first argument.
4162 if (N->getNumOperands() > 0)
4163 Inp0 = Op.getOperand(i: 0);
4164
4165 switch (Opc) {
4166 case ISD::ANY_EXTEND:
4167 case ISD::SIGN_EXTEND:
4168 case ISD::ZERO_EXTEND:
4169 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4170 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4171 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4172 }
4173 break;
4174 case ISD::TRUNCATE:
4175 // Handle truncate to boolean vector when the input is not a
4176 // standard HVX vector type. See comment in LowerHvxOperationWrapper.
4177 if (ty(Op).getVectorElementType() == MVT::i1 &&
4178 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
4179 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
4180 Results.push_back(Elt: T);
4181 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4182 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4183 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4184 }
4185 break;
4186 case ISD::SETCC:
4187 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4188 if (SDValue T = WidenHvxSetCC(Op, DAG))
4189 Results.push_back(Elt: T);
4190 }
4191 break;
4192 case ISD::LOAD: {
4193 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4194 SDValue Load = WidenHvxLoad(Op, DAG);
4195 assert(Load->getOpcode() == ISD::MERGE_VALUES);
4196 Results.push_back(Elt: Load.getOperand(i: 0));
4197 Results.push_back(Elt: Load.getOperand(i: 1));
4198 }
4199 break;
4200 }
4201 case ISD::BITCAST:
4202 if (isHvxBoolTy(Ty: ty(Op: Inp0))) {
4203 SDValue C = LowerHvxBitcast(Op, DAG);
4204 Results.push_back(Elt: C);
4205 }
4206 break;
4207 case ISD::FP_TO_SINT:
4208 case ISD::FP_TO_UINT:
4209 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4210 SDValue T = EqualizeFpIntConversion(Op, DAG);
4211 Results.push_back(Elt: T);
4212 }
4213 break;
4214 case HexagonISD::SSAT:
4215 case HexagonISD::USAT:
4216 case HexagonISD::TL_EXTEND:
4217 case HexagonISD::TL_TRUNCATE:
4218 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4219 break;
4220 default:
4221 break;
4222 }
4223}
4224
4225SDValue
4226HexagonTargetLowering::combineTruncateBeforeLegal(SDValue Op,
4227 DAGCombinerInfo &DCI) const {
4228 // Simplify V:v2NiB --(bitcast)--> vNi2B --(truncate)--> vNiB
4229 // to extract-subvector (shuffle V, pick even, pick odd)
4230
4231 assert(Op.getOpcode() == ISD::TRUNCATE);
4232 SelectionDAG &DAG = DCI.DAG;
4233 const SDLoc &dl(Op);
4234
4235 if (Op.getOperand(i: 0).getOpcode() == ISD::BITCAST)
4236 return SDValue();
4237 SDValue Cast = Op.getOperand(i: 0);
4238 SDValue Src = Cast.getOperand(i: 0);
4239
4240 EVT TruncTy = Op.getValueType();
4241 EVT CastTy = Cast.getValueType();
4242 EVT SrcTy = Src.getValueType();
4243 if (SrcTy.isSimple())
4244 return SDValue();
4245 if (SrcTy.getVectorElementType() != TruncTy.getVectorElementType())
4246 return SDValue();
4247 unsigned SrcLen = SrcTy.getVectorNumElements();
4248 unsigned CastLen = CastTy.getVectorNumElements();
4249 if (2 * CastLen != SrcLen)
4250 return SDValue();
4251
4252 SmallVector<int, 128> Mask(SrcLen);
4253 for (int i = 0; i != static_cast<int>(CastLen); ++i) {
4254 Mask[i] = 2 * i;
4255 Mask[i + CastLen] = 2 * i + 1;
4256 }
4257 SDValue Deal =
4258 DAG.getVectorShuffle(VT: SrcTy, dl, N1: Src, N2: DAG.getUNDEF(VT: SrcTy), Mask);
4259 return opSplit(Vec: Deal, dl, DAG).first;
4260}
4261
4262SDValue
4263HexagonTargetLowering::combineConcatOfShuffles(SDValue Op,
4264 SelectionDAG &DAG) const {
4265 // Fold
4266 // concat (shuffle x, y, m1), (shuffle x, y, m2)
4267 // into
4268 // shuffle (concat x, y), undef, m3
4269 if (Op.getNumOperands() != 2)
4270 return SDValue();
4271
4272 const SDLoc &dl(Op);
4273 SDValue V0 = Op.getOperand(i: 0);
4274 SDValue V1 = Op.getOperand(i: 1);
4275
4276 if (V0.getOpcode() != ISD::VECTOR_SHUFFLE)
4277 return SDValue();
4278 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE)
4279 return SDValue();
4280
4281 SetVector<SDValue> Order;
4282 Order.insert(X: V0.getOperand(i: 0));
4283 Order.insert(X: V0.getOperand(i: 1));
4284 Order.insert(X: V1.getOperand(i: 0));
4285 Order.insert(X: V1.getOperand(i: 1));
4286
4287 if (Order.size() > 2)
4288 return SDValue();
4289
4290 // In ISD::VECTOR_SHUFFLE, the types of each input and the type of the
4291 // result must be the same.
4292 EVT InpTy = V0.getValueType();
4293 assert(InpTy.isVector());
4294 unsigned InpLen = InpTy.getVectorNumElements();
4295
4296 SmallVector<int, 128> LongMask;
4297 auto AppendToMask = [&](SDValue Shuffle) {
4298 auto *SV = cast<ShuffleVectorSDNode>(Val: Shuffle.getNode());
4299 ArrayRef<int> Mask = SV->getMask();
4300 SDValue X = Shuffle.getOperand(i: 0);
4301 SDValue Y = Shuffle.getOperand(i: 1);
4302 for (int M : Mask) {
4303 if (M == -1) {
4304 LongMask.push_back(Elt: M);
4305 continue;
4306 }
4307 SDValue Src = static_cast<unsigned>(M) < InpLen ? X : Y;
4308 if (static_cast<unsigned>(M) >= InpLen)
4309 M -= InpLen;
4310
4311 int OutOffset = Order[0] == Src ? 0 : InpLen;
4312 LongMask.push_back(Elt: M + OutOffset);
4313 }
4314 };
4315
4316 AppendToMask(V0);
4317 AppendToMask(V1);
4318
4319 SDValue C0 = Order.front();
4320 SDValue C1 = Order.back(); // Can be same as front
4321 EVT LongTy = InpTy.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4322
4323 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LongTy, Ops: {C0, C1});
4324 return DAG.getVectorShuffle(VT: LongTy, dl, N1: Cat, N2: DAG.getUNDEF(VT: LongTy), Mask: LongMask);
4325}
4326
4327// Reassociate concat(p1, p2, ...) into
4328// concat(concat(p1, ...), concat(pi, ...), ...)
4329// where each inner concat produces a predicate where each bit corresponds
4330// to at most BitBytes bytes.
4331// Concatenating predicates decreases the number of bytes per each predicate
4332// bit.
4333SDValue
4334HexagonTargetLowering::combineConcatOfScalarPreds(SDValue Op, unsigned BitBytes,
4335 SelectionDAG &DAG) const {
4336 const SDLoc &dl(Op);
4337 SmallVector<SDValue> Ops(Op->ops());
4338 MVT ResTy = ty(Op);
4339 MVT InpTy = ty(Op: Ops[0]);
4340 unsigned InpLen = InpTy.getVectorNumElements(); // Scalar predicate
4341 unsigned ResLen = ResTy.getVectorNumElements(); // HVX vector predicate
4342 assert(InpLen <= 8 && "Too long for scalar predicate");
4343 assert(ResLen > 8 && "Too short for HVX vector predicate");
4344
4345 unsigned Bytes = 8 / InpLen; // Bytes-per-bit in input
4346
4347 // Already in the right form?
4348 if (Bytes <= BitBytes)
4349 return Op;
4350
4351 ArrayRef<SDValue> Inputs(Ops);
4352 unsigned SliceLen = Bytes / BitBytes;
4353
4354 SmallVector<SDValue> Cats;
4355 // (8 / BitBytes) is the desired length of the result of the inner concat.
4356 MVT InnerTy = MVT::getVectorVT(VT: MVT::i1, NumElements: 8 / BitBytes);
4357 for (unsigned i = 0; i != ResLen / (8 / BitBytes); ++i) {
4358 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: InnerTy,
4359 Ops: Inputs.slice(N: SliceLen * i, M: SliceLen));
4360 Cats.push_back(Elt: Cat);
4361 }
4362
4363 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Cats);
4364}
4365
4366SDValue HexagonTargetLowering::combineConcatVectorsBeforeLegal(
4367 SDValue Op, DAGCombinerInfo &DCI) const {
4368 MVT ResTy = ty(Op);
4369 MVT ElemTy = ResTy.getVectorElementType();
4370
4371 if (ElemTy != MVT::i1) {
4372 return combineConcatOfShuffles(Op, DAG&: DCI.DAG);
4373 }
4374 return SDValue();
4375}
4376
4377// Create the inner partial reduction MLA that can be efficiently lowered. This
4378// function is used by partial and full reductions.
4379SDValue HexagonTargetLowering::createExtendingPartialReduceMLA(
4380 unsigned Opcode, EVT AccEltType, unsigned AccNumElements, EVT InputType,
4381 const SDValue &A, const SDValue &B, unsigned &RemainingReductionRatio,
4382 const SDLoc &DL, SelectionDAG &DAG) const {
4383 const auto &Subtarget = DAG.getSubtarget<HexagonSubtarget>();
4384 if (!Subtarget.useHVXOps())
4385 return SDValue();
4386
4387 EVT InputEltType = InputType.getVectorElementType();
4388
4389 // Find if an optimized instruction for the sub-reduction is available.
4390 unsigned NativeRatio;
4391 if (AccEltType == MVT::i32 && InputEltType == MVT::i8)
4392 NativeRatio = 4;
4393 else
4394 return SDValue();
4395
4396 // We only handle the case when additional reduction will be needed, i.e.
4397 // input is longer by a larger factor than the result.
4398 ElementCount InputEC = InputType.getVectorElementCount();
4399 if (!InputEC.isKnownMultipleOf(RHS: AccNumElements * NativeRatio))
4400 return SDValue();
4401
4402 unsigned InputNumElements = InputEC.getFixedValue();
4403 RemainingReductionRatio = InputNumElements / (AccNumElements * NativeRatio);
4404 if (RemainingReductionRatio == 1)
4405 return SDValue();
4406
4407 // Create a reduction by the natively supported factor.
4408 EVT IntermediateType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType,
4409 NumElements: InputNumElements / NativeRatio);
4410
4411 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: IntermediateType);
4412 return DAG.getNode(Opcode, DL, VT: IntermediateType, N1: Zero, N2: A, N3: B);
4413}
4414
4415static bool DetectExtendingMultiply(const SDValue &N, EVT ScalarType,
4416 unsigned &Opcode, SDValue &A, SDValue &B) {
4417 SDValue Mul = N;
4418 EVT AccType = Mul.getValueType(); // Vector input type after extension.
4419 if (ScalarType != AccType.getVectorElementType())
4420 return false;
4421 bool swap = false;
4422 if (Mul->getOpcode() != ISD::MUL)
4423 return false;
4424 A = Mul->getOperand(Num: 0);
4425 B = Mul->getOperand(Num: 1);
4426 if (A.getOpcode() == ISD::ZERO_EXTEND) {
4427 if (B.getOpcode() == ISD::ZERO_EXTEND)
4428 Opcode = ISD::PARTIAL_REDUCE_UMLA;
4429 else if (B.getOpcode() == ISD::SIGN_EXTEND) {
4430 swap = true;
4431 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4432 } else
4433 return false;
4434 } else if (A.getOpcode() == ISD::SIGN_EXTEND) {
4435 if (B.getOpcode() == ISD::ZERO_EXTEND)
4436 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4437 else if (B.getOpcode() == ISD::SIGN_EXTEND)
4438 Opcode = ISD::PARTIAL_REDUCE_SMLA;
4439 else
4440 return false;
4441 } else
4442 return false;
4443
4444 // Get multiplication arguments before extension.
4445 A = A->getOperand(Num: 0);
4446 B = B->getOperand(Num: 0);
4447 if (A.getValueType() != B.getValueType())
4448 return false;
4449
4450 if (swap)
4451 std::swap(a&: A, b&: B);
4452
4453 return true;
4454}
4455
4456SDValue HexagonTargetLowering::splitVecReduceAdd(SDNode *N,
4457 SelectionDAG &DAG) const {
4458 if (!Subtarget.useHVXOps())
4459 return SDValue();
4460
4461 EVT ScalarType = N->getValueType(ResNo: 0);
4462 unsigned Opcode;
4463 SDValue A, B;
4464 if (!DetectExtendingMultiply(N: N->getOperand(Num: 0), ScalarType, Opcode, A, B))
4465 return SDValue();
4466
4467 SDLoc DL(N);
4468 unsigned RemainingReductionRatio;
4469 SDValue Partial =
4470 createExtendingPartialReduceMLA(Opcode, AccEltType: ScalarType, AccNumElements: 1, InputType: A.getValueType(),
4471 A, B, RemainingReductionRatio, DL, DAG);
4472 if (!Partial)
4473 return SDValue();
4474
4475 // We could have inserted a trivial MLA and rely on the folding action,
4476 // similar to how vector_partial_reduce_add is lowered to an MLA in
4477 // SelectionDAGBuilder. However, we just replace the final result since we
4478 // have analyzed the input completely.
4479 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: ScalarType, Operand: Partial);
4480}
4481
4482// Shared helper for VECREDUCE_FMIN/FMAX/FMINIMUM/FMAXIMUM on HVX float
4483// vector types. IgnoreNaN=true (FMIN/FMAX): NaN elements are replaced with
4484// the neutral value before the reduction so they don't corrupt the result
4485// even when the hardware pairwise instruction propagates NaN.
4486// IgnoreNaN=false (FMINIMUM/FMAXIMUM): NaN propagates naturally.
4487SDValue HexagonTargetLowering::LowerHvxVecReduceFMinMax(
4488 SDValue Op, unsigned PairwiseOpc, bool IgnoreNaN, SelectionDAG &DAG) const {
4489 SDLoc DL(Op);
4490 SDValue Vec = Op.getOperand(i: 0);
4491 MVT VecTy = ty(Op: Vec);
4492 SDNodeFlags Flags = Op->getFlags();
4493 bool ShouldStripNaN = IgnoreNaN && !Flags.hasNoNaNs();
4494
4495 // Save original input before NaN stripping; needed for the all-NaN fixup.
4496 SDValue OrigVec = Vec;
4497 MVT OrigVecTy = VecTy;
4498
4499 // Replace NaN elements in V with +/-Inf so the tree reduction ignores them.
4500 bool IsMax = (Op.getOpcode() == ISD::VECREDUCE_FMAX);
4501 auto ReplaceNaN = [&](SDValue V, MVT Ty) -> SDValue {
4502 APFloat Neutral = APFloat::getInf(
4503 Sem: Ty.getVectorElementType().getFltSemantics(), /*Negative=*/IsMax);
4504 SDValue NeutralVec = DAG.getConstantFP(Val: Neutral, DL, VT: Ty);
4505 EVT BoolTy = getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: Ty);
4506 SDValue IsNaN = DAG.getSetCC(DL, VT: BoolTy, LHS: V, RHS: V, Cond: ISD::SETUO);
4507 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN, N2: NeutralVec, N3: V);
4508 };
4509
4510 // For a pair vector, strip NaN from each half before the cross-half
4511 // pairwise reduction so a NaN in one half can't corrupt the other.
4512 if (isHvxPairTy(Ty: VecTy)) {
4513 auto [Lo, Hi] = opSplit(Vec, dl: DL, DAG);
4514 MVT SingleTy = ty(Op: Lo);
4515 if (ShouldStripNaN) {
4516 Lo = ReplaceNaN(Lo, SingleTy);
4517 Hi = ReplaceNaN(Hi, SingleTy);
4518 }
4519 Vec = DAG.getNode(Opcode: PairwiseOpc, DL, VT: SingleTy, N1: Lo, N2: Hi, Flags);
4520 VecTy = SingleTy;
4521 } else if (ShouldStripNaN) {
4522 Vec = ReplaceNaN(Vec, VecTy);
4523 }
4524
4525 // Tree reduction using VROR + pairwise op. Each iteration rotates the vector
4526 // by half the remaining element count (in bytes) and takes element-wise
4527 // min/max, halving the active width until element 0 holds the result.
4528 unsigned ElemBytes = VecTy.getScalarSizeInBits() / 8;
4529 unsigned HwLen = Subtarget.getVectorLength();
4530 unsigned NumElems = HwLen / ElemBytes;
4531
4532 SDValue Curr = Vec;
4533 for (unsigned Width = NumElems / 2; Width >= 1; Width /= 2) {
4534 SDValue RotAmt = DAG.getConstant(Val: Width * ElemBytes, DL, VT: MVT::i32);
4535 SDValue Rotated = DAG.getNode(Opcode: HexagonISD::VROR, DL, VT: VecTy, N1: Curr, N2: RotAmt);
4536 Curr = DAG.getNode(Opcode: PairwiseOpc, DL, VT: VecTy, N1: Curr, N2: Rotated, Flags);
4537 }
4538
4539 // Extract element 0 as the scalar result.
4540 MVT ScalarTy = Op.getSimpleValueType();
4541 SDValue Result = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ScalarTy, N1: Curr,
4542 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
4543
4544 // Per llvm.maxnum/minnum semantics, all-NaN input must return NaN. The
4545 // NaN-stripping above replaced every NaN with the neutral value, so an
4546 // all-NaN vector produces the neutral value instead. Fix: if no element
4547 // in the original vector was non-NaN, return NaN.
4548 if (ShouldStripNaN) {
4549 EVT BoolVecTy =
4550 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: OrigVecTy);
4551 // IsOrd[i] == 1 iff OrigVec[i] is not NaN.
4552 SDValue IsOrd = DAG.getSetCC(DL, VT: BoolVecTy, LHS: OrigVec, RHS: OrigVec, Cond: ISD::SETO);
4553 // BoolVecTy is vNi1 with exactly N bits; bitcast to an integer and
4554 // check != 0 to detect whether any element was non-NaN.
4555 unsigned NumBits = BoolVecTy.getSizeInBits();
4556 MVT IntTy = MVT::getIntegerVT(BitWidth: NumBits);
4557 SDValue IsOrdInt = DAG.getBitcast(VT: IntTy, V: IsOrd);
4558 SDValue AnyNonNaN = DAG.getSetCC(DL, VT: MVT::i1, LHS: IsOrdInt,
4559 RHS: DAG.getConstant(Val: 0, DL, VT: IntTy), Cond: ISD::SETNE);
4560 SDValue NaN = DAG.getConstantFP(Val: APFloat::getNaN(Sem: ScalarTy.getFltSemantics()),
4561 DL, VT: ScalarTy);
4562 Result = DAG.getSelect(DL, VT: ScalarTy, Cond: AnyNonNaN, LHS: Result, RHS: NaN);
4563 }
4564
4565 return Result;
4566}
4567
4568SDValue HexagonTargetLowering::LowerHvxVecReduceFMin(SDValue Op,
4569 SelectionDAG &DAG) const {
4570 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMINIMUM, /*IgnoreNaN=*/true, DAG);
4571}
4572
4573SDValue HexagonTargetLowering::LowerHvxVecReduceFMax(SDValue Op,
4574 SelectionDAG &DAG) const {
4575 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMAXIMUM, /*IgnoreNaN=*/true, DAG);
4576}
4577
4578SDValue
4579HexagonTargetLowering::LowerHvxVecReduceFMinimum(SDValue Op,
4580 SelectionDAG &DAG) const {
4581 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMINIMUM, /*IgnoreNaN=*/false, DAG);
4582}
4583
4584SDValue
4585HexagonTargetLowering::LowerHvxVecReduceFMaximum(SDValue Op,
4586 SelectionDAG &DAG) const {
4587 return LowerHvxVecReduceFMinMax(Op, PairwiseOpc: ISD::FMAXIMUM, /*IgnoreNaN=*/false, DAG);
4588}
4589
4590// Lower FMINNUM/FMAXNUM on a single HVX float vector. These ops must ignore
4591// NaN (return the non-NaN operand). Because the hardware vmin/vmax may
4592// propagate NaN, replace NaN in each operand with the neutral value (+/-Inf)
4593// before delegating to FMINIMUM/FMAXIMUM.
4594SDValue HexagonTargetLowering::LowerHvxFMinNum(SDValue Op,
4595 SelectionDAG &DAG) const {
4596 SDLoc DL(Op);
4597 auto A = Op.getOperand(i: 0), B = Op.getOperand(i: 1);
4598 auto Ty = ty(Op);
4599 auto Flags = Op->getFlags();
4600
4601 if (!Flags.hasNoNaNs()) {
4602 auto &Sem = Ty.getVectorElementType().getFltSemantics();
4603 auto PosInf = DAG.getConstantFP(Val: APFloat::getInf(Sem, Negative: false), DL, VT: Ty);
4604 auto BoolTy =
4605 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: Ty);
4606 auto IsNaN_A = DAG.getSetCC(DL, VT: BoolTy, LHS: A, RHS: A, Cond: ISD::SETUO);
4607 auto IsNaN_B = DAG.getSetCC(DL, VT: BoolTy, LHS: B, RHS: B, Cond: ISD::SETUO);
4608 // Per llvm.minnum: if both operands are NaN, return NaN. Replace NaN
4609 // with +Inf so the hardware min ignores single-operand NaN, then restore
4610 // NaN for lanes where both inputs were NaN.
4611 auto BothNaN = DAG.getNode(Opcode: ISD::AND, DL, VT: BoolTy, N1: IsNaN_A, N2: IsNaN_B);
4612 A = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_A, N2: PosInf, N3: A);
4613 B = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_B, N2: PosInf, N3: B);
4614 auto Res = DAG.getNode(Opcode: ISD::FMINIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4615 auto NaN = DAG.getConstantFP(Val: APFloat::getNaN(Sem), DL, VT: Ty);
4616 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: BothNaN, N2: NaN, N3: Res);
4617 }
4618 return DAG.getNode(Opcode: ISD::FMINIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4619}
4620
4621SDValue HexagonTargetLowering::LowerHvxFMaxNum(SDValue Op,
4622 SelectionDAG &DAG) const {
4623 SDLoc DL(Op);
4624 auto A = Op.getOperand(i: 0), B = Op.getOperand(i: 1);
4625 auto Ty = ty(Op);
4626 auto Flags = Op->getFlags();
4627
4628 if (!Flags.hasNoNaNs()) {
4629 auto &Sem = Ty.getVectorElementType().getFltSemantics();
4630 auto NegInf = DAG.getConstantFP(Val: APFloat::getInf(Sem, Negative: true), DL, VT: Ty);
4631 auto BoolTy =
4632 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: Ty);
4633 auto IsNaN_A = DAG.getSetCC(DL, VT: BoolTy, LHS: A, RHS: A, Cond: ISD::SETUO);
4634 auto IsNaN_B = DAG.getSetCC(DL, VT: BoolTy, LHS: B, RHS: B, Cond: ISD::SETUO);
4635 // Per llvm.maxnum: if both operands are NaN, return NaN. Replace NaN
4636 // with -Inf so the hardware max ignores single-operand NaN, then restore
4637 // NaN for lanes where both inputs were NaN.
4638 auto BothNaN = DAG.getNode(Opcode: ISD::AND, DL, VT: BoolTy, N1: IsNaN_A, N2: IsNaN_B);
4639 A = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_A, N2: NegInf, N3: A);
4640 B = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: IsNaN_B, N2: NegInf, N3: B);
4641 auto Res = DAG.getNode(Opcode: ISD::FMAXIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4642 auto NaN = DAG.getConstantFP(Val: APFloat::getNaN(Sem), DL, VT: Ty);
4643 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: Ty, N1: BothNaN, N2: NaN, N3: Res);
4644 }
4645 return DAG.getNode(Opcode: ISD::FMAXIMUM, DL, VT: Ty, N1: A, N2: B, Flags);
4646}
4647
4648// When possible, separate an MLA reduction with extended operands but
4649// unsupported reduction factor into an extending partial reduction that
4650// can be efficiently lowered, and a follow-up partial reduction.
4651// partial_reduce_mla(a, x, y) ->
4652// partial_reduce_mla(a, partial_reduce_mla(0, x, y), 1)
4653SDValue
4654HexagonTargetLowering::splitExtendingPartialReduceMLA(SDNode *N,
4655 SelectionDAG &DAG) const {
4656 if (!Subtarget.useHVXOps())
4657 return SDValue();
4658
4659 SDValue Acc = N->getOperand(Num: 0);
4660 SDValue A = N->getOperand(Num: 1);
4661 SDValue B = N->getOperand(Num: 2);
4662 if (A.getValueType() != B.getValueType())
4663 return SDValue();
4664
4665 // The types should be declared as custom, but do not split already legal
4666 // operation.
4667 EVT AccType = Acc.getValueType();
4668 EVT InputType = A.getValueType();
4669 if (getPartialReduceMLAAction(Opc: N->getOpcode(), AccVT: AccType, InputVT: InputType) != Custom)
4670 return SDValue();
4671
4672 SDLoc DL(N);
4673 unsigned RemainingReductionRatio;
4674 SDValue Partial = createExtendingPartialReduceMLA(
4675 Opcode: N->getOpcode(), AccEltType: AccType.getVectorElementType(),
4676 AccNumElements: AccType.getVectorNumElements(), InputType, A, B, RemainingReductionRatio,
4677 DL, DAG);
4678 if (!Partial)
4679 return SDValue();
4680 assert(RemainingReductionRatio <= MaxExpandMLA);
4681
4682 // Create the reduction for the remaining ratio.
4683 EVT IntermediateType = Partial->getOperand(Num: 0).getValueType();
4684 SDValue One = DAG.getConstant(Val: 1, DL, VT: IntermediateType);
4685 return DAG.getNode(Opcode: N->getOpcode() == ISD::PARTIAL_REDUCE_UMLA
4686 ? ISD::PARTIAL_REDUCE_UMLA
4687 : ISD::PARTIAL_REDUCE_SUMLA,
4688 DL, VT: AccType, N1: Acc, N2: Partial, N3: One);
4689}
4690
4691SDValue
4692HexagonTargetLowering::LowerHvxPartialReduceMLA(SDValue Op,
4693 SelectionDAG &DAG) const {
4694 const SDLoc &DL(Op);
4695 SDValue Acc = Op.getOperand(i: 0);
4696 SDValue A = Op.getOperand(i: 1);
4697 SDValue B = Op.getOperand(i: 2);
4698
4699 // Split the input vectors into units of one HVX vector length.
4700 unsigned HwVectorSizeInBits = Subtarget.getVectorLength() * 8;
4701
4702 EVT AccType = Acc.getValueType();
4703 EVT AccEltType = AccType.getVectorElementType();
4704 unsigned AccSubvectorNumElements =
4705 HwVectorSizeInBits / AccEltType.getSizeInBits();
4706 EVT AccSubvectorType =
4707 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType, NumElements: AccSubvectorNumElements);
4708
4709 EVT InputType = A.getValueType();
4710 assert(InputType.getSizeInBits() % HwVectorSizeInBits == 0);
4711 EVT InputEltType = InputType.getVectorElementType();
4712 unsigned InputSubvectorNumElements =
4713 HwVectorSizeInBits / InputEltType.getSizeInBits();
4714 EVT InputSubvectorType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InputEltType,
4715 NumElements: InputSubvectorNumElements);
4716
4717 unsigned SubvectorNum = InputType.getFixedSizeInBits() / HwVectorSizeInBits;
4718 SmallVector<SDValue, MaxExpandMLA> Subvectors;
4719
4720 for (unsigned I = 0; I != SubvectorNum; ++I) {
4721 SDValue SubvectorAcc = DAG.getExtractSubvector(DL, VT: AccSubvectorType, Vec: Acc,
4722 Idx: I * AccSubvectorNumElements);
4723 SDValue SubvectorA = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: A,
4724 Idx: I * InputSubvectorNumElements);
4725 SDValue SubvectorB = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: B,
4726 Idx: I * InputSubvectorNumElements);
4727 SDValue SubvectorMLA = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: AccSubvectorType,
4728 N1: SubvectorAcc, N2: SubvectorA, N3: SubvectorB);
4729 Subvectors.push_back(Elt: SubvectorMLA);
4730 }
4731
4732 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: AccType, Ops: Subvectors);
4733}
4734
4735// Lower fcmp oeq on HVX float vectors for architectures before v81, which
4736// lack a dedicated floating-point equality instruction.
4737//
4738// Correct IEEE-754 semantics: oeq(a,b) is true iff a==b and neither is NaN.
4739// We use the available float-GT instruction (V6_vgtsf/V6_vgthf) for the
4740// inequality check and bit manipulation for NaN detection:
4741//
4742// oeq(a, b) = NOT(ogt(a,b) OR ogt(b,a) OR isNaN(a) OR isNaN(b))
4743//
4744// where isNaN(x) = ((int_bits(x) & AbsMask) > NaNThreshold)
4745// f32: AbsMask=0x7FFFFFFF, NaNThreshold=0x7F800000
4746// f16: AbsMask=0x7FFF, NaNThreshold=0x7C00
4747//
4748// This handles +0/-0 correctly because float-GT treats them as equal, so
4749// neither ogt(+0,-0) nor ogt(-0,+0) is ever true.
4750//
4751// Example f32 assembly (no NaNs case):
4752// q0 = vcmp.eq(v0.w, v1.w) // bitwise comparison should just work
4753//
4754// Example f32 assembly (NaN-present case):
4755// q0 = vcmp.gt(v0.sf, v1.sf) // ogt(a,b)
4756// q0 |= vcmp.gt(v1.sf, v0.sf) // |= ogt(b,a)
4757// r0 = ##0x7FFFFFFF
4758// v2 = vsplat(r0) // AbsMask broadcast
4759// r1 = ##0x7F800000
4760// v3 = vsplat(r1) // NaNThresh broadcast
4761// v4 = vand(v0, v2) // int_bits(a) & AbsMask
4762// v5 = vand(v1, v2) // int_bits(b) & AbsMask
4763// q0 |= vcmp.gt(v4.w, v3.w) // |= isNaN(a)
4764// q0 |= vcmp.gt(v5.w, v3.w) // |= isNaN(b)
4765// // q0 now holds AnyFalse; result = XOR(q0, allones) = oeq
4766SDValue HexagonTargetLowering::LowerHvxFpSetoeq(SDValue Op,
4767 SelectionDAG &DAG) const {
4768 auto ResTy = ty(Op);
4769 auto A = Op.getOperand(i: 0), B = Op->getOperand(Num: 1);
4770 MVT FloatTy = ty(Op: A);
4771 MVT ElemTy = FloatTy.getVectorElementType();
4772 bool IsF32 = (ElemTy == MVT::f32);
4773 if (!IsF32) {
4774 assert((ElemTy == MVT::f16));
4775 }
4776 const SDLoc &DL(Op);
4777 MVT IntElemTy = IsF32 ? MVT::i32 : MVT::i16;
4778 MVT IntVecTy = tyVector(Ty: FloatTy, ElemTy: IntElemTy);
4779
4780 // Under nnan semantics NaN cannot appear, so integer equality is both
4781 // correct and cheaper (one instruction vs the float-GT sequence).
4782 bool NoNaN = Op->getFlags().hasNoNaNs();
4783 if (NoNaN) {
4784 SDValue IA = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: A);
4785 SDValue IB = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: B);
4786 return DAG.getSetCC(DL, VT: ResTy, LHS: IA, RHS: IB, Cond: ISD::SETEQ);
4787 }
4788
4789 // Float GT comparisons (IEEE-754: false whenever either operand is NaN).
4790 SDValue QAgtB = DAG.getSetCC(DL, VT: ResTy, LHS: A, RHS: B, Cond: ISD::SETOGT);
4791 SDValue QBgtA = DAG.getSetCC(DL, VT: ResTy, LHS: B, RHS: A, Cond: ISD::SETOGT);
4792
4793 // OR all "false" conditions together, then invert.
4794 SDValue AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: QAgtB, N2: QBgtA);
4795
4796 // Detect NaN by checking whether the unbiased exponent/mantissa field
4797 // exceeds the largest finite value.
4798 // f32: (bits & 0x7FFFFFFF) > 0x7F800000
4799 // f16: (bits & 0x7FFF) > 0x7C00
4800 uint64_t AbsMask = IsF32 ? 0x7FFFFFFFull : 0x7FFFull;
4801 uint64_t NaNThresh = IsF32 ? 0x7F800000ull : 0x7C00ull;
4802
4803 SDValue IA = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: A);
4804 SDValue IB = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: B);
4805 SDValue MaskVec = DAG.getConstant(Val: AbsMask, DL, VT: IntVecTy);
4806 SDValue ThreshVec = DAG.getConstant(Val: NaNThresh, DL, VT: IntVecTy);
4807 SDValue QNanA =
4808 DAG.getSetCC(DL, VT: ResTy, LHS: DAG.getNode(Opcode: ISD::AND, DL, VT: IntVecTy, N1: IA, N2: MaskVec),
4809 RHS: ThreshVec, Cond: ISD::SETGT);
4810 SDValue QNanB =
4811 DAG.getSetCC(DL, VT: ResTy, LHS: DAG.getNode(Opcode: ISD::AND, DL, VT: IntVecTy, N1: IB, N2: MaskVec),
4812 RHS: ThreshVec, Cond: ISD::SETGT);
4813 AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: AnyFalse, N2: QNanA);
4814 AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: AnyFalse, N2: QNanB);
4815
4816 // Result = NOT(<Is A gt B>, <IS B gt A>, <IS A NaN>, <IS B NaN>)
4817 // Use XOR with ones to simulate logical not.
4818 return DAG.getNode(Opcode: ISD::XOR, DL, VT: ResTy, N1: AnyFalse,
4819 N2: DAG.getConstant(Val: 1, DL, VT: ResTy));
4820}
4821
4822SDValue
4823HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
4824 const {
4825 const SDLoc &dl(N);
4826 SelectionDAG &DAG = DCI.DAG;
4827 SDValue Op(N, 0);
4828 unsigned Opc = Op.getOpcode();
4829
4830 SmallVector<SDValue, 4> Ops(N->ops());
4831
4832 if (Opc == ISD::TRUNCATE)
4833 return combineTruncateBeforeLegal(Op, DCI);
4834 if (Opc == ISD::CONCAT_VECTORS)
4835 return combineConcatVectorsBeforeLegal(Op, DCI);
4836
4837 if (DCI.isBeforeLegalizeOps())
4838 return SDValue();
4839
4840 switch (Opc) {
4841 case HexagonISD::V2Q:
4842 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) {
4843 if (const auto *C = dyn_cast<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
4844 return C->isZero() ? DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: ty(Op))
4845 : DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: ty(Op));
4846 }
4847 break;
4848 case HexagonISD::Q2V:
4849 if (Ops[0].getOpcode() == HexagonISD::QTRUE)
4850 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ty(Op),
4851 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
4852 if (Ops[0].getOpcode() == HexagonISD::QFALSE)
4853 return getZero(dl, Ty: ty(Op), DAG);
4854 break;
4855 case HexagonISD::VINSERTW0:
4856 if (isUndef(Op: Ops[1]))
4857 return Ops[0];
4858 break;
4859 case HexagonISD::VROR: {
4860 if (Ops[0].getOpcode() == HexagonISD::VROR) {
4861 SDValue Vec = Ops[0].getOperand(i: 0);
4862 SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(i: 1);
4863 SDValue Rot = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ty(Op: Rot0), Ops: {Rot0, Rot1});
4864 return DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ty(Op), Ops: {Vec, Rot});
4865 }
4866 break;
4867 }
4868 }
4869
4870 return SDValue();
4871}
4872
4873bool
4874HexagonTargetLowering::shouldSplitToHvx(MVT Ty, SelectionDAG &DAG) const {
4875 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4876 return false;
4877 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4878 if (Action == TargetLoweringBase::TypeSplitVector)
4879 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4880 return false;
4881}
4882
4883bool
4884HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const {
4885 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4886 return false;
4887 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4888 if (Action == TargetLoweringBase::TypeWidenVector)
4889 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4890 return false;
4891}
4892
4893bool
4894HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const {
4895 if (!Subtarget.useHVXOps())
4896 return false;
4897 // If the type of any result, or any operand type are HVX vector types,
4898 // this is an HVX operation.
4899 auto IsHvxTy = [this](EVT Ty) {
4900 return Ty.isSimple() && Subtarget.isHVXVectorType(VecTy: Ty.getSimpleVT(), IncludeBool: true);
4901 };
4902 auto IsHvxOp = [this](SDValue Op) {
4903 return Op.getValueType().isSimple() &&
4904 Subtarget.isHVXVectorType(VecTy: ty(Op), IncludeBool: true);
4905 };
4906 if (llvm::any_of(Range: N->values(), P: IsHvxTy) || llvm::any_of(Range: N->ops(), P: IsHvxOp))
4907 return true;
4908
4909 // Check if this could be an HVX operation after type widening.
4910 auto IsWidenedToHvx = [this, &DAG](SDValue Op) {
4911 if (!Op.getValueType().isSimple())
4912 return false;
4913 MVT ValTy = ty(Op);
4914 return ValTy.isVector() && shouldWidenToHvx(Ty: ValTy, DAG);
4915 };
4916
4917 for (int i = 0, e = N->getNumValues(); i != e; ++i) {
4918 if (IsWidenedToHvx(SDValue(N, i)))
4919 return true;
4920 }
4921 return llvm::any_of(Range: N->ops(), P: IsWidenedToHvx);
4922}
4923