1//===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "HexagonISelLowering.h"
10#include "HexagonRegisterInfo.h"
11#include "HexagonSubtarget.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/MemoryLocation.h"
15#include "llvm/CodeGen/MachineBasicBlock.h"
16#include "llvm/CodeGen/MachineFunction.h"
17#include "llvm/CodeGen/MachineInstr.h"
18#include "llvm/CodeGen/MachineOperand.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/TargetInstrInfo.h"
21#include "llvm/IR/IntrinsicsHexagon.h"
22#include "llvm/Support/CommandLine.h"
23
24#include <algorithm>
25#include <string>
26#include <utility>
27
28using namespace llvm;
29
30static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
31 cl::Hidden, cl::init(Val: 16),
32 cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
33
34static cl::opt<bool>
35 EnableFpFastConvert("hexagon-fp-fast-convert", cl::Hidden, cl::init(Val: false),
36 cl::desc("Enable FP fast conversion routine."));
37
38static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
39static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
40static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
41static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
42
43static const unsigned MaxExpandMLA = 8;
44
45static std::tuple<unsigned, unsigned, unsigned> getIEEEProperties(MVT Ty) {
46 // For a float scalar type, return (exp-bits, exp-bias, fraction-bits)
47 MVT ElemTy = Ty.getScalarType();
48 switch (ElemTy.SimpleTy) {
49 case MVT::f16:
50 return std::make_tuple(args: 5, args: 15, args: 10);
51 case MVT::f32:
52 return std::make_tuple(args: 8, args: 127, args: 23);
53 case MVT::f64:
54 return std::make_tuple(args: 11, args: 1023, args: 52);
55 default:
56 break;
57 }
58 llvm_unreachable(("Unexpected type: " + EVT(ElemTy).getEVTString()).c_str());
59}
60
61void
62HexagonTargetLowering::initializeHVXLowering() {
63 if (Subtarget.useHVX64BOps()) {
64 addRegisterClass(VT: MVT::v64i8, RC: &Hexagon::HvxVRRegClass);
65 addRegisterClass(VT: MVT::v32i16, RC: &Hexagon::HvxVRRegClass);
66 addRegisterClass(VT: MVT::v16i32, RC: &Hexagon::HvxVRRegClass);
67 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxWRRegClass);
68 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxWRRegClass);
69 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxWRRegClass);
70 // These "short" boolean vector types should be legal because
71 // they will appear as results of vector compares. If they were
72 // not legal, type legalization would try to make them legal
73 // and that would require using operations that do not use or
74 // produce such types. That, in turn, would imply using custom
75 // nodes, which would be unoptimizable by the DAG combiner.
76 // The idea is to rely on target-independent operations as much
77 // as possible.
78 addRegisterClass(VT: MVT::v16i1, RC: &Hexagon::HvxQRRegClass);
79 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
80 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
81 } else if (Subtarget.useHVX128BOps()) {
82 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxVRRegClass);
83 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxVRRegClass);
84 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxVRRegClass);
85 addRegisterClass(VT: MVT::v256i8, RC: &Hexagon::HvxWRRegClass);
86 addRegisterClass(VT: MVT::v128i16, RC: &Hexagon::HvxWRRegClass);
87 addRegisterClass(VT: MVT::v64i32, RC: &Hexagon::HvxWRRegClass);
88 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
89 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
90 addRegisterClass(VT: MVT::v128i1, RC: &Hexagon::HvxQRRegClass);
91 if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) {
92 addRegisterClass(VT: MVT::v32f32, RC: &Hexagon::HvxVRRegClass);
93 addRegisterClass(VT: MVT::v64f16, RC: &Hexagon::HvxVRRegClass);
94 addRegisterClass(VT: MVT::v64f32, RC: &Hexagon::HvxWRRegClass);
95 addRegisterClass(VT: MVT::v128f16, RC: &Hexagon::HvxWRRegClass);
96 }
97 if (Subtarget.useHVXV81Ops()) {
98 addRegisterClass(VT: MVT::v64bf16, RC: &Hexagon::HvxVRRegClass);
99 addRegisterClass(VT: MVT::v128bf16, RC: &Hexagon::HvxWRRegClass);
100 }
101 }
102
103 // Set up operation actions.
104
105 bool Use64b = Subtarget.useHVX64BOps();
106 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
107 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
108 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
109 MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32;
110 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
111
112 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
113 setOperationAction(Op: Opc, VT: FromTy, Action: Promote);
114 AddPromotedToType(Opc, OrigVT: FromTy, DestVT: ToTy);
115 };
116
117 // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
118 // Note: v16i1 -> i16 is handled in type legalization instead of op
119 // legalization.
120 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
121 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
122 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
123 setOperationAction(Op: ISD::BITCAST, VT: MVT::v16i1, Action: Custom);
124 setOperationAction(Op: ISD::BITCAST, VT: MVT::v128i1, Action: Custom);
125 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
126 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteV, Action: Legal);
127 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteW, Action: Legal);
128 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
129
130 if (Subtarget.useHVX128BOps()) {
131 setOperationAction(Op: ISD::BITCAST, VT: MVT::v32i1, Action: Custom);
132 setOperationAction(Op: ISD::BITCAST, VT: MVT::v64i1, Action: Custom);
133 setOperationAction(Op: ISD::STORE, VT: MVT::v32i1, Action: Custom);
134 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i1, Action: Custom);
135 setOperationAction(Op: ISD::STORE, VT: MVT::v64i1, Action: Custom);
136 setOperationAction(Op: ISD::LOAD, VT: MVT::v64i1, Action: Custom);
137 setOperationAction(Op: ISD::STORE, VT: MVT::v128i1, Action: Custom);
138 setOperationAction(Op: ISD::LOAD, VT: MVT::v128i1, Action: Custom);
139 }
140 if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() &&
141 Subtarget.useHVXFloatingPoint()) {
142
143 static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 };
144 static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 };
145
146 for (MVT T : FloatV) {
147 setOperationAction(Op: ISD::FADD, VT: T, Action: Legal);
148 setOperationAction(Op: ISD::FSUB, VT: T, Action: Legal);
149 setOperationAction(Op: ISD::FMUL, VT: T, Action: Legal);
150 setOperationAction(Op: ISD::FMINIMUMNUM, VT: T, Action: Legal);
151 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: T, Action: Legal);
152 setOperationAction(Op: ISD::FMINNUM, VT: T, Action: Legal);
153 setOperationAction(Op: ISD::FMAXNUM, VT: T, Action: Legal);
154
155 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
156 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
157
158 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
159 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
160
161 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
162 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
163 // Custom-lower BUILD_VECTOR. The standard (target-independent)
164 // handling of it would convert it to a load, which is not always
165 // the optimal choice.
166 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
167 }
168
169
170 // BUILD_VECTOR with f16 operands cannot be promoted without
171 // promoting the result, so lower the node to vsplat or constant pool
172 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
173 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::f16, Action: Custom);
174 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::f16, Action: Custom);
175
176 // Vector shuffle is always promoted to ByteV and a bitcast to f16 is
177 // generated.
178 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128f16, ByteW);
179 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f16, ByteV);
180 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f32, ByteW);
181 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v32f32, ByteV);
182
183 // For HVX <v81 there is no hardware float-equality instruction; only
184 // float-GT (V6_vgtsf/V6_vgthf) is available. The integer-equality
185 // fallback (V6_veqw/V6_veqh) silently treats NaN as equal to itself
186 // because the bit patterns match. Mark SETCC as Custom for the
187 // single-vector float types so we can synthesise the correct
188 // ordered-equal predicate in LowerHvxFpSetoeq.
189 if (!Subtarget.useHVXV81Ops())
190 for (MVT T : FloatV)
191 setOperationAction(Op: ISD::SETCC, VT: T, Action: Custom);
192
193 if (Subtarget.useHVXV81Ops()) {
194 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128bf16, ByteW);
195 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64bf16, ByteV);
196 setPromoteTo(ISD::SETCC, MVT::v64bf16, MVT::v64f32);
197 setPromoteTo(ISD::FADD, MVT::v64bf16, MVT::v64f32);
198 setPromoteTo(ISD::FSUB, MVT::v64bf16, MVT::v64f32);
199 setPromoteTo(ISD::FMUL, MVT::v64bf16, MVT::v64f32);
200 setPromoteTo(ISD::FMINNUM, MVT::v64bf16, MVT::v64f32);
201 setPromoteTo(ISD::FMAXNUM, MVT::v64bf16, MVT::v64f32);
202
203 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::v64bf16, Action: Legal);
204 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
205 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
206
207 setOperationAction(Op: ISD::LOAD, VT: MVT::v128bf16, Action: Custom);
208 setOperationAction(Op: ISD::STORE, VT: MVT::v128bf16, Action: Custom);
209
210 setOperationAction(Op: ISD::MLOAD, VT: MVT::v64bf16, Action: Custom);
211 setOperationAction(Op: ISD::MSTORE, VT: MVT::v64bf16, Action: Custom);
212 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v64bf16, Action: Custom);
213 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v64bf16, Action: Custom);
214
215 setOperationAction(Op: ISD::MLOAD, VT: MVT::v128bf16, Action: Custom);
216 setOperationAction(Op: ISD::MSTORE, VT: MVT::v128bf16, Action: Custom);
217 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v128bf16, Action: Custom);
218 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v128bf16, Action: Custom);
219
220 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::bf16, Action: Custom);
221 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::bf16, Action: Custom);
222 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::bf16, Action: Custom);
223 }
224
225 for (MVT P : FloatW) {
226 setOperationAction(Op: ISD::LOAD, VT: P, Action: Custom);
227 setOperationAction(Op: ISD::STORE, VT: P, Action: Custom);
228 setOperationAction(Op: ISD::FADD, VT: P, Action: Custom);
229 setOperationAction(Op: ISD::FSUB, VT: P, Action: Custom);
230 setOperationAction(Op: ISD::FMUL, VT: P, Action: Custom);
231 setOperationAction(Op: ISD::FMINIMUMNUM, VT: P, Action: Custom);
232 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: P, Action: Custom);
233 setOperationAction(Op: ISD::FMINNUM, VT: P, Action: Custom);
234 setOperationAction(Op: ISD::FMAXNUM, VT: P, Action: Custom);
235 setOperationAction(Op: ISD::SETCC, VT: P, Action: Custom);
236 setOperationAction(Op: ISD::VSELECT, VT: P, Action: Custom);
237
238 // Custom-lower BUILD_VECTOR. The standard (target-independent)
239 // handling of it would convert it to a load, which is not always
240 // the optimal choice.
241 setOperationAction(Op: ISD::BUILD_VECTOR, VT: P, Action: Custom);
242 // Make concat-vectors custom to handle concats of more than 2 vectors.
243 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: P, Action: Custom);
244
245 setOperationAction(Op: ISD::MLOAD, VT: P, Action: Custom);
246 setOperationAction(Op: ISD::MSTORE, VT: P, Action: Custom);
247 }
248
249 if (Subtarget.useHVXQFloatOps()) {
250 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Custom);
251 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
252 } else if (Subtarget.useHVXIEEEFPOps()) {
253 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Legal);
254 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
255 }
256 }
257
258 for (MVT T : LegalV) {
259 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
260 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
261
262 setOperationAction(Op: ISD::ABS, VT: T, Action: Legal);
263 setOperationAction(Op: ISD::AND, VT: T, Action: Legal);
264 setOperationAction(Op: ISD::OR, VT: T, Action: Legal);
265 setOperationAction(Op: ISD::XOR, VT: T, Action: Legal);
266 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
267 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
268 setOperationAction(Op: ISD::MUL, VT: T, Action: Legal);
269 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Legal);
270 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Legal);
271 setOperationAction(Op: ISD::SELECT, VT: T, Action: Legal);
272 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
273 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
274 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
275 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
276 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
277 if (T != ByteV) {
278 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
279 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
280 setOperationAction(Op: ISD::BSWAP, VT: T, Action: Legal);
281 }
282
283 setOperationAction(Op: ISD::SMIN, VT: T, Action: Legal);
284 setOperationAction(Op: ISD::SMAX, VT: T, Action: Legal);
285 if (T.getScalarType() != MVT::i32) {
286 setOperationAction(Op: ISD::UMIN, VT: T, Action: Legal);
287 setOperationAction(Op: ISD::UMAX, VT: T, Action: Legal);
288 }
289
290 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
291 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
292 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
293 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
294 if (T.getScalarType() != MVT::i32) {
295 setOperationAction(Op: ISD::MULHS, VT: T, Action: Legal);
296 setOperationAction(Op: ISD::MULHU, VT: T, Action: Legal);
297 }
298
299 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
300 // Make concat-vectors custom to handle concats of more than 2 vectors.
301 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
302 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
303 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: T, Action: Custom);
304 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
305 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: T, Action: Custom);
306 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
307 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
308 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
309 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
310 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
311 if (T != ByteV) {
312 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
313 // HVX only has shifts of words and halfwords.
314 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
315 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
316 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
317
318 // Promote all shuffles to operate on vectors of bytes.
319 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
320 }
321
322 if (Subtarget.useHVXFloatingPoint()) {
323 // Same action for both QFloat and IEEE.
324 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
325 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
326 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
327 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
328 }
329
330 setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Expand);
331 setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Expand);
332 setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Expand);
333 setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Expand);
334 setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Expand);
335 setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Expand);
336 setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Expand);
337 }
338
339 for (MVT T : LegalW) {
340 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
341 // independent) handling of it would convert it to a load, which is
342 // not always the optimal choice.
343 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
344 // Make concat-vectors custom to handle concats of more than 2 vectors.
345 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
346
347 // Custom-lower these operations for pairs. Expand them into a concat
348 // of the corresponding operations on individual vectors.
349 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
350 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
351 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
352 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Custom);
353 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
354 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
355 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
356 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Custom);
357
358 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
359 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
360 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
361 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
362 setOperationAction(Op: ISD::ABS, VT: T, Action: Custom);
363 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Custom);
364 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
365 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Custom);
366
367 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
368 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
369 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
370 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
371 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
372 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
373 setOperationAction(Op: ISD::MUL, VT: T, Action: Custom);
374 setOperationAction(Op: ISD::MULHS, VT: T, Action: Custom);
375 setOperationAction(Op: ISD::MULHU, VT: T, Action: Custom);
376 setOperationAction(Op: ISD::AND, VT: T, Action: Custom);
377 setOperationAction(Op: ISD::OR, VT: T, Action: Custom);
378 setOperationAction(Op: ISD::XOR, VT: T, Action: Custom);
379 setOperationAction(Op: ISD::SETCC, VT: T, Action: Custom);
380 setOperationAction(Op: ISD::VSELECT, VT: T, Action: Custom);
381 if (T != ByteW) {
382 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
383 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
384 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
385
386 // Promote all shuffles to operate on vectors of bytes.
387 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
388 }
389 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
390 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
391
392 setOperationAction(Op: ISD::SMIN, VT: T, Action: Custom);
393 setOperationAction(Op: ISD::SMAX, VT: T, Action: Custom);
394 if (T.getScalarType() != MVT::i32) {
395 setOperationAction(Op: ISD::UMIN, VT: T, Action: Custom);
396 setOperationAction(Op: ISD::UMAX, VT: T, Action: Custom);
397 }
398
399 if (Subtarget.useHVXFloatingPoint()) {
400 // Same action for both QFloat and IEEE.
401 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
402 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
403 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
404 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
405 }
406 }
407
408 // Legalize all of these to HexagonISD::[SU]MUL_LOHI.
409 setOperationAction(Op: ISD::MULHS, VT: WordV, Action: Custom); // -> _LOHI
410 setOperationAction(Op: ISD::MULHU, VT: WordV, Action: Custom); // -> _LOHI
411 setOperationAction(Op: ISD::SMUL_LOHI, VT: WordV, Action: Custom);
412 setOperationAction(Op: ISD::UMUL_LOHI, VT: WordV, Action: Custom);
413
414 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v64f16, Action: Expand);
415 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v64f16, Action: Expand);
416 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v64f16, Action: Expand);
417 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v64f16, Action: Expand);
418 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v64f16, Action: Expand);
419 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v64f16, Action: Expand);
420 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v64f16, Action: Expand);
421 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v64f16, Action: Expand);
422 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v64f16, Action: Expand);
423 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v64f16, Action: Expand);
424 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v64f16, Action: Expand);
425 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v64f16, Action: Expand);
426 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v64f16, Action: Expand);
427 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v64f16, Action: Expand);
428
429 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v32f32, Action: Expand);
430 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v32f32, Action: Expand);
431 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v32f32, Action: Expand);
432 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v32f32, Action: Expand);
433 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v32f32, Action: Expand);
434 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v32f32, Action: Expand);
435 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v32f32, Action: Expand);
436 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v32f32, Action: Expand);
437 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v32f32, Action: Expand);
438 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v32f32, Action: Expand);
439 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v32f32, Action: Expand);
440 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v32f32, Action: Expand);
441 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v32f32, Action: Expand);
442 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v32f32, Action: Expand);
443
444 // Boolean vectors.
445
446 for (MVT T : LegalW) {
447 // Boolean types for vector pairs will overlap with the boolean
448 // types for single vectors, e.g.
449 // v64i8 -> v64i1 (single)
450 // v64i16 -> v64i1 (pair)
451 // Set these actions first, and allow the single actions to overwrite
452 // any duplicates.
453 MVT BoolW = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
454 setOperationAction(Op: ISD::SETCC, VT: BoolW, Action: Custom);
455 setOperationAction(Op: ISD::AND, VT: BoolW, Action: Custom);
456 setOperationAction(Op: ISD::OR, VT: BoolW, Action: Custom);
457 setOperationAction(Op: ISD::XOR, VT: BoolW, Action: Custom);
458 // Masked load/store takes a mask that may need splitting.
459 setOperationAction(Op: ISD::MLOAD, VT: BoolW, Action: Custom);
460 setOperationAction(Op: ISD::MSTORE, VT: BoolW, Action: Custom);
461 }
462
463 for (MVT T : LegalV) {
464 MVT BoolV = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
465 setOperationAction(Op: ISD::BUILD_VECTOR, VT: BoolV, Action: Custom);
466 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: BoolV, Action: Custom);
467 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: BoolV, Action: Custom);
468 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: BoolV, Action: Custom);
469 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: BoolV, Action: Custom);
470 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: BoolV, Action: Custom);
471 setOperationAction(Op: ISD::SELECT, VT: BoolV, Action: Custom);
472 setOperationAction(Op: ISD::AND, VT: BoolV, Action: Legal);
473 setOperationAction(Op: ISD::OR, VT: BoolV, Action: Legal);
474 setOperationAction(Op: ISD::XOR, VT: BoolV, Action: Legal);
475 }
476
477 if (Use64b) {
478 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
479 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
480 } else {
481 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
482 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
483 }
484
485 // Handle store widening for short vectors.
486 unsigned HwLen = Subtarget.getVectorLength();
487 for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
488 if (ElemTy == MVT::i1)
489 continue;
490 int ElemWidth = ElemTy.getFixedSizeInBits();
491 int MaxElems = (8*HwLen) / ElemWidth;
492 for (int N = 2; N < MaxElems; N *= 2) {
493 MVT VecTy = MVT::getVectorVT(VT: ElemTy, NumElements: N);
494 auto Action = getPreferredVectorAction(VT: VecTy);
495 if (Action == TargetLoweringBase::TypeWidenVector) {
496 setOperationAction(Op: ISD::LOAD, VT: VecTy, Action: Custom);
497 setOperationAction(Op: ISD::STORE, VT: VecTy, Action: Custom);
498 setOperationAction(Op: ISD::SETCC, VT: VecTy, Action: Custom);
499 setOperationAction(Op: ISD::TRUNCATE, VT: VecTy, Action: Custom);
500 setOperationAction(Op: ISD::ANY_EXTEND, VT: VecTy, Action: Custom);
501 setOperationAction(Op: ISD::SIGN_EXTEND, VT: VecTy, Action: Custom);
502 setOperationAction(Op: ISD::ZERO_EXTEND, VT: VecTy, Action: Custom);
503 if (Subtarget.useHVXFloatingPoint()) {
504 setOperationAction(Op: ISD::FP_TO_SINT, VT: VecTy, Action: Custom);
505 setOperationAction(Op: ISD::FP_TO_UINT, VT: VecTy, Action: Custom);
506 setOperationAction(Op: ISD::SINT_TO_FP, VT: VecTy, Action: Custom);
507 setOperationAction(Op: ISD::UINT_TO_FP, VT: VecTy, Action: Custom);
508 }
509
510 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: N);
511 if (!isTypeLegal(VT: BoolTy))
512 setOperationAction(Op: ISD::SETCC, VT: BoolTy, Action: Custom);
513 }
514 }
515 }
516
517 // Include cases which are not hander earlier
518 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v32i1, Action: Custom);
519 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v64i1, Action: Custom);
520 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v32i1, Action: Custom);
521
522 setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT});
523
524 setTargetDAGCombine({ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA,
525 ISD::PARTIAL_REDUCE_SUMLA});
526
527 // Partial MLA reductions.
528 {
529 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
530 ISD::PARTIAL_REDUCE_UMLA,
531 ISD::PARTIAL_REDUCE_SUMLA};
532
533 auto HvxType = [=](MVT ScalarT, unsigned Factor = 1) {
534 return MVT::getVectorVT(VT: ScalarT, NumElements: Subtarget.getVectorLength() * Factor *
535 8 / ScalarT.getSizeInBits());
536 };
537
538 // Tuple of (Acc element type, input element type, vector pair).
539 // The assumption is both the input and reduction result are of the same
540 // size so the reduction ratio is the same as the ratio of element type
541 // sizes. This may not hold for all available instructions.
542 typedef std::tuple<MVT, MVT, bool> ReductionSignature;
543
544 static const std::vector<ReductionSignature> NativeReductions = {
545 {MVT::i32, MVT::i8, false},
546 };
547
548 for (const auto &R : NativeReductions) {
549
550 MVT AccType = std::get<0>(t: R);
551 MVT InputType = std::get<1>(t: R);
552 unsigned Factor = std::get<2>(t: R) ? 2 : 1;
553
554 // The native size is legal.
555 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: HvxType(AccType), InputVT: HvxType(InputType),
556 Action: Legal);
557
558 // Allow custom partial MLA reductions on larger vectors than legally
559 // supported. These reduction must be declared as Custom (or Legal)
560 // for foldPartialReduceMLAMulOp() to fold the multiply by one pattern
561 // inserted when the partial reduction intrinsic is converted to
562 // PARTIAL_REDUCE_U/S/SUMLA. Otherwise, the Split action will apply
563 // on the original pattern, including the extensions and multiplies,
564 // which will make it impossible to match.
565 // There are two independent ways to extend the
566 // input size: 1. to concatenate the result - output vector is
567 // proportionally extended, 2) to reduce the result - the output vector
568 // size stays the same. We limit allowed combinations so that the total
569 // number of generated reduction instructions is limited by a constant
570 // number. This limit is arbitrary and can be revised. On one hand, it is
571 // convenient to have more choices; on the other hand, there is a
572 // diminishing benefit of very long sequences, which should probably be
573 // written as loops instead.
574 for (unsigned ConcatFactor = 1; ConcatFactor <= MaxExpandMLA;
575 ConcatFactor <<= 1)
576 for (unsigned ReductionFactor = 1; ReductionFactor <= MaxExpandMLA;
577 ReductionFactor <<= 1)
578 if (ConcatFactor * ReductionFactor != 1 &&
579 ConcatFactor * ReductionFactor <= MaxExpandMLA)
580 setPartialReduceMLAAction(
581 Opcodes: MLAOps, AccVT: HvxType(AccType, Factor * ConcatFactor),
582 InputVT: HvxType(InputType, Factor * ConcatFactor * ReductionFactor),
583 Action: Custom);
584 }
585 }
586}
587
588unsigned
589HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
590 // Early exit for invalid input types
591 if (!VecTy.isVector())
592 return ~0u;
593
594 MVT ElemTy = VecTy.getVectorElementType();
595 unsigned VecLen = VecTy.getVectorNumElements();
596 unsigned HwLen = Subtarget.getVectorLength();
597
598 // Split vectors of i1 that exceed byte vector length.
599 if (ElemTy == MVT::i1 && VecLen > HwLen)
600 return TargetLoweringBase::TypeSplitVector;
601
602 ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
603 // For shorter vectors of i1, widen them if any of the corresponding
604 // vectors of integers needs to be widened.
605 if (ElemTy == MVT::i1) {
606 for (MVT T : Tys) {
607 assert(T != MVT::i1);
608 auto A = getPreferredHvxVectorAction(VecTy: MVT::getVectorVT(VT: T, NumElements: VecLen));
609 if (A != ~0u)
610 return A;
611 }
612 return ~0u;
613 }
614
615 // If the size of VecTy is at least half of the vector length,
616 // widen the vector. Note: the threshold was not selected in
617 // any scientific way.
618 if (llvm::is_contained(Range&: Tys, Element: ElemTy)) {
619 unsigned VecWidth = VecTy.getSizeInBits();
620 unsigned HwWidth = 8*HwLen;
621 if (VecWidth > 2*HwWidth)
622 return TargetLoweringBase::TypeSplitVector;
623
624 bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0;
625 if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth)
626 return TargetLoweringBase::TypeWidenVector;
627 if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
628 return TargetLoweringBase::TypeWidenVector;
629 }
630
631 // Defer to default.
632 return ~0u;
633}
634
635unsigned
636HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const {
637 unsigned Opc = Op.getOpcode();
638 switch (Opc) {
639 case HexagonISD::SMUL_LOHI:
640 case HexagonISD::UMUL_LOHI:
641 case HexagonISD::USMUL_LOHI:
642 return TargetLoweringBase::Custom;
643 }
644 return TargetLoweringBase::Legal;
645}
646
647SDValue
648HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
649 const SDLoc &dl, SelectionDAG &DAG) const {
650 SmallVector<SDValue,4> IntOps;
651 IntOps.push_back(Elt: DAG.getConstant(Val: IntId, DL: dl, VT: MVT::i32));
652 append_range(C&: IntOps, R&: Ops);
653 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: ResTy, Ops: IntOps);
654}
655
656MVT
657HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
658 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
659
660 MVT ElemTy = Tys.first.getVectorElementType();
661 return MVT::getVectorVT(VT: ElemTy, NumElements: Tys.first.getVectorNumElements() +
662 Tys.second.getVectorNumElements());
663}
664
665HexagonTargetLowering::TypePair
666HexagonTargetLowering::typeSplit(MVT VecTy) const {
667 assert(VecTy.isVector());
668 unsigned NumElem = VecTy.getVectorNumElements();
669 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
670 MVT HalfTy = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: NumElem/2);
671 return { HalfTy, HalfTy };
672}
673
674MVT
675HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
676 MVT ElemTy = VecTy.getVectorElementType();
677 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() * Factor);
678 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
679}
680
681MVT
682HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
683 MVT ElemTy = VecTy.getVectorElementType();
684 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() / Factor);
685 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
686}
687
688SDValue
689HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
690 SelectionDAG &DAG) const {
691 if (ty(Op: Vec).getVectorElementType() == ElemTy)
692 return Vec;
693 MVT CastTy = tyVector(Ty: Vec.getValueType().getSimpleVT(), ElemTy);
694 return DAG.getBitcast(VT: CastTy, V: Vec);
695}
696
697SDValue
698HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
699 SelectionDAG &DAG) const {
700 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: typeJoin(Tys: ty(Ops)),
701 N1: Ops.first, N2: Ops.second);
702}
703
704HexagonTargetLowering::VectorPair
705HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
706 SelectionDAG &DAG) const {
707 TypePair Tys = typeSplit(VecTy: ty(Op: Vec));
708 if (Vec.getOpcode() == HexagonISD::QCAT)
709 return VectorPair(Vec.getOperand(i: 0), Vec.getOperand(i: 1));
710 return DAG.SplitVector(N: Vec, DL: dl, LoVT: Tys.first, HiVT: Tys.second);
711}
712
713bool
714HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
715 return Subtarget.isHVXVectorType(VecTy: Ty) &&
716 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
717}
718
719bool
720HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
721 return Subtarget.isHVXVectorType(VecTy: Ty) &&
722 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
723}
724
725bool
726HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
727 return Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true) &&
728 Ty.getVectorElementType() == MVT::i1;
729}
730
731bool HexagonTargetLowering::allowsHvxMemoryAccess(
732 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
733 // Bool vectors are excluded by default, but make it explicit to
734 // emphasize that bool vectors cannot be loaded or stored.
735 // Also, disallow double vector stores (to prevent unnecessary
736 // store widening in DAG combiner).
737 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
738 return false;
739 if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
740 return false;
741 if (Fast)
742 *Fast = 1;
743 return true;
744}
745
746bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
747 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
748 if (!Subtarget.isHVXVectorType(VecTy))
749 return false;
750 // XXX Should this be false? vmemu are a bit slower than vmem.
751 if (Fast)
752 *Fast = 1;
753 return true;
754}
755
756void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection(
757 MachineInstr &MI, SDNode *Node) const {
758 unsigned Opc = MI.getOpcode();
759 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
760 MachineBasicBlock &MB = *MI.getParent();
761 MachineFunction &MF = *MB.getParent();
762 MachineRegisterInfo &MRI = MF.getRegInfo();
763 DebugLoc DL = MI.getDebugLoc();
764 auto At = MI.getIterator();
765
766 switch (Opc) {
767 case Hexagon::PS_vsplatib:
768 if (Subtarget.useHVXV62Ops()) {
769 // SplatV = A2_tfrsi #imm
770 // OutV = V6_lvsplatb SplatV
771 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
772 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
773 .add(MO: MI.getOperand(i: 1));
774 Register OutV = MI.getOperand(i: 0).getReg();
775 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
776 .addReg(RegNo: SplatV);
777 } else {
778 // SplatV = A2_tfrsi #imm:#imm:#imm:#imm
779 // OutV = V6_lvsplatw SplatV
780 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
781 const MachineOperand &InpOp = MI.getOperand(i: 1);
782 assert(InpOp.isImm());
783 uint32_t V = InpOp.getImm() & 0xFF;
784 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
785 .addImm(Val: V << 24 | V << 16 | V << 8 | V);
786 Register OutV = MI.getOperand(i: 0).getReg();
787 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
788 }
789 MB.erase(I: At);
790 break;
791 case Hexagon::PS_vsplatrb:
792 if (Subtarget.useHVXV62Ops()) {
793 // OutV = V6_lvsplatb Inp
794 Register OutV = MI.getOperand(i: 0).getReg();
795 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
796 .add(MO: MI.getOperand(i: 1));
797 } else {
798 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
799 const MachineOperand &InpOp = MI.getOperand(i: 1);
800 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::S2_vsplatrb), DestReg: SplatV)
801 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
802 Register OutV = MI.getOperand(i: 0).getReg();
803 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV)
804 .addReg(RegNo: SplatV);
805 }
806 MB.erase(I: At);
807 break;
808 case Hexagon::PS_vsplatih:
809 if (Subtarget.useHVXV62Ops()) {
810 // SplatV = A2_tfrsi #imm
811 // OutV = V6_lvsplath SplatV
812 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
813 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
814 .add(MO: MI.getOperand(i: 1));
815 Register OutV = MI.getOperand(i: 0).getReg();
816 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
817 .addReg(RegNo: SplatV);
818 } else {
819 // SplatV = A2_tfrsi #imm:#imm
820 // OutV = V6_lvsplatw SplatV
821 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
822 const MachineOperand &InpOp = MI.getOperand(i: 1);
823 assert(InpOp.isImm());
824 uint32_t V = InpOp.getImm() & 0xFFFF;
825 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
826 .addImm(Val: V << 16 | V);
827 Register OutV = MI.getOperand(i: 0).getReg();
828 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
829 }
830 MB.erase(I: At);
831 break;
832 case Hexagon::PS_vsplatrh:
833 if (Subtarget.useHVXV62Ops()) {
834 // OutV = V6_lvsplath Inp
835 Register OutV = MI.getOperand(i: 0).getReg();
836 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
837 .add(MO: MI.getOperand(i: 1));
838 } else {
839 // SplatV = A2_combine_ll Inp, Inp
840 // OutV = V6_lvsplatw SplatV
841 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
842 const MachineOperand &InpOp = MI.getOperand(i: 1);
843 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_combine_ll), DestReg: SplatV)
844 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg())
845 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
846 Register OutV = MI.getOperand(i: 0).getReg();
847 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
848 }
849 MB.erase(I: At);
850 break;
851 case Hexagon::PS_vsplatiw:
852 case Hexagon::PS_vsplatrw:
853 if (Opc == Hexagon::PS_vsplatiw) {
854 // SplatV = A2_tfrsi #imm
855 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
856 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
857 .add(MO: MI.getOperand(i: 1));
858 MI.getOperand(i: 1).ChangeToRegister(Reg: SplatV, isDef: false);
859 }
860 // OutV = V6_lvsplatw SplatV/Inp
861 MI.setDesc(TII.get(Opcode: Hexagon::V6_lvsplatw));
862 break;
863 }
864}
865
866SDValue
867HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
868 SelectionDAG &DAG) const {
869 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
870 ElemIdx = DAG.getBitcast(VT: MVT::i32, V: ElemIdx);
871
872 unsigned ElemWidth = ElemTy.getSizeInBits();
873 if (ElemWidth == 8)
874 return ElemIdx;
875
876 unsigned L = Log2_32(Value: ElemWidth/8);
877 const SDLoc &dl(ElemIdx);
878 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32,
879 Ops: {ElemIdx, DAG.getConstant(Val: L, DL: dl, VT: MVT::i32)});
880}
881
882SDValue
883HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
884 SelectionDAG &DAG) const {
885 unsigned ElemWidth = ElemTy.getSizeInBits();
886 assert(ElemWidth >= 8 && ElemWidth <= 32);
887 if (ElemWidth == 32)
888 return Idx;
889
890 if (ty(Op: Idx) != MVT::i32)
891 Idx = DAG.getBitcast(VT: MVT::i32, V: Idx);
892 const SDLoc &dl(Idx);
893 SDValue Mask = DAG.getConstant(Val: 32/ElemWidth - 1, DL: dl, VT: MVT::i32);
894 SDValue SubIdx = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, Ops: {Idx, Mask});
895 return SubIdx;
896}
897
898SDValue
899HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
900 SDValue Op1, ArrayRef<int> Mask,
901 SelectionDAG &DAG) const {
902 MVT OpTy = ty(Op: Op0);
903 assert(OpTy == ty(Op1));
904
905 MVT ElemTy = OpTy.getVectorElementType();
906 if (ElemTy == MVT::i8)
907 return DAG.getVectorShuffle(VT: OpTy, dl, N1: Op0, N2: Op1, Mask);
908 assert(ElemTy.getSizeInBits() >= 8);
909
910 MVT ResTy = tyVector(Ty: OpTy, ElemTy: MVT::i8);
911 unsigned ElemSize = ElemTy.getSizeInBits() / 8;
912
913 SmallVector<int,128> ByteMask;
914 for (int M : Mask) {
915 if (M < 0) {
916 for (unsigned I = 0; I != ElemSize; ++I)
917 ByteMask.push_back(Elt: -1);
918 } else {
919 int NewM = M*ElemSize;
920 for (unsigned I = 0; I != ElemSize; ++I)
921 ByteMask.push_back(Elt: NewM+I);
922 }
923 }
924 assert(ResTy.getVectorNumElements() == ByteMask.size());
925 return DAG.getVectorShuffle(VT: ResTy, dl, N1: opCastElem(Vec: Op0, ElemTy: MVT::i8, DAG),
926 N2: opCastElem(Vec: Op1, ElemTy: MVT::i8, DAG), Mask: ByteMask);
927}
928
929SDValue
930HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
931 const SDLoc &dl, MVT VecTy,
932 SelectionDAG &DAG) const {
933 unsigned VecLen = Values.size();
934 MachineFunction &MF = DAG.getMachineFunction();
935 MVT ElemTy = VecTy.getVectorElementType();
936 unsigned ElemWidth = ElemTy.getSizeInBits();
937 unsigned HwLen = Subtarget.getVectorLength();
938
939 unsigned ElemSize = ElemWidth / 8;
940 assert(ElemSize*VecLen == HwLen);
941 SmallVector<SDValue,32> Words;
942
943 if (VecTy.getVectorElementType() != MVT::i32 &&
944 !(Subtarget.useHVXFloatingPoint() &&
945 VecTy.getVectorElementType() == MVT::f32)) {
946 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
947 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
948 MVT PartVT = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: OpsPerWord);
949 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
950 SDValue W = buildVector32(Elem: Values.slice(N: i, M: OpsPerWord), dl, VecTy: PartVT, DAG);
951 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: W));
952 }
953 } else {
954 for (SDValue V : Values)
955 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V));
956 }
957 auto isSplat = [] (ArrayRef<SDValue> Values, SDValue &SplatV) {
958 unsigned NumValues = Values.size();
959 assert(NumValues > 0);
960 bool IsUndef = true;
961 for (unsigned i = 0; i != NumValues; ++i) {
962 if (Values[i].isUndef())
963 continue;
964 IsUndef = false;
965 if (!SplatV.getNode())
966 SplatV = Values[i];
967 else if (SplatV != Values[i])
968 return false;
969 }
970 if (IsUndef)
971 SplatV = Values[0];
972 return true;
973 };
974
975 unsigned NumWords = Words.size();
976 SDValue SplatV;
977 bool IsSplat = isSplat(Words, SplatV);
978 if (IsSplat && isUndef(Op: SplatV))
979 return DAG.getUNDEF(VT: VecTy);
980 if (IsSplat) {
981 assert(SplatV.getNode());
982 if (isNullConstant(V: SplatV))
983 return getZero(dl, Ty: VecTy, DAG);
984 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
985 SDValue S = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordTy, Operand: SplatV);
986 return DAG.getBitcast(VT: VecTy, V: S);
987 }
988
989 // Delay recognizing constant vectors until here, so that we can generate
990 // a vsplat.
991 SmallVector<ConstantInt*, 128> Consts(VecLen);
992 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
993 if (AllConst) {
994 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
995 (Constant**)Consts.end());
996 Constant *CV = ConstantVector::get(V: Tmp);
997 Align Alignment(HwLen);
998 SDValue CP = LowerConstantPool(
999 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
1000 DAG);
1001 return DAG.getLoad(VT: VecTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
1002 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
1003 }
1004
1005 // A special case is a situation where the vector is built entirely from
1006 // elements extracted from another vector. This could be done via a shuffle
1007 // more efficiently, but typically, the size of the source vector will not
1008 // match the size of the vector being built (which precludes the use of a
1009 // shuffle directly).
1010 // This only handles a single source vector, and the vector being built
1011 // should be of a sub-vector type of the source vector type.
1012 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
1013 SmallVectorImpl<int> &SrcIdx) {
1014 SDValue Vec;
1015 for (SDValue V : Values) {
1016 if (isUndef(Op: V)) {
1017 SrcIdx.push_back(Elt: -1);
1018 continue;
1019 }
1020 if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1021 return false;
1022 // All extracts should come from the same vector.
1023 SDValue T = V.getOperand(i: 0);
1024 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
1025 return false;
1026 Vec = T;
1027 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
1028 if (C == nullptr)
1029 return false;
1030 int I = C->getSExtValue();
1031 assert(I >= 0 && "Negative element index");
1032 SrcIdx.push_back(Elt: I);
1033 }
1034 SrcVec = Vec;
1035 return true;
1036 };
1037
1038 SmallVector<int,128> ExtIdx;
1039 SDValue ExtVec;
1040 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
1041 MVT ExtTy = ty(Op: ExtVec);
1042 unsigned ExtLen = ExtTy.getVectorNumElements();
1043 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
1044 // Construct a new shuffle mask that will produce a vector with the same
1045 // number of elements as the input vector, and such that the vector we
1046 // want will be the initial subvector of it.
1047 SmallVector<int,128> Mask;
1048 BitVector Used(ExtLen);
1049
1050 for (int M : ExtIdx) {
1051 Mask.push_back(Elt: M);
1052 if (M >= 0)
1053 Used.set(M);
1054 }
1055 // Fill the rest of the mask with the unused elements of ExtVec in hopes
1056 // that it will result in a permutation of ExtVec's elements. It's still
1057 // fine if it doesn't (e.g. if undefs are present, or elements are
1058 // repeated), but permutations can always be done efficiently via vdelta
1059 // and vrdelta.
1060 for (unsigned I = 0; I != ExtLen; ++I) {
1061 if (Mask.size() == ExtLen)
1062 break;
1063 if (!Used.test(Idx: I))
1064 Mask.push_back(Elt: I);
1065 }
1066
1067 SDValue S = DAG.getVectorShuffle(VT: ExtTy, dl, N1: ExtVec,
1068 N2: DAG.getUNDEF(VT: ExtTy), Mask);
1069 return ExtLen == VecLen ? S : LoHalf(V: S, DAG);
1070 }
1071 }
1072
1073 // Find most common element to initialize vector with. This is to avoid
1074 // unnecessary vinsert/valign for cases where the same value is present
1075 // many times. Creates a histogram of the vector's elements to find the
1076 // most common element n.
1077 assert(4*Words.size() == Subtarget.getVectorLength());
1078 int VecHist[32];
1079 int n = 0;
1080 for (unsigned i = 0; i != NumWords; ++i) {
1081 VecHist[i] = 0;
1082 if (Words[i].isUndef())
1083 continue;
1084 for (unsigned j = i; j != NumWords; ++j)
1085 if (Words[i] == Words[j])
1086 VecHist[i]++;
1087
1088 if (VecHist[i] > VecHist[n])
1089 n = i;
1090 }
1091
1092 SDValue HalfV = getZero(dl, Ty: VecTy, DAG);
1093 if (VecHist[n] > 1) {
1094 // Always splat at word (i32) granularity so that the SPLAT_VECTOR node
1095 // is selected as PS_vsplatrw (word broadcast) rather than PS_vsplatrb
1096 // (byte broadcast of the low byte only), which would corrupt multi-byte
1097 // element types.
1098 MVT WordVecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
1099 SDValue WordSplat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordVecTy, Operand: Words[n]);
1100 SDValue SplatV = DAG.getBitcast(VT: VecTy, V: WordSplat);
1101 HalfV = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: VecTy,
1102 Ops: {HalfV, SplatV, DAG.getConstant(Val: HwLen/2, DL: dl, VT: MVT::i32)});
1103 }
1104 SDValue HalfV0 = HalfV;
1105 SDValue HalfV1 = HalfV;
1106
1107 // Construct two halves in parallel, then or them together. Rn and Rm count
1108 // number of rotations needed before the next element. One last rotation is
1109 // performed post-loop to position the last element.
1110 int Rn = 0, Rm = 0;
1111 SDValue Sn, Sm;
1112 SDValue N = HalfV0;
1113 SDValue M = HalfV1;
1114 for (unsigned i = 0; i != NumWords/2; ++i) {
1115 // Rotate by element count since last insertion.
1116 if (Words[i] != Words[n] || VecHist[n] <= 1) {
1117 Sn = DAG.getConstant(Val: Rn, DL: dl, VT: MVT::i32);
1118 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1119 N = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1120 Ops: {HalfV0, Words[i]});
1121 Rn = 0;
1122 }
1123 if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) {
1124 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1125 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1126 M = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1127 Ops: {HalfV1, Words[i+NumWords/2]});
1128 Rm = 0;
1129 }
1130 Rn += 4;
1131 Rm += 4;
1132 }
1133 // Perform last rotation.
1134 Sn = DAG.getConstant(Val: Rn+HwLen/2, DL: dl, VT: MVT::i32);
1135 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1136 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1137 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1138
1139 SDValue T0 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV0);
1140 SDValue T1 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV1);
1141
1142 SDValue DstV = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ty(Op: T0), Ops: {T0, T1});
1143
1144 SDValue OutV =
1145 DAG.getBitcast(VT: tyVector(Ty: ty(Op: DstV), ElemTy: VecTy.getVectorElementType()), V: DstV);
1146 return OutV;
1147}
1148
1149SDValue
1150HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
1151 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
1152 MVT PredTy = ty(Op: PredV);
1153 unsigned HwLen = Subtarget.getVectorLength();
1154 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1155
1156 if (Subtarget.isHVXVectorType(VecTy: PredTy, IncludeBool: true)) {
1157 // Move the vector predicate SubV to a vector register, and scale it
1158 // down to match the representation (bytes per type element) that VecV
1159 // uses. The scaling down will pick every 2nd or 4th (every Scale-th
1160 // in general) element and put them at the front of the resulting
1161 // vector. This subvector will then be inserted into the Q2V of VecV.
1162 // To avoid having an operation that generates an illegal type (short
1163 // vector), generate a full size vector.
1164 //
1165 SDValue T = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: PredV);
1166 SmallVector<int,128> Mask(HwLen);
1167 // Scale = BitBytes(PredV) / Given BitBytes.
1168 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
1169 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
1170
1171 for (unsigned i = 0; i != HwLen; ++i) {
1172 unsigned Num = i % Scale;
1173 unsigned Off = i / Scale;
1174 Mask[BlockLen*Num + Off] = i;
1175 }
1176 SDValue S = DAG.getVectorShuffle(VT: ByteTy, dl, N1: T, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1177 if (!ZeroFill)
1178 return S;
1179 // Fill the bytes beyond BlockLen with 0s.
1180 // V6_pred_scalar2 cannot fill the entire predicate, so it only works
1181 // when BlockLen < HwLen.
1182 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1183 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1184 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1185 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1186 SDValue M = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Q);
1187 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ByteTy, N1: S, N2: M);
1188 }
1189
1190 // Make sure that this is a valid scalar predicate.
1191 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
1192
1193 unsigned Bytes = 8 / PredTy.getVectorNumElements();
1194 SmallVector<SDValue,4> Words[2];
1195 unsigned IdxW = 0;
1196
1197 SDValue W0 = isUndef(Op: PredV)
1198 ? DAG.getUNDEF(VT: MVT::i64)
1199 : DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: PredV);
1200 Words[IdxW].push_back(Elt: HiHalf(V: W0, DAG));
1201 Words[IdxW].push_back(Elt: LoHalf(V: W0, DAG));
1202
1203 while (Bytes < BitBytes) {
1204 IdxW ^= 1;
1205 Words[IdxW].clear();
1206
1207 if (Bytes < 4) {
1208 for (const SDValue &W : Words[IdxW ^ 1]) {
1209 SDValue T = expandPredicate(Vec32: W, dl, DAG);
1210 Words[IdxW].push_back(Elt: HiHalf(V: T, DAG));
1211 Words[IdxW].push_back(Elt: LoHalf(V: T, DAG));
1212 }
1213 } else {
1214 for (const SDValue &W : Words[IdxW ^ 1]) {
1215 Words[IdxW].push_back(Elt: W);
1216 Words[IdxW].push_back(Elt: W);
1217 }
1218 }
1219 Bytes *= 2;
1220 }
1221
1222 assert(Bytes == BitBytes);
1223 SDValue Vec = ZeroFill ? getZero(dl, Ty: ByteTy, DAG) : DAG.getUNDEF(VT: ByteTy);
1224 SDValue S4 = DAG.getConstant(Val: HwLen-4, DL: dl, VT: MVT::i32);
1225 for (const SDValue &W : Words[IdxW]) {
1226 Vec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Vec, N2: S4);
1227 Vec = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: ByteTy, N1: Vec, N2: W);
1228 }
1229
1230 return Vec;
1231}
1232
1233SDValue
1234HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
1235 const SDLoc &dl, MVT VecTy,
1236 SelectionDAG &DAG) const {
1237 // Construct a vector V of bytes, such that a comparison V >u 0 would
1238 // produce the required vector predicate.
1239 unsigned VecLen = Values.size();
1240 unsigned HwLen = Subtarget.getVectorLength();
1241 assert(VecLen <= HwLen || VecLen == 8*HwLen);
1242 SmallVector<SDValue,128> Bytes;
1243 bool AllT = true, AllF = true;
1244
1245 auto IsTrue = [] (SDValue V) {
1246 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1247 return !N->isZero();
1248 return false;
1249 };
1250 auto IsFalse = [] (SDValue V) {
1251 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1252 return N->isZero();
1253 return false;
1254 };
1255
1256 if (VecLen <= HwLen) {
1257 // In the hardware, each bit of a vector predicate corresponds to a byte
1258 // of a vector register. Calculate how many bytes does a bit of VecTy
1259 // correspond to.
1260 assert(HwLen % VecLen == 0);
1261 unsigned BitBytes = HwLen / VecLen;
1262 for (SDValue V : Values) {
1263 AllT &= IsTrue(V);
1264 AllF &= IsFalse(V);
1265
1266 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(Op: V, DL: dl, VT: MVT::i8)
1267 : DAG.getUNDEF(VT: MVT::i8);
1268 for (unsigned B = 0; B != BitBytes; ++B)
1269 Bytes.push_back(Elt: Ext);
1270 }
1271 } else {
1272 // There are as many i1 values, as there are bits in a vector register.
1273 // Divide the values into groups of 8 and check that each group consists
1274 // of the same value (ignoring undefs).
1275 for (unsigned I = 0; I != VecLen; I += 8) {
1276 unsigned B = 0;
1277 // Find the first non-undef value in this group.
1278 for (; B != 8; ++B) {
1279 if (!Values[I+B].isUndef())
1280 break;
1281 }
1282 SDValue F = Values[I+B];
1283 AllT &= IsTrue(F);
1284 AllF &= IsFalse(F);
1285
1286 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(Op: F, DL: dl, VT: MVT::i8)
1287 : DAG.getUNDEF(VT: MVT::i8);
1288 Bytes.push_back(Elt: Ext);
1289 // Verify that the rest of values in the group are the same as the
1290 // first.
1291 for (; B != 8; ++B)
1292 assert(Values[I+B].isUndef() || Values[I+B] == F);
1293 }
1294 }
1295
1296 if (AllT)
1297 return DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: VecTy);
1298 if (AllF)
1299 return DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: VecTy);
1300
1301 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1302 SDValue ByteVec = buildHvxVectorReg(Values: Bytes, dl, VecTy: ByteTy, DAG);
1303 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1304}
1305
1306SDValue
1307HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
1308 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1309 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1310
1311 unsigned ElemWidth = ElemTy.getSizeInBits();
1312 assert(ElemWidth >= 8 && ElemWidth <= 32);
1313 (void)ElemWidth;
1314
1315 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1316 SDValue ExWord = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1317 Ops: {VecV, ByteIdx});
1318 if (ElemTy == MVT::i32)
1319 return ExWord;
1320
1321 // Have an extracted word, need to extract the smaller element out of it.
1322 // 1. Extract the bits of (the original) IdxV that correspond to the index
1323 // of the desired element in the 32-bit word.
1324 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1325 // 2. Extract the element from the word.
1326 SDValue ExVec = DAG.getBitcast(VT: tyVector(Ty: ty(Op: ExWord), ElemTy), V: ExWord);
1327 return extractVector(VecV: ExVec, IdxV: SubIdx, dl, ValTy: ElemTy, ResTy: MVT::i32, DAG);
1328}
1329
1330SDValue
1331HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
1332 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1333 // Implement other return types if necessary.
1334 assert(ResTy == MVT::i1);
1335
1336 unsigned HwLen = Subtarget.getVectorLength();
1337 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1338 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1339
1340 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1341 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1342 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1343
1344 SDValue ExtB = extractHvxElementReg(VecV: ByteVec, IdxV, dl, ResTy: MVT::i32, DAG);
1345 SDValue Zero = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1346 return getInstr(MachineOpc: Hexagon::C2_cmpgtui, dl, Ty: MVT::i1, Ops: {ExtB, Zero}, DAG);
1347}
1348
1349SDValue
1350HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
1351 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1352 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1353
1354 unsigned ElemWidth = ElemTy.getSizeInBits();
1355 assert(ElemWidth >= 8 && ElemWidth <= 32);
1356 (void)ElemWidth;
1357
1358 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
1359 SDValue ByteIdxV) {
1360 MVT VecTy = ty(Op: VecV);
1361 unsigned HwLen = Subtarget.getVectorLength();
1362 SDValue MaskV =
1363 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
1364 Ops: {ByteIdxV, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)});
1365 SDValue RotV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {VecV, MaskV});
1366 SDValue InsV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy, Ops: {RotV, ValV});
1367 SDValue SubV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1368 Ops: {DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32), MaskV});
1369 SDValue TorV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {InsV, SubV});
1370 return TorV;
1371 };
1372
1373 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1374 if (ElemTy == MVT::i32)
1375 return InsertWord(VecV, ValV, ByteIdx);
1376
1377 // If this is not inserting a 32-bit word, convert it into such a thing.
1378 // 1. Extract the existing word from the target vector.
1379 SDValue WordIdx = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
1380 Ops: {ByteIdx, DAG.getConstant(Val: 2, DL: dl, VT: MVT::i32)});
1381 SDValue Ext = extractHvxElementReg(VecV: opCastElem(Vec: VecV, ElemTy: MVT::i32, DAG), IdxV: WordIdx,
1382 dl, ResTy: MVT::i32, DAG);
1383
1384 // 2. Treating the extracted word as a 32-bit vector, insert the given
1385 // value into it.
1386 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1387 MVT SubVecTy = tyVector(Ty: ty(Op: Ext), ElemTy);
1388 SDValue Ins = insertVector(VecV: DAG.getBitcast(VT: SubVecTy, V: Ext),
1389 ValV, IdxV: SubIdx, dl, ValTy: ElemTy, DAG);
1390
1391 // 3. Insert the 32-bit word back into the original vector.
1392 return InsertWord(VecV, Ins, ByteIdx);
1393}
1394
1395SDValue
1396HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
1397 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1398 unsigned HwLen = Subtarget.getVectorLength();
1399 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1400 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1401
1402 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1403 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1404 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1405 ValV = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i32, Operand: ValV);
1406
1407 SDValue InsV = insertHvxElementReg(VecV: ByteVec, IdxV, ValV, dl, DAG);
1408 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ty(Op: VecV), Operand: InsV);
1409}
1410
1411SDValue
1412HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV,
1413 SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1414 MVT VecTy = ty(Op: VecV);
1415 unsigned HwLen = Subtarget.getVectorLength();
1416 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1417 MVT ElemTy = VecTy.getVectorElementType();
1418 unsigned ElemWidth = ElemTy.getSizeInBits();
1419
1420 // If the source vector is a vector pair, get the single vector containing
1421 // the subvector of interest. The subvector will never overlap two single
1422 // vectors.
1423 if (isHvxPairTy(Ty: VecTy)) {
1424 unsigned SubIdx = Hexagon::vsub_lo;
1425 if (Idx * ElemWidth >= 8 * HwLen) {
1426 SubIdx = Hexagon::vsub_hi;
1427 Idx -= VecTy.getVectorNumElements() / 2;
1428 }
1429
1430 VecTy = typeSplit(VecTy).first;
1431 VecV = DAG.getTargetExtractSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV);
1432 if (VecTy == ResTy)
1433 return VecV;
1434 }
1435
1436 // The only meaningful subvectors of a single HVX vector are those that
1437 // fit in a scalar register.
1438 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
1439
1440 MVT WordTy = tyVector(Ty: VecTy, ElemTy: MVT::i32);
1441 SDValue WordVec = DAG.getBitcast(VT: WordTy, V: VecV);
1442 unsigned WordIdx = (Idx*ElemWidth) / 32;
1443
1444 SDValue W0Idx = DAG.getConstant(Val: WordIdx, DL: dl, VT: MVT::i32);
1445 SDValue W0 = extractHvxElementReg(VecV: WordVec, IdxV: W0Idx, dl, ResTy: MVT::i32, DAG);
1446 if (ResTy.getSizeInBits() == 32)
1447 return DAG.getBitcast(VT: ResTy, V: W0);
1448
1449 SDValue W1Idx = DAG.getConstant(Val: WordIdx+1, DL: dl, VT: MVT::i32);
1450 SDValue W1 = extractHvxElementReg(VecV: WordVec, IdxV: W1Idx, dl, ResTy: MVT::i32, DAG);
1451 SDValue WW = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::i64, DAG);
1452 return DAG.getBitcast(VT: ResTy, V: WW);
1453}
1454
1455SDValue
1456HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
1457 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1458 MVT VecTy = ty(Op: VecV);
1459 unsigned HwLen = Subtarget.getVectorLength();
1460 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1461 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1462 // IdxV is required to be a constant.
1463 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1464
1465 unsigned ResLen = ResTy.getVectorNumElements();
1466 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1467 unsigned Offset = Idx * BitBytes;
1468 SDValue Undef = DAG.getUNDEF(VT: ByteTy);
1469 SmallVector<int,128> Mask;
1470
1471 if (Subtarget.isHVXVectorType(VecTy: ResTy, IncludeBool: true)) {
1472 // Converting between two vector predicates. Since the result is shorter
1473 // than the source, it will correspond to a vector predicate with the
1474 // relevant bits replicated. The replication count is the ratio of the
1475 // source and target vector lengths.
1476 unsigned Rep = VecTy.getVectorNumElements() / ResLen;
1477 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
1478 for (unsigned i = 0; i != HwLen/Rep; ++i) {
1479 for (unsigned j = 0; j != Rep; ++j)
1480 Mask.push_back(Elt: i + Offset);
1481 }
1482 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1483 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: ShuffV);
1484 }
1485
1486 // Converting between a vector predicate and a scalar predicate. In the
1487 // vector predicate, a group of BitBytes bits will correspond to a single
1488 // i1 element of the source vector type. Those bits will all have the same
1489 // value. The same will be true for ByteVec, where each byte corresponds
1490 // to a bit in the vector predicate.
1491 // The algorithm is to traverse the ByteVec, going over the i1 values from
1492 // the source vector, and generate the corresponding representation in an
1493 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
1494 // elements so that the interesting 8 bytes will be in the low end of the
1495 // vector.
1496 unsigned Rep = 8 / ResLen;
1497 // Make sure the output fill the entire vector register, so repeat the
1498 // 8-byte groups as many times as necessary.
1499 for (unsigned r = 0; r != HwLen / 8; ++r) {
1500 // This will generate the indexes of the 8 interesting bytes.
1501 for (unsigned i = 0; i != ResLen; ++i) {
1502 for (unsigned j = 0; j != Rep; ++j)
1503 Mask.push_back(Elt: Offset + i*BitBytes);
1504 }
1505 }
1506
1507 SDValue Zero = getZero(dl, Ty: MVT::i32, DAG);
1508 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1509 // Combine the two low words from ShuffV into a v8i8, and byte-compare
1510 // them against 0.
1511 SDValue W0 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32, Ops: {ShuffV, Zero});
1512 SDValue W1 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1513 Ops: {ShuffV, DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32)});
1514 SDValue Vec64 = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::v8i8, DAG);
1515 return getInstr(MachineOpc: Hexagon::A4_vcmpbgtui, dl, Ty: ResTy,
1516 Ops: {Vec64, DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32)}, DAG);
1517}
1518
1519SDValue
1520HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
1521 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1522 MVT VecTy = ty(Op: VecV);
1523 MVT SubTy = ty(Op: SubV);
1524 unsigned HwLen = Subtarget.getVectorLength();
1525 MVT ElemTy = VecTy.getVectorElementType();
1526 unsigned ElemWidth = ElemTy.getSizeInBits();
1527
1528 bool IsPair = isHvxPairTy(Ty: VecTy);
1529 MVT SingleTy = MVT::getVectorVT(VT: ElemTy, NumElements: (8*HwLen)/ElemWidth);
1530 // The two single vectors that VecV consists of, if it's a pair.
1531 SDValue V0, V1;
1532 SDValue SingleV = VecV;
1533 SDValue PickHi;
1534
1535 if (IsPair) {
1536 V0 = LoHalf(V: VecV, DAG);
1537 V1 = HiHalf(V: VecV, DAG);
1538
1539 SDValue HalfV = DAG.getConstant(Val: SingleTy.getVectorNumElements(),
1540 DL: dl, VT: MVT::i32);
1541 PickHi = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: IdxV, RHS: HalfV, Cond: ISD::SETUGT);
1542 if (isHvxSingleTy(Ty: SubTy)) {
1543 if (const auto *CN = dyn_cast<const ConstantSDNode>(Val: IdxV.getNode())) {
1544 unsigned Idx = CN->getZExtValue();
1545 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
1546 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
1547 return DAG.getTargetInsertSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV, Subreg: SubV);
1548 }
1549 // If IdxV is not a constant, generate the two variants: with the
1550 // SubV as the high and as the low subregister, and select the right
1551 // pair based on the IdxV.
1552 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SubV, V1});
1553 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SubV});
1554 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1555 }
1556 // The subvector being inserted must be entirely contained in one of
1557 // the vectors V0 or V1. Set SingleV to the correct one, and update
1558 // IdxV to be the index relative to the beginning of that vector.
1559 SDValue S = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: IdxV, N2: HalfV);
1560 IdxV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: PickHi, N2: S, N3: IdxV);
1561 SingleV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: SingleTy, N1: PickHi, N2: V1, N3: V0);
1562 }
1563
1564 // The only meaningful subvectors of a single HVX vector are those that
1565 // fit in a scalar register.
1566 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
1567 // Convert IdxV to be index in bytes.
1568 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1569 if (!IdxN || !IdxN->isZero()) {
1570 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1571 N2: DAG.getConstant(Val: ElemWidth/8, DL: dl, VT: MVT::i32));
1572 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: IdxV);
1573 }
1574 // When inserting a single word, the rotation back to the original position
1575 // would be by HwLen-Idx, but if two words are inserted, it will need to be
1576 // by (HwLen-4)-Idx.
1577 unsigned RolBase = HwLen;
1578 if (SubTy.getSizeInBits() == 32) {
1579 SDValue V = DAG.getBitcast(VT: MVT::i32, V: SubV);
1580 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: V);
1581 } else {
1582 SDValue V = DAG.getBitcast(VT: MVT::i64, V: SubV);
1583 SDValue R0 = LoHalf(V, DAG);
1584 SDValue R1 = HiHalf(V, DAG);
1585 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R0);
1586 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV,
1587 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
1588 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R1);
1589 RolBase = HwLen-4;
1590 }
1591 // If the vector wasn't ror'ed, don't ror it back.
1592 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1593 SDValue RolV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1594 N1: DAG.getConstant(Val: RolBase, DL: dl, VT: MVT::i32), N2: IdxV);
1595 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: RolV);
1596 }
1597
1598 if (IsPair) {
1599 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SingleV, V1});
1600 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SingleV});
1601 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1602 }
1603 return SingleV;
1604}
1605
1606SDValue
1607HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1608 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1609 MVT VecTy = ty(Op: VecV);
1610 MVT SubTy = ty(Op: SubV);
1611 assert(Subtarget.isHVXVectorType(VecTy, true));
1612 // VecV is an HVX vector predicate. SubV may be either an HVX vector
1613 // predicate as well, or it can be a scalar predicate.
1614
1615 unsigned VecLen = VecTy.getVectorNumElements();
1616 unsigned HwLen = Subtarget.getVectorLength();
1617 assert(HwLen % VecLen == 0 && "Unexpected vector type");
1618
1619 unsigned Scale = VecLen / SubTy.getVectorNumElements();
1620 unsigned BitBytes = HwLen / VecLen;
1621 unsigned BlockLen = HwLen / Scale;
1622
1623 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1624 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1625 SDValue ByteSub = createHvxPrefixPred(PredV: SubV, dl, BitBytes, ZeroFill: false, DAG);
1626 SDValue ByteIdx;
1627
1628 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1629 if (!IdxN || !IdxN->isZero()) {
1630 ByteIdx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1631 N2: DAG.getConstant(Val: BitBytes, DL: dl, VT: MVT::i32));
1632 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteIdx);
1633 }
1634
1635 // ByteVec is the target vector VecV rotated in such a way that the
1636 // subvector should be inserted at index 0. Generate a predicate mask
1637 // and use vmux to do the insertion.
1638 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1639 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1640 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1641 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1642 ByteVec = getInstr(MachineOpc: Hexagon::V6_vmux, dl, Ty: ByteTy, Ops: {Q, ByteSub, ByteVec}, DAG);
1643 // Rotate ByteVec back, and convert to a vector predicate.
1644 if (!IdxN || !IdxN->isZero()) {
1645 SDValue HwLenV = DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32);
1646 SDValue ByteXdi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: HwLenV, N2: ByteIdx);
1647 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteXdi);
1648 }
1649 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1650}
1651
1652SDValue
1653HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1654 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1655 // Sign- and any-extending of a vector predicate to a vector register is
1656 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1657 // a vector of 1s (where the 1s are of type matching the vector type).
1658 assert(Subtarget.isHVXVectorType(ResTy));
1659 if (!ZeroExt)
1660 return DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ResTy, Operand: VecV);
1661
1662 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1663 SDValue True = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
1664 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1665 SDValue False = getZero(dl, Ty: ResTy, DAG);
1666 return DAG.getSelect(DL: dl, VT: ResTy, Cond: VecV, LHS: True, RHS: False);
1667}
1668
1669SDValue
1670HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1671 MVT ResTy, SelectionDAG &DAG) const {
1672 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1673 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1674 // vector register. The remaining bits of the vector register are
1675 // unspecified.
1676
1677 MachineFunction &MF = DAG.getMachineFunction();
1678 unsigned HwLen = Subtarget.getVectorLength();
1679 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1680 MVT PredTy = ty(Op: VecQ);
1681 unsigned PredLen = PredTy.getVectorNumElements();
1682 assert(HwLen % PredLen == 0);
1683 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8*HwLen/PredLen), NumElements: PredLen);
1684
1685 Type *Int8Ty = Type::getInt8Ty(C&: *DAG.getContext());
1686 SmallVector<Constant*, 128> Tmp;
1687 // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
1688 // These are bytes with the LSB rotated left with respect to their index.
1689 for (unsigned i = 0; i != HwLen/8; ++i) {
1690 for (unsigned j = 0; j != 8; ++j)
1691 Tmp.push_back(Elt: ConstantInt::get(Ty: Int8Ty, V: 1ull << j));
1692 }
1693 Constant *CV = ConstantVector::get(V: Tmp);
1694 Align Alignment(HwLen);
1695 SDValue CP = LowerConstantPool(
1696 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
1697 DAG);
1698 SDValue Bytes =
1699 DAG.getLoad(VT: ByteTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
1700 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
1701
1702 // Select the bytes that correspond to true bits in the vector predicate.
1703 SDValue Sel = DAG.getSelect(DL: dl, VT: VecTy, Cond: VecQ, LHS: DAG.getBitcast(VT: VecTy, V: Bytes),
1704 RHS: getZero(dl, Ty: VecTy, DAG));
1705 // Calculate the OR of all bytes in each group of 8. That will compress
1706 // all the individual bits into a single byte.
1707 // First, OR groups of 4, via vrmpy with 0x01010101.
1708 SDValue All1 =
1709 DAG.getSplatBuildVector(VT: MVT::v4i8, DL: dl, Op: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1710 SDValue Vrmpy = getInstr(MachineOpc: Hexagon::V6_vrmpyub, dl, Ty: ByteTy, Ops: {Sel, All1}, DAG);
1711 // Then rotate the accumulated vector by 4 bytes, and do the final OR.
1712 SDValue Rot = getInstr(MachineOpc: Hexagon::V6_valignbi, dl, Ty: ByteTy,
1713 Ops: {Vrmpy, Vrmpy, DAG.getTargetConstant(Val: 4, DL: dl, VT: MVT::i32)}, DAG);
1714 SDValue Vor = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, Ops: {Vrmpy, Rot});
1715
1716 // Pick every 8th byte and coalesce them at the beginning of the output.
1717 // For symmetry, coalesce every 1+8th byte after that, then every 2+8th
1718 // byte and so on.
1719 SmallVector<int,128> Mask;
1720 for (unsigned i = 0; i != HwLen; ++i)
1721 Mask.push_back(Elt: (8*i) % HwLen + i/(HwLen/8));
1722 SDValue Collect =
1723 DAG.getVectorShuffle(VT: ByteTy, dl, N1: Vor, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1724 return DAG.getBitcast(VT: ResTy, V: Collect);
1725}
1726
1727SDValue
1728HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed,
1729 const SDLoc &dl, SelectionDAG &DAG) const {
1730 // Take a vector and resize the element type to match the given type.
1731 MVT InpTy = ty(Op: VecV);
1732 if (InpTy == ResTy)
1733 return VecV;
1734
1735 unsigned InpWidth = InpTy.getSizeInBits();
1736 unsigned ResWidth = ResTy.getSizeInBits();
1737
1738 if (InpTy.isFloatingPoint()) {
1739 return InpWidth < ResWidth
1740 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: ResTy, Operand: VecV)
1741 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ResTy, N1: VecV,
1742 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
1743 }
1744
1745 assert(InpTy.isInteger());
1746
1747 if (InpWidth < ResWidth) {
1748 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1749 return DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ResTy, Operand: VecV);
1750 } else {
1751 unsigned NarOpc = Signed ? HexagonISD::SSAT : HexagonISD::USAT;
1752 return DAG.getNode(Opcode: NarOpc, DL: dl, VT: ResTy, N1: VecV, N2: DAG.getValueType(ResTy));
1753 }
1754}
1755
1756SDValue
1757HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx,
1758 SelectionDAG &DAG) const {
1759 assert(ty(Vec).getSizeInBits() % SubTy.getSizeInBits() == 0);
1760
1761 const SDLoc &dl(Vec);
1762 unsigned ElemIdx = SubIdx * SubTy.getVectorNumElements();
1763 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubTy,
1764 Ops: {Vec, DAG.getConstant(Val: ElemIdx, DL: dl, VT: MVT::i32)});
1765}
1766
1767SDValue
1768HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1769 const {
1770 const SDLoc &dl(Op);
1771 MVT VecTy = ty(Op);
1772
1773 unsigned Size = Op.getNumOperands();
1774 SmallVector<SDValue,128> Ops;
1775 for (unsigned i = 0; i != Size; ++i)
1776 Ops.push_back(Elt: Op.getOperand(i));
1777
1778 if (VecTy.getVectorElementType() == MVT::i1)
1779 return buildHvxVectorPred(Values: Ops, dl, VecTy, DAG);
1780
1781 // In case of MVT::f16 BUILD_VECTOR, since MVT::f16 is
1782 // not a legal type, just bitcast the node to use i16
1783 // types and bitcast the result back to f16
1784 if (VecTy.getVectorElementType() == MVT::f16 ||
1785 VecTy.getVectorElementType() == MVT::bf16) {
1786 SmallVector<SDValue, 64> NewOps;
1787 for (unsigned i = 0; i != Size; i++)
1788 NewOps.push_back(Elt: DAG.getBitcast(VT: MVT::i16, V: Ops[i]));
1789
1790 SDValue T0 =
1791 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: tyVector(Ty: VecTy, ElemTy: MVT::i16), Ops: NewOps);
1792 return DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: VecTy.getVectorElementType()), V: T0);
1793 }
1794
1795 // First, split the BUILD_VECTOR for vector pairs. We could generate
1796 // some pairs directly (via splat), but splats should be generated
1797 // by the combiner prior to getting here.
1798 if (VecTy.getSizeInBits() == 16 * Subtarget.getVectorLength()) {
1799 ArrayRef<SDValue> A(Ops);
1800 MVT SingleTy = typeSplit(VecTy).first;
1801 SDValue V0 = buildHvxVectorReg(Values: A.take_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1802 SDValue V1 = buildHvxVectorReg(Values: A.drop_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1803 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, N1: V0, N2: V1);
1804 }
1805
1806 return buildHvxVectorReg(Values: Ops, dl, VecTy, DAG);
1807}
1808
1809SDValue
1810HexagonTargetLowering::LowerHvxSplatVector(SDValue Op, SelectionDAG &DAG)
1811 const {
1812 const SDLoc &dl(Op);
1813 MVT VecTy = ty(Op);
1814 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
1815
1816 if (ArgTy == MVT::f16 || ArgTy == MVT::bf16) {
1817 MVT SplatTy = MVT::getVectorVT(VT: MVT::i16, NumElements: VecTy.getVectorNumElements());
1818 SDValue ToInt16 = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: 0));
1819 SDValue ToInt32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: ToInt16);
1820 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: SplatTy, Operand: ToInt32);
1821 return DAG.getBitcast(VT: VecTy, V: Splat);
1822 }
1823
1824 return SDValue();
1825}
1826
1827SDValue
1828HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1829 const {
1830 // Vector concatenation of two integer (non-bool) vectors does not need
1831 // special lowering. Custom-lower concats of bool vectors and expand
1832 // concats of more than 2 vectors.
1833 MVT VecTy = ty(Op);
1834 const SDLoc &dl(Op);
1835 unsigned NumOp = Op.getNumOperands();
1836 if (VecTy.getVectorElementType() != MVT::i1) {
1837 if (NumOp == 2)
1838 return Op;
1839 // Expand the other cases into a build-vector.
1840 SmallVector<SDValue,8> Elems;
1841 for (SDValue V : Op.getNode()->ops())
1842 DAG.ExtractVectorElements(Op: V, Args&: Elems);
1843 // A vector of i16 will be broken up into a build_vector of i16's.
1844 // This is a problem, since at the time of operation legalization,
1845 // all operations are expected to be type-legalized, and i16 is not
1846 // a legal type. If any of the extracted elements is not of a valid
1847 // type, sign-extend it to a valid one.
1848 for (SDValue &V : Elems) {
1849 MVT Ty = ty(Op: V);
1850 if (!isTypeLegal(VT: Ty)) {
1851 MVT NTy = typeLegalize(Ty, DAG);
1852 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1853 V = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NTy,
1854 N1: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NTy,
1855 N1: V.getOperand(i: 0), N2: V.getOperand(i: 1)),
1856 N2: DAG.getValueType(Ty));
1857 continue;
1858 }
1859 // A few less complicated cases.
1860 switch (V.getOpcode()) {
1861 case ISD::Constant:
1862 V = DAG.getSExtOrTrunc(Op: V, DL: dl, VT: NTy);
1863 break;
1864 case ISD::UNDEF:
1865 V = DAG.getUNDEF(VT: NTy);
1866 break;
1867 case ISD::TRUNCATE:
1868 V = V.getOperand(i: 0);
1869 break;
1870 default:
1871 llvm_unreachable("Unexpected vector element");
1872 }
1873 }
1874 }
1875 return DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Elems);
1876 }
1877
1878 assert(VecTy.getVectorElementType() == MVT::i1);
1879 unsigned HwLen = Subtarget.getVectorLength();
1880 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1881
1882 SDValue Op0 = Op.getOperand(i: 0);
1883
1884 // If the operands are HVX types (i.e. not scalar predicates), then
1885 // defer the concatenation, and create QCAT instead.
1886 if (Subtarget.isHVXVectorType(VecTy: ty(Op: Op0), IncludeBool: true)) {
1887 if (NumOp == 2)
1888 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: Op0, N2: Op.getOperand(i: 1));
1889
1890 ArrayRef<SDUse> U(Op.getNode()->ops());
1891 SmallVector<SDValue, 4> SV(U);
1892 ArrayRef<SDValue> Ops(SV);
1893
1894 MVT HalfTy = typeSplit(VecTy).first;
1895 SDValue V0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1896 Ops: Ops.take_front(N: NumOp/2));
1897 SDValue V1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1898 Ops: Ops.take_back(N: NumOp/2));
1899 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: V0, N2: V1);
1900 }
1901
1902 // Count how many bytes (in a vector register) each bit in VecTy
1903 // corresponds to.
1904 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1905
1906 // Make sure that createHvxPrefixPred will only ever need to expand
1907 // the predicate, i.e. bytes-per-bit in the input is not greater than
1908 // the target bytes-per-bit in the result.
1909 SDValue Combined = combineConcatOfScalarPreds(Op, BitBytes, DAG);
1910 SmallVector<SDValue,8> Prefixes;
1911 for (SDValue V : Combined.getNode()->op_values()) {
1912 SDValue P = createHvxPrefixPred(PredV: V, dl, BitBytes, ZeroFill: true, DAG);
1913 Prefixes.push_back(Elt: P);
1914 }
1915
1916 unsigned InpLen = ty(Op: Combined.getOperand(i: 0)).getVectorNumElements();
1917 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1918 SDValue S = DAG.getConstant(Val: HwLen - InpLen*BitBytes, DL: dl, VT: MVT::i32);
1919 SDValue Res = getZero(dl, Ty: ByteTy, DAG);
1920 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1921 Res = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Res, N2: S);
1922 Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, N1: Res, N2: Prefixes[e-i-1]);
1923 }
1924 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: Res);
1925}
1926
1927SDValue
1928HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1929 const {
1930 // Change the type of the extracted element to i32.
1931 SDValue VecV = Op.getOperand(i: 0);
1932 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1933 const SDLoc &dl(Op);
1934 SDValue IdxV = Op.getOperand(i: 1);
1935 if (ElemTy == MVT::i1)
1936 return extractHvxElementPred(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1937
1938 return extractHvxElementReg(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1939}
1940
1941SDValue
1942HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1943 const {
1944 const SDLoc &dl(Op);
1945 MVT VecTy = ty(Op);
1946 SDValue VecV = Op.getOperand(i: 0);
1947 SDValue ValV = Op.getOperand(i: 1);
1948 SDValue IdxV = Op.getOperand(i: 2);
1949 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1950 if (ElemTy == MVT::i1)
1951 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1952
1953 if (ElemTy == MVT::f16 || ElemTy == MVT::bf16) {
1954 SDValue T0 = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
1955 VT: tyVector(Ty: VecTy, ElemTy: MVT::i16),
1956 N1: DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i16), V: VecV),
1957 N2: DAG.getBitcast(VT: MVT::i16, V: ValV), N3: IdxV);
1958 return DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy), V: T0);
1959 }
1960
1961 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1962}
1963
1964SDValue
1965HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1966 const {
1967 SDValue SrcV = Op.getOperand(i: 0);
1968 MVT SrcTy = ty(Op: SrcV);
1969 MVT DstTy = ty(Op);
1970 SDValue IdxV = Op.getOperand(i: 1);
1971 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1972 assert(Idx % DstTy.getVectorNumElements() == 0);
1973 (void)Idx;
1974 const SDLoc &dl(Op);
1975
1976 MVT ElemTy = SrcTy.getVectorElementType();
1977 if (ElemTy == MVT::i1)
1978 return extractHvxSubvectorPred(VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1979
1980 return extractHvxSubvectorReg(OrigOp: Op, VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1981}
1982
1983SDValue
1984HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1985 const {
1986 // Idx does not need to be a constant.
1987 SDValue VecV = Op.getOperand(i: 0);
1988 SDValue ValV = Op.getOperand(i: 1);
1989 SDValue IdxV = Op.getOperand(i: 2);
1990
1991 const SDLoc &dl(Op);
1992 MVT VecTy = ty(Op: VecV);
1993 MVT ElemTy = VecTy.getVectorElementType();
1994 if (ElemTy == MVT::i1)
1995 return insertHvxSubvectorPred(VecV, SubV: ValV, IdxV, dl, DAG);
1996
1997 return insertHvxSubvectorReg(VecV, SubV: ValV, IdxV, dl, DAG);
1998}
1999
2000SDValue
2001HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
2002 // Lower any-extends of boolean vectors to sign-extends, since they
2003 // translate directly to Q2V. Zero-extending could also be done equally
2004 // fast, but Q2V is used/recognized in more places.
2005 // For all other vectors, use zero-extend.
2006 MVT ResTy = ty(Op);
2007 SDValue InpV = Op.getOperand(i: 0);
2008 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2009 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2010 return LowerHvxSignExt(Op, DAG);
2011 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: ResTy, Operand: InpV);
2012}
2013
2014SDValue
2015HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
2016 MVT ResTy = ty(Op);
2017 SDValue InpV = Op.getOperand(i: 0);
2018 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2019 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2020 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: false, DAG);
2021 return Op;
2022}
2023
2024SDValue
2025HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
2026 MVT ResTy = ty(Op);
2027 SDValue InpV = Op.getOperand(i: 0);
2028 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2029 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2030 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: true, DAG);
2031 return Op;
2032}
2033
2034SDValue
2035HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
2036 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
2037 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
2038 const SDLoc &dl(Op);
2039 MVT ResTy = ty(Op);
2040 SDValue InpV = Op.getOperand(i: 0);
2041 assert(ResTy == ty(InpV));
2042
2043 // Calculate the vectors of 1 and bitwidth(x).
2044 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2045 unsigned ElemWidth = ElemTy.getSizeInBits();
2046
2047 SDValue Vec1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2048 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
2049 SDValue VecW = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2050 Operand: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2051 SDValue VecN1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2052 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
2053
2054 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
2055 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
2056 // it separately in custom combine or selection).
2057 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy,
2058 Ops: {DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {InpV, VecN1}),
2059 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {InpV, Vec1})});
2060 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy,
2061 Ops: {VecW, DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: ResTy, Operand: A)});
2062}
2063
2064SDValue
2065HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
2066 const SDLoc &dl(Op);
2067 MVT ResTy = ty(Op);
2068 assert(ResTy.getVectorElementType() == MVT::i32);
2069
2070 SDValue Vs = Op.getOperand(i: 0);
2071 SDValue Vt = Op.getOperand(i: 1);
2072
2073 SDVTList ResTys = DAG.getVTList(VT1: ResTy, VT2: ResTy);
2074 unsigned Opc = Op.getOpcode();
2075
2076 // On HVX v62+ producing the full product is cheap, so legalize MULH to LOHI.
2077 if (Opc == ISD::MULHU)
2078 return DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2079 if (Opc == ISD::MULHS)
2080 return DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2081
2082#ifndef NDEBUG
2083 Op.dump(&DAG);
2084#endif
2085 llvm_unreachable("Unexpected mulh operation");
2086}
2087
2088SDValue
2089HexagonTargetLowering::LowerHvxMulLoHi(SDValue Op, SelectionDAG &DAG) const {
2090 const SDLoc &dl(Op);
2091 unsigned Opc = Op.getOpcode();
2092 SDValue Vu = Op.getOperand(i: 0);
2093 SDValue Vv = Op.getOperand(i: 1);
2094
2095 // If the HI part is not used, convert it to a regular MUL.
2096 if (auto HiVal = Op.getValue(R: 1); HiVal.use_empty()) {
2097 // Need to preserve the types and the number of values.
2098 SDValue Hi = DAG.getUNDEF(VT: ty(Op: HiVal));
2099 SDValue Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: ty(Op), Ops: {Vu, Vv});
2100 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2101 }
2102
2103 bool SignedVu = Opc == HexagonISD::SMUL_LOHI;
2104 bool SignedVv = Opc == HexagonISD::SMUL_LOHI || Opc == HexagonISD::USMUL_LOHI;
2105
2106 // Legal on HVX v62+, but lower it here because patterns can't handle multi-
2107 // valued nodes.
2108 if (Subtarget.useHVXV62Ops())
2109 return emitHvxMulLoHiV62(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2110
2111 if (Opc == HexagonISD::SMUL_LOHI) {
2112 // Direct MULHS expansion is cheaper than doing the whole SMUL_LOHI,
2113 // for other signedness LOHI is cheaper.
2114 if (auto LoVal = Op.getValue(R: 0); LoVal.use_empty()) {
2115 SDValue Hi = emitHvxMulHsV60(A: Vu, B: Vv, dl, DAG);
2116 SDValue Lo = DAG.getUNDEF(VT: ty(Op: LoVal));
2117 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2118 }
2119 }
2120
2121 return emitHvxMulLoHiV60(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2122}
2123
2124SDValue
2125HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
2126 SDValue Val = Op.getOperand(i: 0);
2127 MVT ResTy = ty(Op);
2128 MVT ValTy = ty(Op: Val);
2129 const SDLoc &dl(Op);
2130
2131 if (isHvxBoolTy(Ty: ValTy) && ResTy.isScalarInteger()) {
2132 unsigned HwLen = Subtarget.getVectorLength();
2133 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
2134
2135 // When the predicate is shorter than the predicate register, each boolean
2136 // is represented by multiple consecutive bits in the input register.
2137 // Condense the bits so each boolean is represented by one bit. This only
2138 // handles 2x and 4x compaction ratios.
2139 unsigned PredLen = ValTy.getVectorNumElements();
2140 if (PredLen < HwLen) {
2141 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2142 Val = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Val);
2143 if (HwLen > PredLen * 2) {
2144 assert(HwLen == PredLen * 4);
2145 PredLen *= 2;
2146 Val = getInstr(MachineOpc: Hexagon::V6_vdealh, dl, Ty: ByteTy, Ops: Val, DAG);
2147 }
2148 if (HwLen > PredLen) {
2149 assert(HwLen == PredLen * 2);
2150 Val = getInstr(MachineOpc: Hexagon::V6_vdealb, dl, Ty: ByteTy, Ops: Val, DAG);
2151 }
2152 Val = DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ValTy, Operand: Val);
2153 }
2154
2155 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2156 unsigned BitWidth = ResTy.getSizeInBits();
2157
2158 if (BitWidth < 64) {
2159 SDValue W0 = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
2160 dl, ResTy: MVT::i32, DAG);
2161 if (BitWidth == 32)
2162 return W0;
2163 assert(BitWidth < 32u);
2164 return DAG.getZExtOrTrunc(Op: W0, DL: dl, VT: ResTy);
2165 }
2166
2167 // The result is >= 64 bits. The only options are 64 or 128.
2168 assert(BitWidth == 64 || BitWidth == 128);
2169 SmallVector<SDValue,4> Words;
2170 for (unsigned i = 0; i != BitWidth/32; ++i) {
2171 SDValue W = extractHvxElementReg(
2172 VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl, ResTy: MVT::i32, DAG);
2173 Words.push_back(Elt: W);
2174 }
2175 SmallVector<SDValue,2> Combines;
2176 assert(Words.size() % 2 == 0);
2177 for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
2178 SDValue C = getCombine(Hi: Words[i+1], Lo: Words[i], dl, ResTy: MVT::i64, DAG);
2179 Combines.push_back(Elt: C);
2180 }
2181
2182 if (BitWidth == 64)
2183 return Combines[0];
2184
2185 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: ResTy, Ops: Combines);
2186 }
2187
2188 // Handle bitcast from i32, v2i16, and v4i8 to v32i1.
2189 // Splat the input into a 32-element i32 vector, then AND each element
2190 // with a unique bitmask to isolate individual bits.
2191 auto bitcastI32ToV32I1 = [&](SDValue Val32) {
2192 assert(Val32.getValueType().getSizeInBits() == 32 &&
2193 "Input must be 32 bits");
2194 MVT VecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: 32);
2195 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Val32);
2196 SmallVector<SDValue, 32> Mask;
2197 for (unsigned i = 0; i < 32; ++i)
2198 Mask.push_back(Elt: DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32));
2199
2200 SDValue MaskVec = DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Mask);
2201 SDValue Anded = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: VecTy, N1: Splat, N2: MaskVec);
2202 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: MVT::v32i1, Operand: Anded);
2203 };
2204 // === Case: v32i1 ===
2205 if (ResTy == MVT::v32i1 &&
2206 (ValTy == MVT::i32 || ValTy == MVT::v2i16 || ValTy == MVT::v4i8) &&
2207 Subtarget.useHVX128BOps()) {
2208 SDValue Val32 = Val;
2209 if (ValTy == MVT::v2i16 || ValTy == MVT::v4i8)
2210 Val32 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Val);
2211 return bitcastI32ToV32I1(Val32);
2212 }
2213 // === Case: v64i1 ===
2214 if (ResTy == MVT::v64i1 && ValTy == MVT::i64 && Subtarget.useHVX128BOps()) {
2215 // Split i64 into lo/hi 32-bit halves.
2216 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Val);
2217 SDValue HiShifted = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Val,
2218 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64));
2219 SDValue Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: HiShifted);
2220
2221 // Reuse the same 32-bit logic twice.
2222 SDValue LoRes = bitcastI32ToV32I1(Lo);
2223 SDValue HiRes = bitcastI32ToV32I1(Hi);
2224
2225 // Concatenate into a v64i1 predicate.
2226 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v64i1, N1: LoRes, N2: HiRes);
2227 }
2228
2229 if (isHvxBoolTy(Ty: ResTy) && ValTy.isScalarInteger()) {
2230 // Handle bitcast from i128 -> v128i1 and i64 -> v64i1.
2231 unsigned BitWidth = ValTy.getSizeInBits();
2232 unsigned HwLen = Subtarget.getVectorLength();
2233 assert(BitWidth == HwLen);
2234
2235 MVT ValAsVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: BitWidth / 8);
2236 SDValue ValAsVec = DAG.getBitcast(VT: ValAsVecTy, V: Val);
2237 // Splat each byte of Val 8 times.
2238 // Bytes = [(b0)x8, (b1)x8, ...., (b15)x8]
2239 // where b0, b1,..., b15 are least to most significant bytes of I.
2240 SmallVector<SDValue, 128> Bytes;
2241 // Tmp: 0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80, 0x01,0x02,0x04,0x08,...
2242 // These are bytes with the LSB rotated left with respect to their index.
2243 SmallVector<SDValue, 128> Tmp;
2244 for (unsigned I = 0; I != HwLen / 8; ++I) {
2245 SDValue Idx = DAG.getConstant(Val: I, DL: dl, VT: MVT::i32);
2246 SDValue Byte =
2247 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i8, N1: ValAsVec, N2: Idx);
2248 for (unsigned J = 0; J != 8; ++J) {
2249 Bytes.push_back(Elt: Byte);
2250 Tmp.push_back(Elt: DAG.getConstant(Val: 1ull << J, DL: dl, VT: MVT::i8));
2251 }
2252 }
2253
2254 MVT ConstantVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2255 SDValue ConstantVec = DAG.getBuildVector(VT: ConstantVecTy, DL: dl, Ops: Tmp);
2256 SDValue I2V = buildHvxVectorReg(Values: Bytes, dl, VecTy: ConstantVecTy, DAG);
2257
2258 // Each Byte in the I2V will be set iff corresponding bit is set in Val.
2259 I2V = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ConstantVecTy, Ops: {I2V, ConstantVec});
2260 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: I2V);
2261 }
2262
2263 return Op;
2264}
2265
2266SDValue HexagonTargetLowering::LowerHvxStore(SDValue Op,
2267 SelectionDAG &DAG) const {
2268 const SDLoc &dl(Op);
2269 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
2270 SDValue Val = SN->getValue();
2271 MVT ValTy = ty(Op: Val);
2272
2273 // Check if this is a store of an HVX bool vector (predicate)
2274 if (!isHvxBoolTy(Ty: ValTy))
2275 return SDValue();
2276
2277 unsigned NumBits = ValTy.getVectorNumElements();
2278 MachineMemOperand *MMO = SN->getMemOperand();
2279
2280 // Check alignment requirements based on predicate size
2281 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2282 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2283 return SDValue();
2284
2285 unsigned HwLen = Subtarget.getVectorLength();
2286 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
2287
2288 // Compress the predicate into a vector register
2289 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2290
2291 // Extract words from the compressed vector
2292 SmallVector<SDValue, 4> Words;
2293 for (unsigned i = 0; i != NumBits / 32; ++i) {
2294 SDValue W = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl,
2295 ResTy: MVT::i32, DAG);
2296 Words.push_back(Elt: W);
2297 }
2298
2299 SDValue Chain = SN->getChain();
2300 SDValue BasePtr = SN->getBasePtr();
2301 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2302
2303 if (NumBits == 32)
2304 return DAG.getStore(Chain, dl, Val: Words[0], Ptr: BasePtr, PtrInfo,
2305 Alignment: MMO->getBaseAlign());
2306
2307 if (NumBits == 64) {
2308 SDValue W64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2309 return DAG.getStore(Chain, dl, Val: W64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2310 }
2311
2312 if (NumBits == 128) {
2313 SDValue Lo64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2314 SDValue Hi64 = getCombine(Hi: Words[3], Lo: Words[2], dl, ResTy: MVT::i64, DAG);
2315
2316 Chain =
2317 DAG.getStore(Chain, dl, Val: Lo64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2318
2319 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2320 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2321 return DAG.getStore(Chain, dl, Val: Hi64, Ptr: Ptr8, PtrInfo: PtrInfo.getWithOffset(O: 8),
2322 Alignment: Align(8));
2323 }
2324
2325 return SDValue();
2326}
2327
2328SDValue HexagonTargetLowering::LowerHvxLoad(SDValue Op,
2329 SelectionDAG &DAG) const {
2330 const SDLoc &dl(Op);
2331 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
2332 MVT ResTy = ty(Op);
2333
2334 // Check if this is a load of an HVX bool vector (predicate)
2335 if (!isHvxBoolTy(Ty: ResTy))
2336 return SDValue();
2337
2338 unsigned NumBits = ResTy.getVectorNumElements();
2339 MachineMemOperand *MMO = LN->getMemOperand();
2340
2341 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2342 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2343 return SDValue();
2344
2345 SDValue Chain = LN->getChain();
2346 SDValue BasePtr = LN->getBasePtr();
2347 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2348
2349 if (NumBits == 32) {
2350 SDValue W32 =
2351 DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2352 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v32i1, Operand: W32);
2353 SDValue Ops[] = {Pred, W32.getValue(R: 1)};
2354 return DAG.getMergeValues(Ops, dl);
2355 }
2356
2357 if (NumBits == 64) {
2358 SDValue W64 =
2359 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2360 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: W64);
2361 SDValue Ops[] = {Pred, W64.getValue(R: 1)};
2362 return DAG.getMergeValues(Ops, dl);
2363 }
2364
2365 if (NumBits == 128) {
2366 SDValue Lo64 =
2367 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2368 Chain = Lo64.getValue(R: 1);
2369
2370 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2371 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2372 SDValue Hi64 = DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: Ptr8,
2373 PtrInfo: PtrInfo.getWithOffset(O: 8), Alignment: Align(8));
2374
2375 SDValue LoPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Lo64);
2376 SDValue HiPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Hi64);
2377 SDValue Pred =
2378 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v128i1, N1: LoPred, N2: HiPred);
2379
2380 SDValue Ops[] = {Pred, Hi64.getValue(R: 1)};
2381 return DAG.getMergeValues(Ops, dl);
2382 }
2383
2384 return SDValue();
2385}
2386
2387SDValue
2388HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
2389 // Sign- and zero-extends are legal.
2390 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
2391 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL: SDLoc(Op), VT: ty(Op),
2392 Operand: Op.getOperand(i: 0));
2393}
2394
2395SDValue
2396HexagonTargetLowering::LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const {
2397 MVT ResTy = ty(Op);
2398 if (ResTy.getVectorElementType() != MVT::i1)
2399 return Op;
2400
2401 const SDLoc &dl(Op);
2402 unsigned HwLen = Subtarget.getVectorLength();
2403 unsigned VecLen = ResTy.getVectorNumElements();
2404 assert(HwLen % VecLen == 0);
2405 unsigned ElemSize = HwLen / VecLen;
2406
2407 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemSize * 8), NumElements: VecLen);
2408 SDValue S =
2409 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: Op.getOperand(i: 0),
2410 N2: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 1)),
2411 N3: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 2)));
2412 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: S);
2413}
2414
2415SDValue
2416HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
2417 if (SDValue S = getVectorShiftByInt(Op, DAG))
2418 return S;
2419 return Op;
2420}
2421
2422SDValue
2423HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op,
2424 SelectionDAG &DAG) const {
2425 unsigned Opc = Op.getOpcode();
2426 assert(Opc == ISD::FSHL || Opc == ISD::FSHR);
2427
2428 // Make sure the shift amount is within the range of the bitwidth
2429 // of the element type.
2430 SDValue A = Op.getOperand(i: 0);
2431 SDValue B = Op.getOperand(i: 1);
2432 SDValue S = Op.getOperand(i: 2);
2433
2434 MVT InpTy = ty(Op: A);
2435 MVT ElemTy = InpTy.getVectorElementType();
2436
2437 const SDLoc &dl(Op);
2438 unsigned ElemWidth = ElemTy.getSizeInBits();
2439 bool IsLeft = Opc == ISD::FSHL;
2440
2441 // The expansion into regular shifts produces worse code for i8 and for
2442 // right shift of i32 on v65+.
2443 bool UseShifts = ElemTy != MVT::i8;
2444 if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32)
2445 UseShifts = false;
2446
2447 if (SDValue SplatV = getSplatValue(Op: S, DAG); SplatV && UseShifts) {
2448 // If this is a funnel shift by a scalar, lower it into regular shifts.
2449 SDValue Mask = DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: MVT::i32);
2450 SDValue ModS =
2451 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
2452 Ops: {DAG.getZExtOrTrunc(Op: SplatV, DL: dl, VT: MVT::i32), Mask});
2453 SDValue NegS =
2454 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2455 Ops: {DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32), ModS});
2456 SDValue IsZero =
2457 DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ModS, RHS: getZero(dl, Ty: MVT::i32, DAG), Cond: ISD::SETEQ);
2458 // FSHL A, B => A << | B >>n
2459 // FSHR A, B => A <<n | B >>
2460 SDValue Part1 =
2461 DAG.getNode(Opcode: HexagonISD::VASL, DL: dl, VT: InpTy, Ops: {A, IsLeft ? ModS : NegS});
2462 SDValue Part2 =
2463 DAG.getNode(Opcode: HexagonISD::VLSR, DL: dl, VT: InpTy, Ops: {B, IsLeft ? NegS : ModS});
2464 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Part1, Part2});
2465 // If the shift amount was 0, pick A or B, depending on the direction.
2466 // The opposite shift will also be by 0, so the "Or" will be incorrect.
2467 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InpTy, Ops: {IsZero, (IsLeft ? A : B), Or});
2468 }
2469
2470 SDValue Mask = DAG.getSplatBuildVector(
2471 VT: InpTy, DL: dl, Op: DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: ElemTy));
2472
2473 unsigned MOpc = Opc == ISD::FSHL ? HexagonISD::MFSHL : HexagonISD::MFSHR;
2474 return DAG.getNode(Opcode: MOpc, DL: dl, VT: ty(Op),
2475 Ops: {A, B, DAG.getNode(Opcode: ISD::AND, DL: dl, VT: InpTy, Ops: {S, Mask})});
2476}
2477
2478SDValue
2479HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
2480 const SDLoc &dl(Op);
2481 unsigned IntNo = Op.getConstantOperandVal(i: 0);
2482 SmallVector<SDValue> Ops(Op->ops());
2483
2484 auto Swap = [&](SDValue P) {
2485 return DAG.getMergeValues(Ops: {P.getValue(R: 1), P.getValue(R: 0)}, dl);
2486 };
2487
2488 switch (IntNo) {
2489 case Intrinsic::hexagon_V6_pred_typecast:
2490 case Intrinsic::hexagon_V6_pred_typecast_128B: {
2491 MVT ResTy = ty(Op), InpTy = ty(Op: Ops[1]);
2492 if (isHvxBoolTy(Ty: ResTy) && isHvxBoolTy(Ty: InpTy)) {
2493 if (ResTy == InpTy)
2494 return Ops[1];
2495 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: ResTy, Operand: Ops[1]);
2496 }
2497 break;
2498 }
2499 case Intrinsic::hexagon_V6_vmpyss_parts:
2500 case Intrinsic::hexagon_V6_vmpyss_parts_128B:
2501 return Swap(DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2502 Ops: {Ops[1], Ops[2]}));
2503 case Intrinsic::hexagon_V6_vmpyuu_parts:
2504 case Intrinsic::hexagon_V6_vmpyuu_parts_128B:
2505 return Swap(DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2506 Ops: {Ops[1], Ops[2]}));
2507 case Intrinsic::hexagon_V6_vmpyus_parts:
2508 case Intrinsic::hexagon_V6_vmpyus_parts_128B: {
2509 return Swap(DAG.getNode(Opcode: HexagonISD::USMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2510 Ops: {Ops[1], Ops[2]}));
2511 }
2512 } // switch
2513
2514 return Op;
2515}
2516
2517SDValue
2518HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const {
2519 const SDLoc &dl(Op);
2520 unsigned HwLen = Subtarget.getVectorLength();
2521 MachineFunction &MF = DAG.getMachineFunction();
2522 auto *MaskN = cast<MaskedLoadStoreSDNode>(Val: Op.getNode());
2523 SDValue Mask = MaskN->getMask();
2524 SDValue Chain = MaskN->getChain();
2525 SDValue Base = MaskN->getBasePtr();
2526 auto *MemOp = MF.getMachineMemOperand(MMO: MaskN->getMemOperand(), Offset: 0, Size: HwLen);
2527
2528 unsigned Opc = Op->getOpcode();
2529 assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE);
2530
2531 if (Opc == ISD::MLOAD) {
2532 MVT ValTy = ty(Op);
2533 SDValue Load = DAG.getLoad(VT: ValTy, dl, Chain, Ptr: Base, MMO: MemOp);
2534 SDValue Thru = cast<MaskedLoadSDNode>(Val: MaskN)->getPassThru();
2535 if (isUndef(Op: Thru))
2536 return Load;
2537 SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: Mask, N2: Load, N3: Thru);
2538 return DAG.getMergeValues(Ops: {VSel, Load.getValue(R: 1)}, dl);
2539 }
2540
2541 // MSTORE
2542 // HVX only has aligned masked stores.
2543
2544 // TODO: Fold negations of the mask into the store.
2545 unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
2546 SDValue Value = cast<MaskedStoreSDNode>(Val: MaskN)->getValue();
2547 SDValue Offset0 = DAG.getTargetConstant(Val: 0, DL: dl, VT: ty(Op: Base));
2548
2549 if (MaskN->getAlign().value() % HwLen == 0) {
2550 SDValue Store = getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2551 Ops: {Mask, Base, Offset0, Value, Chain}, DAG);
2552 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Store.getNode()), NewMemRefs: {MemOp});
2553 return Store;
2554 }
2555
2556 // Unaligned case.
2557 auto StoreAlign = [&](SDValue V, SDValue A) {
2558 SDValue Z = getZero(dl, Ty: ty(Op: V), DAG);
2559 // TODO: use funnel shifts?
2560 // vlalign(Vu,Vv,Rt) rotates the pair Vu:Vv left by Rt and takes the
2561 // upper half.
2562 SDValue LoV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {V, Z, A}, DAG);
2563 SDValue HiV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {Z, V, A}, DAG);
2564 return std::make_pair(x&: LoV, y&: HiV);
2565 };
2566
2567 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2568 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
2569 SDValue MaskV = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Mask);
2570 VectorPair Tmp = StoreAlign(MaskV, Base);
2571 VectorPair MaskU = {DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.first),
2572 DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.second)};
2573 VectorPair ValueU = StoreAlign(Value, Base);
2574
2575 SDValue Offset1 = DAG.getTargetConstant(Val: HwLen, DL: dl, VT: MVT::i32);
2576 SDValue StoreLo =
2577 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2578 Ops: {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
2579 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreLo.getNode()), NewMemRefs: {MemOp});
2580
2581 // If the store fits within one HwLen-aligned block, the high half's predicate
2582 // is always all-zeros and the vmem(Base+HwLen) can be elided entirely.
2583 // Proof: addr % StoreAlign == 0 and StoreMemSize <= StoreAlign implies
2584 // addr % HwLen <= HwLen - StoreAlign, so addr % HwLen + StoreMemSize
2585 // <= HwLen.
2586 // Without this guard, Hexagon v73+ probes the TLB for vmem(Base+HwLen) even
2587 // when the predicate is all-zeros, causing a TLBMISS if that page is
2588 // unmapped.
2589 uint64_t StoreMemSize = MaskN->getMemoryVT().getStoreSize().getFixedValue();
2590 if (StoreMemSize <= MaskN->getAlign().value())
2591 return StoreLo;
2592
2593 SDValue StoreHi =
2594 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2595 Ops: {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
2596 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreHi.getNode()), NewMemRefs: {MemOp});
2597 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: {StoreLo, StoreHi});
2598}
2599
2600SDValue HexagonTargetLowering::LowerHvxFpExtend(SDValue Op,
2601 SelectionDAG &DAG) const {
2602 // This conversion only applies to QFloat. IEEE extension from f16 to f32
2603 // is legal (done via a pattern).
2604 assert(Subtarget.useHVXQFloatOps());
2605
2606 assert(Op->getOpcode() == ISD::FP_EXTEND);
2607
2608 MVT VecTy = ty(Op);
2609 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
2610 const SDLoc &dl(Op);
2611
2612 if (ArgTy == MVT::v64bf16) {
2613 MVT HalfTy = typeSplit(VecTy).first;
2614 SDValue BF16Vec = Op.getOperand(i: 0);
2615 SDValue Zeroes =
2616 getInstr(MachineOpc: Hexagon::V6_vxor, dl, Ty: HalfTy, Ops: {BF16Vec, BF16Vec}, DAG);
2617 // Interleave zero vector with the bf16 vector, with zeroes in the lower
2618 // half of each 32 bit lane, effectively extending the bf16 values to fp32
2619 // values.
2620 SDValue ShuffVec =
2621 getInstr(MachineOpc: Hexagon::V6_vshufoeh, dl, Ty: VecTy, Ops: {BF16Vec, Zeroes}, DAG);
2622 VectorPair VecPair = opSplit(Vec: ShuffVec, dl, DAG);
2623 SDValue Result = getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2624 Ops: {VecPair.second, VecPair.first,
2625 DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)},
2626 DAG);
2627 return Result;
2628 }
2629
2630 assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16);
2631
2632 SDValue F16Vec = Op.getOperand(i: 0);
2633
2634 APFloat FloatVal = APFloat(1.0f);
2635 bool Ignored;
2636 FloatVal.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
2637 SDValue Fp16Ones = DAG.getConstantFP(Val: FloatVal, DL: dl, VT: ArgTy);
2638 SDValue VmpyVec =
2639 getInstr(MachineOpc: Hexagon::V6_vmpy_qf32_hf, dl, Ty: VecTy, Ops: {F16Vec, Fp16Ones}, DAG);
2640
2641 MVT HalfTy = typeSplit(VecTy).first;
2642 VectorPair Pair = opSplit(Vec: VmpyVec, dl, DAG);
2643 SDValue LoVec =
2644 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.first}, DAG);
2645 SDValue HiVec =
2646 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.second}, DAG);
2647
2648 SDValue ShuffVec =
2649 getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2650 Ops: {HiVec, LoVec, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)}, DAG);
2651
2652 return ShuffVec;
2653}
2654
2655SDValue
2656HexagonTargetLowering::LowerHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
2657 // Catch invalid conversion ops (just in case).
2658 assert(Op.getOpcode() == ISD::FP_TO_SINT ||
2659 Op.getOpcode() == ISD::FP_TO_UINT);
2660
2661 MVT ResTy = ty(Op);
2662 MVT FpTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2663 MVT IntTy = ResTy.getVectorElementType();
2664
2665 if (Subtarget.useHVXIEEEFPOps()) {
2666 // There are only conversions from f16.
2667 if (FpTy == MVT::f16) {
2668 // Other int types aren't legal in HVX, so we shouldn't see them here.
2669 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2670 // Conversions to i8 and i16 are legal.
2671 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2672 return Op;
2673 }
2674 }
2675
2676 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2677 return EqualizeFpIntConversion(Op, DAG);
2678
2679 return ExpandHvxFpToInt(Op, DAG);
2680}
2681
2682// For vector type v32i1 uint_to_fp/sint_to_fp to v32f32:
2683// R1 = #1, R2 holds the v32i1 param
2684// V1 = vsplat(R1)
2685// V2 = vsplat(R2)
2686// Q0 = vand(V1,R1)
2687// V0.w=prefixsum(Q0)
2688// V0.w=vsub(V0.w,V1.w)
2689// V2.w = vlsr(V2.w,V0.w)
2690// V2 = vand(V2,V1)
2691// V2.sf = V2.w
2692SDValue HexagonTargetLowering::LowerHvxPred32ToFp(SDValue PredOp,
2693 SelectionDAG &DAG) const {
2694
2695 MVT ResTy = ty(Op: PredOp);
2696 const SDLoc &dl(PredOp);
2697
2698 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2699 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2700 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2701 Op1: SDValue(RegConst, 0));
2702 SDNode *PredTransfer =
2703 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2704 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2705 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2706 Op1: SDValue(PredTransfer, 0));
2707 SDNode *SplatParam = DAG.getMachineNode(
2708 Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2709 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: PredOp.getOperand(i: 0)));
2710 SDNode *Vsub =
2711 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2712 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2713 SDNode *IndexShift =
2714 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2715 Op1: SDValue(SplatParam, 0), Op2: SDValue(Vsub, 0));
2716 SDNode *MaskOff =
2717 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2718 Op1: SDValue(IndexShift, 0), Op2: SDValue(SplatConst, 0));
2719 SDNode *Convert = DAG.getMachineNode(Opcode: Hexagon::V6_vconv_sf_w, dl, VT: ResTy,
2720 Op1: SDValue(MaskOff, 0));
2721 return SDValue(Convert, 0);
2722}
2723
2724// For vector type v64i1 uint_to_fo to v64f16:
2725// i64 R32 = bitcast v64i1 R3:2 (R3:2 holds v64i1)
2726// R3 = subreg_high (R32)
2727// R2 = subreg_low (R32)
2728// R1 = #1
2729// V1 = vsplat(R1)
2730// V2 = vsplat(R2)
2731// V3 = vsplat(R3)
2732// Q0 = vand(V1,R1)
2733// V0.w=prefixsum(Q0)
2734// V0.w=vsub(V0.w,V1.w)
2735// V2.w = vlsr(V2.w,V0.w)
2736// V3.w = vlsr(V3.w,V0.w)
2737// V2 = vand(V2,V1)
2738// V3 = vand(V3,V1)
2739// V2.h = vpacke(V3.w,V2.w)
2740// V2.hf = V2.h
2741SDValue HexagonTargetLowering::LowerHvxPred64ToFp(SDValue PredOp,
2742 SelectionDAG &DAG) const {
2743
2744 MVT ResTy = ty(Op: PredOp);
2745 const SDLoc &dl(PredOp);
2746
2747 SDValue Inp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: PredOp.getOperand(i: 0));
2748 // Get the hi and lo regs
2749 SDValue HiReg =
2750 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_hi, DL: dl, VT: MVT::i32, Operand: Inp);
2751 SDValue LoReg =
2752 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_lo, DL: dl, VT: MVT::i32, Operand: Inp);
2753 // Get constant #1 and splat into vector V1
2754 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2755 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2756 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2757 Op1: SDValue(RegConst, 0));
2758 // Splat the hi and lo args
2759 SDNode *SplatHi =
2760 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2761 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: HiReg));
2762 SDNode *SplatLo =
2763 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2764 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: LoReg));
2765 // vand between splatted const and const
2766 SDNode *PredTransfer =
2767 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2768 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2769 // Get the prefixsum
2770 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2771 Op1: SDValue(PredTransfer, 0));
2772 // Get the vsub
2773 SDNode *Vsub =
2774 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2775 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2776 // Get vlsr for hi and lo
2777 SDNode *IndexShift_hi =
2778 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2779 Op1: SDValue(SplatHi, 0), Op2: SDValue(Vsub, 0));
2780 SDNode *IndexShift_lo =
2781 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2782 Op1: SDValue(SplatLo, 0), Op2: SDValue(Vsub, 0));
2783 // Get vand of hi and lo
2784 SDNode *MaskOff_hi =
2785 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2786 Op1: SDValue(IndexShift_hi, 0), Op2: SDValue(SplatConst, 0));
2787 SDNode *MaskOff_lo =
2788 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2789 Op1: SDValue(IndexShift_lo, 0), Op2: SDValue(SplatConst, 0));
2790 // Pack them
2791 SDNode *Pack =
2792 DAG.getMachineNode(Opcode: Hexagon::V6_vpackeh, dl, VT: MVT::v64i16,
2793 Op1: SDValue(MaskOff_hi, 0), Op2: SDValue(MaskOff_lo, 0));
2794 SDNode *Convert =
2795 DAG.getMachineNode(Opcode: Hexagon::V6_vconv_hf_h, dl, VT: ResTy, Op1: SDValue(Pack, 0));
2796 return SDValue(Convert, 0);
2797}
2798
2799SDValue
2800HexagonTargetLowering::LowerHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
2801 // Catch invalid conversion ops (just in case).
2802 assert(Op.getOpcode() == ISD::SINT_TO_FP ||
2803 Op.getOpcode() == ISD::UINT_TO_FP);
2804
2805 MVT ResTy = ty(Op);
2806 MVT IntTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2807 MVT FpTy = ResTy.getVectorElementType();
2808
2809 if (Op.getOpcode() == ISD::UINT_TO_FP || Op.getOpcode() == ISD::SINT_TO_FP) {
2810 if (ResTy == MVT::v32f32 && ty(Op: Op.getOperand(i: 0)) == MVT::v32i1)
2811 return LowerHvxPred32ToFp(PredOp: Op, DAG);
2812 if (ResTy == MVT::v64f16 && ty(Op: Op.getOperand(i: 0)) == MVT::v64i1)
2813 return LowerHvxPred64ToFp(PredOp: Op, DAG);
2814 }
2815
2816 if (Subtarget.useHVXIEEEFPOps()) {
2817 // There are only conversions to f16.
2818 if (FpTy == MVT::f16) {
2819 // Other int types aren't legal in HVX, so we shouldn't see them here.
2820 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2821 // i8, i16 -> f16 is legal.
2822 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2823 return Op;
2824 }
2825 }
2826
2827 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2828 return EqualizeFpIntConversion(Op, DAG);
2829
2830 return ExpandHvxIntToFp(Op, DAG);
2831}
2832
2833HexagonTargetLowering::TypePair
2834HexagonTargetLowering::typeExtendToWider(MVT Ty0, MVT Ty1) const {
2835 // Compare the widths of elements of the two types, and extend the narrower
2836 // type to match the with of the wider type. For vector types, apply this
2837 // to the element type.
2838 assert(Ty0.isVector() == Ty1.isVector());
2839
2840 MVT ElemTy0 = Ty0.getScalarType();
2841 MVT ElemTy1 = Ty1.getScalarType();
2842
2843 unsigned Width0 = ElemTy0.getSizeInBits();
2844 unsigned Width1 = ElemTy1.getSizeInBits();
2845 unsigned MaxWidth = std::max(a: Width0, b: Width1);
2846
2847 auto getScalarWithWidth = [](MVT ScalarTy, unsigned Width) {
2848 if (ScalarTy.isInteger())
2849 return MVT::getIntegerVT(BitWidth: Width);
2850 assert(ScalarTy.isFloatingPoint());
2851 return MVT::getFloatingPointVT(BitWidth: Width);
2852 };
2853
2854 MVT WideETy0 = getScalarWithWidth(ElemTy0, MaxWidth);
2855 MVT WideETy1 = getScalarWithWidth(ElemTy1, MaxWidth);
2856
2857 if (!Ty0.isVector()) {
2858 // Both types are scalars.
2859 return {WideETy0, WideETy1};
2860 }
2861
2862 // Vector types.
2863 unsigned NumElem = Ty0.getVectorNumElements();
2864 assert(NumElem == Ty1.getVectorNumElements());
2865
2866 return {MVT::getVectorVT(VT: WideETy0, NumElements: NumElem),
2867 MVT::getVectorVT(VT: WideETy1, NumElements: NumElem)};
2868}
2869
2870HexagonTargetLowering::TypePair
2871HexagonTargetLowering::typeWidenToWider(MVT Ty0, MVT Ty1) const {
2872 // Compare the numbers of elements of two vector types, and widen the
2873 // narrower one to match the number of elements in the wider one.
2874 assert(Ty0.isVector() && Ty1.isVector());
2875
2876 unsigned Len0 = Ty0.getVectorNumElements();
2877 unsigned Len1 = Ty1.getVectorNumElements();
2878 if (Len0 == Len1)
2879 return {Ty0, Ty1};
2880
2881 unsigned MaxLen = std::max(a: Len0, b: Len1);
2882 return {MVT::getVectorVT(VT: Ty0.getVectorElementType(), NumElements: MaxLen),
2883 MVT::getVectorVT(VT: Ty1.getVectorElementType(), NumElements: MaxLen)};
2884}
2885
2886MVT
2887HexagonTargetLowering::typeLegalize(MVT Ty, SelectionDAG &DAG) const {
2888 EVT LegalTy = getTypeToTransformTo(Context&: *DAG.getContext(), VT: Ty);
2889 assert(LegalTy.isSimple());
2890 return LegalTy.getSimpleVT();
2891}
2892
2893MVT
2894HexagonTargetLowering::typeWidenToHvx(MVT Ty) const {
2895 unsigned HwWidth = 8 * Subtarget.getVectorLength();
2896 assert(Ty.getSizeInBits() <= HwWidth);
2897 if (Ty.getSizeInBits() == HwWidth)
2898 return Ty;
2899
2900 MVT ElemTy = Ty.getScalarType();
2901 return MVT::getVectorVT(VT: ElemTy, NumElements: HwWidth / ElemTy.getSizeInBits());
2902}
2903
2904HexagonTargetLowering::VectorPair
2905HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B,
2906 const SDLoc &dl, bool Signed, SelectionDAG &DAG) const {
2907 // Compute A+B, return {A+B, O}, where O = vector predicate indicating
2908 // whether an overflow has occurred.
2909 MVT ResTy = ty(Op: A);
2910 assert(ResTy == ty(B));
2911 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: ResTy.getVectorNumElements());
2912
2913 if (!Signed) {
2914 // V62+ has V6_vaddcarry, but it requires input predicate, so it doesn't
2915 // save any instructions.
2916 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2917 SDValue Ovf = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Add, RHS: A, Cond: ISD::SETULT);
2918 return {Add, Ovf};
2919 }
2920
2921 // Signed overflow has happened, if:
2922 // (A, B have the same sign) and (A+B has a different sign from either)
2923 // i.e. (~A xor B) & ((A+B) xor B), then check the sign bit
2924 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2925 SDValue NotA =
2926 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {A, DAG.getAllOnesConstant(DL: dl, VT: ResTy)});
2927 SDValue Xor0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {NotA, B});
2928 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {Add, B});
2929 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy, Ops: {Xor0, Xor1});
2930 SDValue MSB =
2931 DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: getZero(dl, Ty: ResTy, DAG), Cond: ISD::SETLT);
2932 return {Add, MSB};
2933}
2934
2935HexagonTargetLowering::VectorPair
2936HexagonTargetLowering::emitHvxShiftRightRnd(SDValue Val, unsigned Amt,
2937 bool Signed, SelectionDAG &DAG) const {
2938 // Shift Val right by Amt bits, round the result to the nearest integer,
2939 // tie-break by rounding halves to even integer.
2940
2941 const SDLoc &dl(Val);
2942 MVT ValTy = ty(Op: Val);
2943
2944 // This should also work for signed integers.
2945 //
2946 // uint tmp0 = inp + ((1 << (Amt-1)) - 1);
2947 // bool ovf = (inp > tmp0);
2948 // uint rup = inp & (1 << (Amt+1));
2949 //
2950 // uint tmp1 = inp >> (Amt-1); // tmp1 == tmp2 iff
2951 // uint tmp2 = tmp0 >> (Amt-1); // the Amt-1 lower bits were all 0
2952 // uint tmp3 = tmp2 + rup;
2953 // uint frac = (tmp1 != tmp2) ? tmp2 >> 1 : tmp3 >> 1;
2954 unsigned ElemWidth = ValTy.getVectorElementType().getSizeInBits();
2955 MVT ElemTy = MVT::getIntegerVT(BitWidth: ElemWidth);
2956 MVT IntTy = tyVector(Ty: ValTy, ElemTy);
2957 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: IntTy.getVectorNumElements());
2958 unsigned ShRight = Signed ? ISD::SRA : ISD::SRL;
2959
2960 SDValue Inp = DAG.getBitcast(VT: IntTy, V: Val);
2961 SDValue LowBits = DAG.getConstant(Val: (1ull << (Amt - 1)) - 1, DL: dl, VT: IntTy);
2962
2963 SDValue AmtP1 = DAG.getConstant(Val: 1ull << Amt, DL: dl, VT: IntTy);
2964 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntTy, Ops: {Inp, AmtP1});
2965 SDValue Zero = getZero(dl, Ty: IntTy, DAG);
2966 SDValue Bit = DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: Zero, Cond: ISD::SETNE);
2967 SDValue Rup = DAG.getZExtOrTrunc(Op: Bit, DL: dl, VT: IntTy);
2968 auto [Tmp0, Ovf] = emitHvxAddWithOverflow(A: Inp, B: LowBits, dl, Signed, DAG);
2969
2970 SDValue AmtM1 = DAG.getConstant(Val: Amt - 1, DL: dl, VT: IntTy);
2971 SDValue Tmp1 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Inp, N2: AmtM1);
2972 SDValue Tmp2 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Tmp0, N2: AmtM1);
2973 SDValue Tmp3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntTy, N1: Tmp2, N2: Rup);
2974
2975 SDValue Eq = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Tmp1, RHS: Tmp2, Cond: ISD::SETEQ);
2976 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntTy);
2977 SDValue Tmp4 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp2, One});
2978 SDValue Tmp5 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp3, One});
2979 SDValue Mux = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: IntTy, Ops: {Eq, Tmp5, Tmp4});
2980 return {Mux, Ovf};
2981}
2982
2983SDValue
2984HexagonTargetLowering::emitHvxMulHsV60(SDValue A, SDValue B, const SDLoc &dl,
2985 SelectionDAG &DAG) const {
2986 MVT VecTy = ty(Op: A);
2987 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
2988 assert(VecTy.getVectorElementType() == MVT::i32);
2989
2990 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
2991
2992 // mulhs(A,B) =
2993 // = [(Hi(A)*2^16 + Lo(A)) *s (Hi(B)*2^16 + Lo(B))] >> 32
2994 // = [Hi(A)*2^16 *s Hi(B)*2^16 + Hi(A) *su Lo(B)*2^16
2995 // + Lo(A) *us (Hi(B)*2^16 + Lo(B))] >> 32
2996 // = [Hi(A) *s Hi(B)*2^32 + Hi(A) *su Lo(B)*2^16 + Lo(A) *us B] >> 32
2997 // The low half of Lo(A)*Lo(B) will be discarded (it's not added to
2998 // anything, so it cannot produce any carry over to higher bits),
2999 // so everything in [] can be shifted by 16 without loss of precision.
3000 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + Lo(A)*B >> 16] >> 16
3001 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + V6_vmpyewuh(A,B)] >> 16
3002 // The final additions need to make sure to properly maintain any carry-
3003 // out bits.
3004 //
3005 // Hi(B) Lo(B)
3006 // Hi(A) Lo(A)
3007 // --------------
3008 // Lo(B)*Lo(A) | T0 = V6_vmpyewuh(B,A) does this,
3009 // Hi(B)*Lo(A) | + dropping the low 16 bits
3010 // Hi(A)*Lo(B) | T2
3011 // Hi(B)*Hi(A)
3012
3013 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh, dl, Ty: VecTy, Ops: {B, A}, DAG);
3014 // T1 = get Hi(A) into low halves.
3015 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {A, S16}, DAG);
3016 // P0 = interleaved T1.h*B.uh (full precision product)
3017 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyhus, dl, Ty: PairTy, Ops: {T1, B}, DAG);
3018 // T2 = T1.even(h) * B.even(uh), i.e. Hi(A)*Lo(B)
3019 SDValue T2 = LoHalf(V: P0, DAG);
3020 // We need to add T0+T2, recording the carry-out, which will be 1<<16
3021 // added to the final sum.
3022 // P1 = interleaved even/odd 32-bit (unsigned) sums of 16-bit halves
3023 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3024 // P2 = interleaved even/odd 32-bit (signed) sums of 16-bit halves
3025 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vaddhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3026 // T3 = full-precision(T0+T2) >> 16
3027 // The low halves are added-unsigned, the high ones are added-signed.
3028 SDValue T3 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3029 Ops: {HiHalf(V: P2, DAG), LoHalf(V: P1, DAG), S16}, DAG);
3030 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {B, S16}, DAG);
3031 // P3 = interleaved Hi(B)*Hi(A) (full precision),
3032 // which is now Lo(T1)*Lo(T4), so we want to keep the even product.
3033 SDValue P3 = getInstr(MachineOpc: Hexagon::V6_vmpyhv, dl, Ty: PairTy, Ops: {T1, T4}, DAG);
3034 SDValue T5 = LoHalf(V: P3, DAG);
3035 // Add:
3036 SDValue T6 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {T3, T5});
3037 return T6;
3038}
3039
3040SDValue
3041HexagonTargetLowering::emitHvxMulLoHiV60(SDValue A, bool SignedA, SDValue B,
3042 bool SignedB, const SDLoc &dl,
3043 SelectionDAG &DAG) const {
3044 MVT VecTy = ty(Op: A);
3045 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3046 assert(VecTy.getVectorElementType() == MVT::i32);
3047
3048 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
3049
3050 if (SignedA && !SignedB) {
3051 // Make A:unsigned, B:signed.
3052 std::swap(a&: A, b&: B);
3053 std::swap(a&: SignedA, b&: SignedB);
3054 }
3055
3056 // Do halfword-wise multiplications for unsigned*unsigned product, then
3057 // add corrections for signed and unsigned*signed.
3058
3059 SDValue Lo, Hi;
3060
3061 // P0:lo = (uu) products of low halves of A and B,
3062 // P0:hi = (uu) products of high halves.
3063 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, B}, DAG);
3064
3065 // Swap low/high halves in B
3066 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_lvsplatw, dl, Ty: VecTy,
3067 Ops: {DAG.getConstant(Val: 0x02020202, DL: dl, VT: MVT::i32)}, DAG);
3068 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vdelta, dl, Ty: VecTy, Ops: {B, T0}, DAG);
3069 // P1 = products of even/odd halfwords.
3070 // P1:lo = (uu) products of even(A.uh) * odd(B.uh)
3071 // P1:hi = (uu) products of odd(A.uh) * even(B.uh)
3072 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, T1}, DAG);
3073
3074 // P2:lo = low halves of P1:lo + P1:hi,
3075 // P2:hi = high halves of P1:lo + P1:hi.
3076 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy,
3077 Ops: {HiHalf(V: P1, DAG), LoHalf(V: P1, DAG)}, DAG);
3078 // Still need to add the high halves of P0:lo to P2:lo
3079 SDValue T2 =
3080 getInstr(MachineOpc: Hexagon::V6_vlsrw, dl, Ty: VecTy, Ops: {LoHalf(V: P0, DAG), S16}, DAG);
3081 SDValue T3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {LoHalf(V: P2, DAG), T2});
3082
3083 // The high halves of T3 will contribute to the HI part of LOHI.
3084 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3085 Ops: {HiHalf(V: P2, DAG), T3, S16}, DAG);
3086
3087 // The low halves of P2 need to be added to high halves of the LO part.
3088 Lo = getInstr(MachineOpc: Hexagon::V6_vaslw_acc, dl, Ty: VecTy,
3089 Ops: {LoHalf(V: P0, DAG), LoHalf(V: P2, DAG), S16}, DAG);
3090 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {HiHalf(V: P0, DAG), T4});
3091
3092 if (SignedA) {
3093 assert(SignedB && "Signed A and unsigned B should have been inverted");
3094
3095 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3096 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3097 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3098 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3099 SDValue X0 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: VecTy, Ops: {Q0, B, Zero});
3100 SDValue X1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, X0, A}, DAG);
3101 Hi = getInstr(MachineOpc: Hexagon::V6_vsubw, dl, Ty: VecTy, Ops: {Hi, X1}, DAG);
3102 } else if (SignedB) {
3103 // Same correction as for mulhus:
3104 // mulhus(A.uw,B.w) = mulhu(A.uw,B.uw) - (A.w if B < 0)
3105 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3106 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3107 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3108 Hi = getInstr(MachineOpc: Hexagon::V6_vsubwq, dl, Ty: VecTy, Ops: {Q1, Hi, A}, DAG);
3109 } else {
3110 assert(!SignedA && !SignedB);
3111 }
3112
3113 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3114}
3115
3116SDValue
3117HexagonTargetLowering::emitHvxMulLoHiV62(SDValue A, bool SignedA,
3118 SDValue B, bool SignedB,
3119 const SDLoc &dl,
3120 SelectionDAG &DAG) const {
3121 MVT VecTy = ty(Op: A);
3122 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3123 assert(VecTy.getVectorElementType() == MVT::i32);
3124
3125 if (SignedA && !SignedB) {
3126 // Make A:unsigned, B:signed.
3127 std::swap(a&: A, b&: B);
3128 std::swap(a&: SignedA, b&: SignedB);
3129 }
3130
3131 // Do S*S first, then make corrections for U*S or U*U if needed.
3132 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh_64, dl, Ty: PairTy, Ops: {A, B}, DAG);
3133 SDValue P1 =
3134 getInstr(MachineOpc: Hexagon::V6_vmpyowh_64_acc, dl, Ty: PairTy, Ops: {P0, A, B}, DAG);
3135 SDValue Lo = LoHalf(V: P1, DAG);
3136 SDValue Hi = HiHalf(V: P1, DAG);
3137
3138 if (!SignedB) {
3139 assert(!SignedA && "Signed A and unsigned B should have been inverted");
3140 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3141 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3142
3143 // Mulhu(X, Y) = Mulhs(X, Y) + (X, if Y < 0) + (Y, if X < 0).
3144 // def: Pat<(VecI32 (mulhu HVI32:$A, HVI32:$B)),
3145 // (V6_vaddw (HiHalf (Muls64O $A, $B)),
3146 // (V6_vaddwq (V6_vgtw (V6_vd0), $B),
3147 // (V6_vandvqv (V6_vgtw (V6_vd0), $A), $B),
3148 // $A))>;
3149 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3150 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3151 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vandvqv, dl, Ty: VecTy, Ops: {Q0, B}, DAG);
3152 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, T0, A}, DAG);
3153 Hi = getInstr(MachineOpc: Hexagon::V6_vaddw, dl, Ty: VecTy, Ops: {Hi, T1}, DAG);
3154 } else if (!SignedA) {
3155 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3156 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3157
3158 // Mulhus(unsigned X, signed Y) = Mulhs(X, Y) + (Y, if X < 0).
3159 // def: Pat<(VecI32 (HexagonMULHUS HVI32:$A, HVI32:$B)),
3160 // (V6_vaddwq (V6_vgtw (V6_vd0), $A),
3161 // (HiHalf (Muls64O $A, $B)),
3162 // $B)>;
3163 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3164 Hi = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q0, Hi, B}, DAG);
3165 }
3166
3167 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3168}
3169
3170SDValue
3171HexagonTargetLowering::EqualizeFpIntConversion(SDValue Op, SelectionDAG &DAG)
3172 const {
3173 // Rewrite conversion between integer and floating-point in such a way that
3174 // the integer type is extended/narrowed to match the bitwidth of the
3175 // floating-point type, combined with additional integer-integer extensions
3176 // or narrowings to match the original input/result types.
3177 // E.g. f32 -> i8 ==> f32 -> i32 -> i8
3178 //
3179 // The input/result types are not required to be legal, but if they are
3180 // legal, this function should not introduce illegal types.
3181
3182 unsigned Opc = Op.getOpcode();
3183 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT ||
3184 Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3185
3186 SDValue Inp = Op.getOperand(i: 0);
3187 MVT InpTy = ty(Op: Inp);
3188 MVT ResTy = ty(Op);
3189
3190 if (InpTy == ResTy)
3191 return Op;
3192
3193 const SDLoc &dl(Op);
3194 bool Signed = Opc == ISD::FP_TO_SINT || Opc == ISD::SINT_TO_FP;
3195
3196 auto [WInpTy, WResTy] = typeExtendToWider(Ty0: InpTy, Ty1: ResTy);
3197 SDValue WInp = resizeToWidth(VecV: Inp, ResTy: WInpTy, Signed, dl, DAG);
3198 SDValue Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: WResTy, Operand: WInp);
3199 SDValue Res = resizeToWidth(VecV: Conv, ResTy, Signed, dl, DAG);
3200 return Res;
3201}
3202
3203SDValue
3204HexagonTargetLowering::ExpandHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
3205 unsigned Opc = Op.getOpcode();
3206 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT);
3207
3208 const SDLoc &dl(Op);
3209 SDValue Op0 = Op.getOperand(i: 0);
3210 MVT InpTy = ty(Op: Op0);
3211 MVT ResTy = ty(Op);
3212 assert(InpTy.changeTypeToInteger() == ResTy);
3213
3214 // At this point this is an experiment under a flag.
3215 // In arch before V81 the rounding mode is towards nearest value.
3216 // The C/C++ standard requires rounding towards zero:
3217 // C (C99 and later): ISO/IEC 9899:2018 (C18), section 6.3.1.4 — "When a
3218 // finite value of real floating type is converted to an integer type, the
3219 // fractional part is discarded (i.e., the value is truncated toward zero)."
3220 // C++: ISO/IEC 14882:2020 (C++20), section 7.3.7 — "A prvalue of a
3221 // floating-point type can be converted to a prvalue of an integer type. The
3222 // conversion truncates; that is, the fractional part is discarded."
3223 if (InpTy == MVT::v64f16) {
3224 if (Subtarget.useHVXV81Ops()) {
3225 // This is c/c++ compliant
3226 SDValue ConvVec =
3227 getInstr(MachineOpc: Hexagon::V6_vconv_h_hf_rnd, dl, Ty: ResTy, Ops: {Op0}, DAG);
3228 return ConvVec;
3229 } else if (EnableFpFastConvert) {
3230 // Vd32.h=Vu32.hf same as Q6_Vh_equals_Vhf
3231 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_h_hf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3232 return ConvVec;
3233 }
3234 } else if (EnableFpFastConvert && InpTy == MVT::v32f32) {
3235 // Vd32.w=Vu32.sf same as Q6_Vw_equals_Vsf
3236 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_w_sf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3237 return ConvVec;
3238 }
3239
3240 // int32_t conv_f32_to_i32(uint32_t inp) {
3241 // // s | exp8 | frac23
3242 //
3243 // int neg = (int32_t)inp < 0;
3244 //
3245 // // "expm1" is the actual exponent minus 1: instead of "bias", subtract
3246 // // "bias+1". When the encoded exp is "all-1" (i.e. inf/nan), this will
3247 // // produce a large positive "expm1", which will result in max u/int.
3248 // // In all IEEE formats, bias is the largest positive number that can be
3249 // // represented in bias-width bits (i.e. 011..1).
3250 // int32_t expm1 = (inp << 1) - 0x80000000;
3251 // expm1 >>= 24;
3252 //
3253 // // Always insert the "implicit 1". Subnormal numbers will become 0
3254 // // regardless.
3255 // uint32_t frac = (inp << 8) | 0x80000000;
3256 //
3257 // // "frac" is the fraction part represented as Q1.31. If it was
3258 // // interpreted as uint32_t, it would be the fraction part multiplied
3259 // // by 2^31.
3260 //
3261 // // Calculate the amount of right shift, since shifting further to the
3262 // // left would lose significant bits. Limit it to 32, because we want
3263 // // shifts by 32+ to produce 0, whereas V6_vlsrwv treats the shift
3264 // // amount as a 6-bit signed value (so 33 is same as -31, i.e. shift
3265 // // left by 31). "rsh" can be negative.
3266 // int32_t rsh = min(31 - (expm1 + 1), 32);
3267 //
3268 // frac >>= rsh; // rsh == 32 will produce 0
3269 //
3270 // // Everything up to this point is the same for conversion to signed
3271 // // unsigned integer.
3272 //
3273 // if (neg) // Only for signed int
3274 // frac = -frac; //
3275 // if (rsh <= 0 && neg) // bound = neg ? 0x80000000 : 0x7fffffff
3276 // frac = 0x80000000; // frac = rsh <= 0 ? bound : frac
3277 // if (rsh <= 0 && !neg) //
3278 // frac = 0x7fffffff; //
3279 //
3280 // if (neg) // Only for unsigned int
3281 // frac = 0; //
3282 // if (rsh < 0 && !neg) // frac = rsh < 0 ? 0x7fffffff : frac;
3283 // frac = 0x7fffffff; // frac = neg ? 0 : frac;
3284 //
3285 // return frac;
3286 // }
3287
3288 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: ResTy.getVectorElementCount());
3289
3290 // Zero = V6_vd0();
3291 // Neg = V6_vgtw(Zero, Inp);
3292 // One = V6_lvsplatw(1);
3293 // M80 = V6_lvsplatw(0x80000000);
3294 // Exp00 = V6_vaslwv(Inp, One);
3295 // Exp01 = V6_vsubw(Exp00, M80);
3296 // ExpM1 = V6_vasrw(Exp01, 24);
3297 // Frc00 = V6_vaslw(Inp, 8);
3298 // Frc01 = V6_vor(Frc00, M80);
3299 // Rsh00 = V6_vsubw(V6_lvsplatw(30), ExpM1);
3300 // Rsh01 = V6_vminw(Rsh00, V6_lvsplatw(32));
3301 // Frc02 = V6_vlsrwv(Frc01, Rsh01);
3302
3303 // if signed int:
3304 // Bnd = V6_vmux(Neg, M80, V6_lvsplatw(0x7fffffff))
3305 // Pos = V6_vgtw(Rsh01, Zero);
3306 // Frc13 = V6_vsubw(Zero, Frc02);
3307 // Frc14 = V6_vmux(Neg, Frc13, Frc02);
3308 // Int = V6_vmux(Pos, Frc14, Bnd);
3309 //
3310 // if unsigned int:
3311 // Rsn = V6_vgtw(Zero, Rsh01)
3312 // Frc23 = V6_vmux(Rsn, V6_lvsplatw(0x7fffffff), Frc02)
3313 // Int = V6_vmux(Neg, Zero, Frc23)
3314
3315 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: InpTy);
3316 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3317 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias));
3318
3319 SDValue Inp = DAG.getBitcast(VT: ResTy, V: Op0);
3320 SDValue Zero = getZero(dl, Ty: ResTy, DAG);
3321 SDValue Neg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Inp, RHS: Zero, Cond: ISD::SETLT);
3322 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: ResTy);
3323 SDValue M7F = DAG.getConstant(Val: (1ull << (ElemWidth - 1)) - 1, DL: dl, VT: ResTy);
3324 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResTy);
3325 SDValue Exp00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, One});
3326 SDValue Exp01 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Exp00, M80});
3327 SDValue MNE = DAG.getConstant(Val: ElemWidth - ExpWidth, DL: dl, VT: ResTy);
3328 SDValue ExpM1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: ResTy, Ops: {Exp01, MNE});
3329
3330 SDValue ExpW = DAG.getConstant(Val: ExpWidth, DL: dl, VT: ResTy);
3331 SDValue Frc00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, ExpW});
3332 SDValue Frc01 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ResTy, Ops: {Frc00, M80});
3333
3334 SDValue MN2 = DAG.getConstant(Val: ElemWidth - 2, DL: dl, VT: ResTy);
3335 SDValue Rsh00 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {MN2, ExpM1});
3336 SDValue MW = DAG.getConstant(Val: ElemWidth, DL: dl, VT: ResTy);
3337 SDValue Rsh01 = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ResTy, Ops: {Rsh00, MW});
3338 SDValue Frc02 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ResTy, Ops: {Frc01, Rsh01});
3339
3340 SDValue Int;
3341
3342 if (Opc == ISD::FP_TO_SINT) {
3343 SDValue Bnd = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, M80, M7F});
3344 SDValue Pos = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETGT);
3345 SDValue Frc13 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Zero, Frc02});
3346 SDValue Frc14 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, Frc13, Frc02});
3347 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Pos, Frc14, Bnd});
3348 } else {
3349 assert(Opc == ISD::FP_TO_UINT);
3350 SDValue Rsn = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETLT);
3351 SDValue Frc23 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Rsn, N2: M7F, N3: Frc02);
3352 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Neg, N2: Zero, N3: Frc23);
3353 }
3354
3355 return Int;
3356}
3357
3358SDValue
3359HexagonTargetLowering::ExpandHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
3360 unsigned Opc = Op.getOpcode();
3361 assert(Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3362
3363 const SDLoc &dl(Op);
3364 SDValue Op0 = Op.getOperand(i: 0);
3365 MVT InpTy = ty(Op: Op0);
3366 MVT ResTy = ty(Op);
3367 assert(ResTy.changeTypeToInteger() == InpTy);
3368
3369 // uint32_t vnoc1_rnd(int32_t w) {
3370 // int32_t iszero = w == 0;
3371 // int32_t isneg = w < 0;
3372 // uint32_t u = __builtin_HEXAGON_A2_abs(w);
3373 //
3374 // uint32_t norm_left = __builtin_HEXAGON_S2_cl0(u) + 1;
3375 // uint32_t frac0 = (uint64_t)u << norm_left;
3376 //
3377 // // Rounding:
3378 // uint32_t frac1 = frac0 + ((1 << 8) - 1);
3379 // uint32_t renorm = (frac0 > frac1);
3380 // uint32_t rup = (int)(frac0 << 22) < 0;
3381 //
3382 // uint32_t frac2 = frac0 >> 8;
3383 // uint32_t frac3 = frac1 >> 8;
3384 // uint32_t frac = (frac2 != frac3) ? frac3 >> 1 : (frac3 + rup) >> 1;
3385 //
3386 // int32_t exp = 32 - norm_left + renorm + 127;
3387 // exp <<= 23;
3388 //
3389 // uint32_t sign = 0x80000000 * isneg;
3390 // uint32_t f = sign | exp | frac;
3391 // return iszero ? 0 : f;
3392 // }
3393
3394 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: InpTy.getVectorElementCount());
3395 bool Signed = Opc == ISD::SINT_TO_FP;
3396
3397 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: ResTy);
3398 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3399
3400 SDValue Zero = getZero(dl, Ty: InpTy, DAG);
3401 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: InpTy);
3402 SDValue IsZero = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ);
3403 SDValue Abs = Signed ? DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: InpTy, Operand: Op0) : Op0;
3404 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: InpTy, Operand: Abs);
3405 SDValue NLeft = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Clz, One});
3406 SDValue Frac0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy, Ops: {Abs, NLeft});
3407
3408 auto [Frac, Ovf] = emitHvxShiftRightRnd(Val: Frac0, Amt: ExpWidth + 1, Signed: false, DAG);
3409 if (Signed) {
3410 SDValue IsNeg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETLT);
3411 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: InpTy);
3412 SDValue Sign = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsNeg, M80, Zero});
3413 Frac = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Sign, Frac});
3414 }
3415
3416 SDValue Rnrm = DAG.getZExtOrTrunc(Op: Ovf, DL: dl, VT: InpTy);
3417 SDValue Exp0 = DAG.getConstant(Val: ElemWidth + ExpBias, DL: dl, VT: InpTy);
3418 SDValue Exp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Rnrm, Exp0});
3419 SDValue Exp2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: InpTy, Ops: {Exp1, NLeft});
3420 SDValue Exp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy,
3421 Ops: {Exp2, DAG.getConstant(Val: FracWidth, DL: dl, VT: InpTy)});
3422 SDValue Flt0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Frac, Exp3});
3423 SDValue Flt1 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsZero, Zero, Flt0});
3424 SDValue Flt = DAG.getBitcast(VT: ResTy, V: Flt1);
3425
3426 return Flt;
3427}
3428
3429SDValue
3430HexagonTargetLowering::CreateTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3431 unsigned Opc = Op.getOpcode();
3432 unsigned TLOpc;
3433 switch (Opc) {
3434 case ISD::ANY_EXTEND:
3435 case ISD::SIGN_EXTEND:
3436 case ISD::ZERO_EXTEND:
3437 TLOpc = HexagonISD::TL_EXTEND;
3438 break;
3439 case ISD::TRUNCATE:
3440 TLOpc = HexagonISD::TL_TRUNCATE;
3441 break;
3442#ifndef NDEBUG
3443 Op.dump(&DAG);
3444#endif
3445 llvm_unreachable("Unexpected operator");
3446 }
3447
3448 const SDLoc &dl(Op);
3449 return DAG.getNode(Opcode: TLOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0),
3450 N2: DAG.getUNDEF(VT: MVT::i128), // illegal type
3451 N3: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
3452}
3453
3454SDValue
3455HexagonTargetLowering::RemoveTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3456 assert(Op.getOpcode() == HexagonISD::TL_EXTEND ||
3457 Op.getOpcode() == HexagonISD::TL_TRUNCATE);
3458 unsigned Opc = Op.getConstantOperandVal(i: 2);
3459 return DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: ty(Op), Operand: Op.getOperand(i: 0));
3460}
3461
3462HexagonTargetLowering::VectorPair
3463HexagonTargetLowering::SplitVectorOp(SDValue Op, SelectionDAG &DAG) const {
3464 assert(!Op.isMachineOpcode());
3465 SmallVector<SDValue, 2> OpsL, OpsH;
3466 const SDLoc &dl(Op);
3467
3468 auto SplitVTNode = [&DAG, this](const VTSDNode *N) {
3469 MVT Ty = typeSplit(VecTy: N->getVT().getSimpleVT()).first;
3470 SDValue TV = DAG.getValueType(Ty);
3471 return std::make_pair(x&: TV, y&: TV);
3472 };
3473
3474 for (SDValue A : Op.getNode()->ops()) {
3475 auto [Lo, Hi] =
3476 ty(Op: A).isVector() ? opSplit(Vec: A, dl, DAG) : std::make_pair(x&: A, y&: A);
3477 // Special case for type operand.
3478 switch (Op.getOpcode()) {
3479 case ISD::SIGN_EXTEND_INREG:
3480 case HexagonISD::SSAT:
3481 case HexagonISD::USAT:
3482 if (const auto *N = dyn_cast<const VTSDNode>(Val: A.getNode()))
3483 std::tie(args&: Lo, args&: Hi) = SplitVTNode(N);
3484 break;
3485 }
3486 OpsL.push_back(Elt: Lo);
3487 OpsH.push_back(Elt: Hi);
3488 }
3489
3490 MVT ResTy = ty(Op);
3491 MVT HalfTy = typeSplit(VecTy: ResTy).first;
3492 SDValue L = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsL);
3493 SDValue H = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsH);
3494 return {L, H};
3495}
3496
3497SDValue
3498HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
3499 auto *MemN = cast<MemSDNode>(Val: Op.getNode());
3500 unsigned MemOpc = MemN->getOpcode();
3501 EVT MemTy = MemN->getMemoryVT();
3502
3503 if ((MemOpc == ISD::STORE || MemOpc == ISD::LOAD) &&
3504 (!MemTy.isSimple() || !isHvxPairTy(Ty: MemTy.getSimpleVT())))
3505 return Op;
3506
3507 EVT ValueType;
3508 if (MemOpc == ISD::STORE)
3509 ValueType = ty(Op: cast<StoreSDNode>(Val&: Op)->getValue());
3510 else if (MemOpc == ISD::MSTORE)
3511 ValueType = ty(Op: cast<MaskedStoreSDNode>(Val&: Op)->getValue());
3512 else // ISD::LOAD, ISD::MLOAD.
3513 ValueType = MemN->getValueType(ResNo: 0);
3514
3515 EVT LoVT, HiVT;
3516 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: ValueType);
3517
3518 EVT LoMemVT, HiMemVT;
3519 bool HiIsEmpty = false;
3520 std::tie(args&: LoMemVT, args&: HiMemVT) =
3521 DAG.GetDependentSplitDestVTs(VT: MemTy, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
3522
3523 uint64_t LoSize = LoMemVT.getSizeInBits().getFixedValue() / 8;
3524 uint64_t HiSize = HiMemVT.getSizeInBits().getFixedValue() / 8;
3525
3526 const SDLoc &dl(Op);
3527 SDValue Chain = MemN->getChain();
3528 SDValue Base0 = MemN->getBasePtr();
3529 SDValue Base1 =
3530 DAG.getMemBasePlusOffset(Base: Base0, Offset: TypeSize::getFixed(ExactSize: LoSize), DL: dl);
3531
3532 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
3533 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
3534 MachineFunction &MF = DAG.getMachineFunction();
3535 auto MemSize = [=](uint64_t Size) {
3536 return (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
3537 ? (uint64_t)MemoryLocation::UnknownSize
3538 : Size;
3539 };
3540 // MOp1 will not be used if HiIsEmpty for masked loads and stores (MLOAD and
3541 // MSTORE). Non-masked loads and store are always of double-vector size (see
3542 // isHvxPairTy() check above).
3543 MOp0 = MF.getMachineMemOperand(MMO, Offset: 0, Size: MemSize(LoSize));
3544 MOp1 = MF.getMachineMemOperand(MMO, Offset: LoSize, Size: MemSize(HiSize));
3545 }
3546
3547 if (MemOpc == ISD::LOAD) {
3548 assert(cast<LoadSDNode>(Op)->isUnindexed());
3549 SDValue Load0 = DAG.getLoad(VT: LoVT, dl, Chain, Ptr: Base0, MMO: MOp0);
3550 SDValue Load1 = DAG.getLoad(VT: HiVT, dl, Chain, Ptr: Base1, MMO: MOp1);
3551 return DAG.getMergeValues(
3552 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: Load0,
3553 N2: Load1),
3554 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Load0.getValue(R: 1),
3555 N2: Load1.getValue(R: 1))},
3556 dl);
3557 }
3558 if (MemOpc == ISD::STORE) {
3559 assert(cast<StoreSDNode>(Op)->isUnindexed());
3560 VectorPair Vals = opSplit(Vec: cast<StoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3561 SDValue Store0 = DAG.getStore(Chain, dl, Val: Vals.first, Ptr: Base0, MMO: MOp0);
3562 SDValue Store1 = DAG.getStore(Chain, dl, Val: Vals.second, Ptr: Base1, MMO: MOp1);
3563 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store0, N2: Store1);
3564 }
3565
3566 assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE);
3567
3568 auto MaskN = cast<MaskedLoadStoreSDNode>(Val&: Op);
3569 assert(MaskN->isUnindexed());
3570 VectorPair Masks = opSplit(Vec: MaskN->getMask(), dl, DAG);
3571 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3572
3573 if (MemOpc == ISD::MLOAD) {
3574 VectorPair Thru =
3575 opSplit(Vec: cast<MaskedLoadSDNode>(Val&: Op)->getPassThru(), dl, DAG);
3576 SDValue MLoad0 = DAG.getMaskedLoad(VT: LoVT, dl, Chain, Base: Base0, Offset,
3577 Mask: Masks.first, Src0: Thru.first, MemVT: LoMemVT, MMO: MOp0,
3578 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3579
3580 // The hi masked load has zero storage size. We therefore simply set it to
3581 // the low masked load and rely on subsequent removal from the chain as it
3582 // is unused. See DAGTypeLegalizer::SplitVecRes_MLOAD() for the same logic.
3583 SDValue MLoad1 =
3584 HiIsEmpty ? MLoad0
3585 : DAG.getMaskedLoad(VT: HiVT, dl, Chain, Base: Base1, Offset,
3586 Mask: Masks.second, Src0: Thru.second, MemVT: HiMemVT, MMO: MOp1,
3587 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3588 return DAG.getMergeValues(
3589 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: MLoad0,
3590 N2: MLoad1),
3591 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MLoad0.getValue(R: 1),
3592 N2: MLoad1.getValue(R: 1))},
3593 dl);
3594 }
3595 if (MemOpc == ISD::MSTORE) {
3596 VectorPair Vals = opSplit(Vec: cast<MaskedStoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3597 SDValue MStore0 =
3598 DAG.getMaskedStore(Chain, dl, Val: Vals.first, Base: Base0, Offset, Mask: Masks.first,
3599 MemVT: LoMemVT, MMO: MOp0, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3600 if (HiIsEmpty)
3601 return MStore0;
3602 SDValue MStore1 =
3603 DAG.getMaskedStore(Chain, dl, Val: Vals.second, Base: Base1, Offset, Mask: Masks.second,
3604 MemVT: HiMemVT, MMO: MOp1, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3605 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MStore0, N2: MStore1);
3606 }
3607
3608 std::string Name = "Unexpected operation: " + Op->getOperationName(G: &DAG);
3609 llvm_unreachable(Name.c_str());
3610}
3611
3612SDValue
3613HexagonTargetLowering::WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const {
3614 const SDLoc &dl(Op);
3615 auto *LoadN = cast<LoadSDNode>(Val: Op.getNode());
3616 assert(LoadN->isUnindexed() && "Not widening indexed loads yet");
3617 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3618 "Not widening loads of i1 yet");
3619
3620 SDValue Chain = LoadN->getChain();
3621 SDValue Base = LoadN->getBasePtr();
3622 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3623
3624 MVT ResTy = ty(Op);
3625 unsigned HwLen = Subtarget.getVectorLength();
3626 unsigned ResLen = ResTy.getStoreSize();
3627 assert(ResLen < HwLen && "vsetq(v1) prerequisite");
3628
3629 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3630 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3631 Ops: {DAG.getConstant(Val: ResLen, DL: dl, VT: MVT::i32)}, DAG);
3632
3633 MVT LoadTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
3634 MachineFunction &MF = DAG.getMachineFunction();
3635 auto *MemOp = MF.getMachineMemOperand(MMO: LoadN->getMemOperand(), Offset: 0, Size: HwLen);
3636
3637 SDValue Load = DAG.getMaskedLoad(VT: LoadTy, dl, Chain, Base, Offset, Mask,
3638 Src0: DAG.getUNDEF(VT: LoadTy), MemVT: LoadTy, MMO: MemOp,
3639 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3640 SDValue Value = opCastElem(Vec: Load, ElemTy: ResTy.getVectorElementType(), DAG);
3641 return DAG.getMergeValues(Ops: {Value, Load.getValue(R: 1)}, dl);
3642}
3643
3644SDValue
3645HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const {
3646 const SDLoc &dl(Op);
3647 auto *StoreN = cast<StoreSDNode>(Val: Op.getNode());
3648 assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
3649 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3650 "Not widening stores of i1 yet");
3651
3652 SDValue Chain = StoreN->getChain();
3653 SDValue Base = StoreN->getBasePtr();
3654 SDValue Offset = DAG.getPOISON(VT: MVT::i32);
3655
3656 SDValue Value = opCastElem(Vec: StoreN->getValue(), ElemTy: MVT::i8, DAG);
3657 MVT ValueTy = ty(Op: Value);
3658 unsigned ValueLen = ValueTy.getVectorNumElements();
3659 unsigned HwLen = Subtarget.getVectorLength();
3660 assert(isPowerOf2_32(ValueLen));
3661
3662 for (unsigned Len = ValueLen; Len < HwLen; ) {
3663 Value = opJoin(Ops: {Value, DAG.getUNDEF(VT: ty(Op: Value))}, dl, DAG);
3664 Len = ty(Op: Value).getVectorNumElements(); // This is Len *= 2
3665 }
3666 assert(ty(Value).getVectorNumElements() == HwLen); // Paranoia
3667
3668 assert(ValueLen < HwLen && "vsetq(v1) prerequisite");
3669 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3670 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3671 Ops: {DAG.getConstant(Val: ValueLen, DL: dl, VT: MVT::i32)}, DAG);
3672 MachineFunction &MF = DAG.getMachineFunction();
3673 auto *MemOp = MF.getMachineMemOperand(MMO: StoreN->getMemOperand(), Offset: 0, Size: HwLen);
3674 return DAG.getMaskedStore(Chain, dl, Val: Value, Base, Offset, Mask,
3675 MemVT: StoreN->getMemoryVT(), MMO: MemOp, AM: ISD::UNINDEXED, IsTruncating: false,
3676 IsCompressing: false);
3677}
3678
3679SDValue
3680HexagonTargetLowering::WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const {
3681 const SDLoc &dl(Op);
3682 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
3683 MVT ElemTy = ty(Op: Op0).getVectorElementType();
3684 unsigned HwLen = Subtarget.getVectorLength();
3685
3686 unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits();
3687 assert(WideOpLen * ElemTy.getSizeInBits() == 8 * HwLen);
3688 MVT WideOpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideOpLen);
3689 if (!Subtarget.isHVXVectorType(VecTy: WideOpTy, IncludeBool: true))
3690 return SDValue();
3691
3692 SDValue WideOp0 = appendUndef(Val: Op0, ResTy: WideOpTy, DAG);
3693 SDValue WideOp1 = appendUndef(Val: Op1, ResTy: WideOpTy, DAG);
3694 EVT ResTy =
3695 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: WideOpTy);
3696 SDValue SetCC = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: ResTy,
3697 Ops: {WideOp0, WideOp1, Op.getOperand(i: 2)});
3698
3699 EVT RetTy = typeLegalize(Ty: ty(Op), DAG);
3700 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3701 Ops: {SetCC, getZero(dl, Ty: MVT::i32, DAG)});
3702}
3703
3704SDValue HexagonTargetLowering::WidenHvxTruncateToBool(SDValue Op,
3705 SelectionDAG &DAG) const {
3706 // Handle truncation to boolean vector where the result boolean type
3707 // needs widening (e.g., v16i32 -> v16i1 where v16i1 is not a standard
3708 // HVX predicate type, or v16i8 -> v16i1 in 128-byte mode).
3709 // Widen the input to HVX width, perform the truncate to the widened
3710 // boolean type, then extract the result.
3711 const SDLoc &dl(Op);
3712 SDValue Inp = Op.getOperand(i: 0);
3713 MVT InpTy = ty(Op: Inp);
3714 MVT ResTy = ty(Op);
3715
3716 assert(ResTy.getVectorElementType() == MVT::i1 &&
3717 "Expected boolean result type");
3718
3719 MVT ElemTy = InpTy.getVectorElementType();
3720 unsigned HwLen = Subtarget.getVectorLength();
3721
3722 // Calculate the widened input type that fills the HVX register.
3723 unsigned WideLen = (8 * HwLen) / ElemTy.getSizeInBits();
3724 MVT WideInpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideLen);
3725 if (!Subtarget.isHVXVectorType(VecTy: WideInpTy, IncludeBool: false))
3726 return SDValue();
3727
3728 // Widen the input to HVX width.
3729 SDValue WideInp = appendUndef(Val: Inp, ResTy: WideInpTy, DAG);
3730
3731 // Perform the truncate to widened boolean type.
3732 MVT WideBoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: WideLen);
3733 SDValue WideTrunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: WideBoolTy, Operand: WideInp);
3734
3735 // Extract the result.
3736 EVT RetTy = typeLegalize(Ty: ResTy, DAG);
3737 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3738 Ops: {WideTrunc, getZero(dl, Ty: MVT::i32, DAG)});
3739}
3740
3741SDValue
3742HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
3743 unsigned Opc = Op.getOpcode();
3744 bool IsPairOp = isHvxPairTy(Ty: ty(Op)) ||
3745 llvm::any_of(Range: Op.getNode()->ops(), P: [this] (SDValue V) {
3746 return isHvxPairTy(Ty: ty(Op: V));
3747 });
3748
3749 if (IsPairOp) {
3750 switch (Opc) {
3751 default:
3752 break;
3753 case ISD::LOAD:
3754 case ISD::STORE:
3755 case ISD::MLOAD:
3756 case ISD::MSTORE:
3757 return SplitHvxMemOp(Op, DAG);
3758 case ISD::SINT_TO_FP:
3759 case ISD::UINT_TO_FP:
3760 case ISD::FP_TO_SINT:
3761 case ISD::FP_TO_UINT:
3762 if (ty(Op).getSizeInBits() == ty(Op: Op.getOperand(i: 0)).getSizeInBits())
3763 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3764 break;
3765 case ISD::ABS:
3766 case ISD::CTPOP:
3767 case ISD::CTLZ:
3768 case ISD::CTTZ:
3769 case ISD::MUL:
3770 case ISD::FADD:
3771 case ISD::FSUB:
3772 case ISD::FMUL:
3773 case ISD::FMINIMUMNUM:
3774 case ISD::FMAXIMUMNUM:
3775 case ISD::FMINNUM:
3776 case ISD::FMAXNUM:
3777 case ISD::MULHS:
3778 case ISD::MULHU:
3779 case ISD::AND:
3780 case ISD::OR:
3781 case ISD::XOR:
3782 case ISD::SRA:
3783 case ISD::SHL:
3784 case ISD::SRL:
3785 case ISD::FSHL:
3786 case ISD::FSHR:
3787 case ISD::SMIN:
3788 case ISD::SMAX:
3789 case ISD::UMIN:
3790 case ISD::UMAX:
3791 case ISD::SETCC:
3792 case ISD::VSELECT:
3793 case ISD::SIGN_EXTEND_INREG:
3794 case ISD::SPLAT_VECTOR:
3795 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3796 case ISD::SIGN_EXTEND:
3797 case ISD::ZERO_EXTEND:
3798 // In general, sign- and zero-extends can't be split and still
3799 // be legal. The only exception is extending bool vectors.
3800 if (ty(Op: Op.getOperand(i: 0)).getVectorElementType() == MVT::i1)
3801 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3802 break;
3803 }
3804 }
3805
3806 switch (Opc) {
3807 default:
3808 break;
3809 // clang-format off
3810 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
3811 case ISD::SPLAT_VECTOR: return LowerHvxSplatVector(Op, DAG);
3812 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
3813 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
3814 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
3815 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
3816 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
3817 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
3818 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
3819 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
3820 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
3821 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
3822 case ISD::SELECT: return LowerHvxSelect(Op, DAG);
3823 case ISD::SRA:
3824 case ISD::SHL:
3825 case ISD::SRL: return LowerHvxShift(Op, DAG);
3826 case ISD::FSHL:
3827 case ISD::FSHR: return LowerHvxFunnelShift(Op, DAG);
3828 case ISD::MULHS:
3829 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
3830 case ISD::SMUL_LOHI:
3831 case ISD::UMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3832 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
3833 case ISD::SETCC: {
3834 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
3835 if (CC == ISD::SETOEQ &&
3836 ty(Op: Op.getOperand(i: 0)).getScalarType().isFloatingPoint())
3837 return LowerHvxFpSetoeq(Op, DAG);
3838 return Op;
3839 }
3840 case ISD::INTRINSIC_VOID: return Op;
3841 case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG);
3842 case ISD::MLOAD:
3843 case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG);
3844 // Unaligned loads will be handled by the default lowering.
3845 case ISD::LOAD: return LowerHvxLoad(Op, DAG);
3846 case ISD::STORE: return LowerHvxStore(Op, DAG);
3847 case ISD::FP_EXTEND: return LowerHvxFpExtend(Op, DAG);
3848 case ISD::FP_TO_SINT:
3849 case ISD::FP_TO_UINT: return LowerHvxFpToInt(Op, DAG);
3850 case ISD::SINT_TO_FP:
3851 case ISD::UINT_TO_FP: return LowerHvxIntToFp(Op, DAG);
3852
3853 // Special nodes:
3854 case HexagonISD::SMUL_LOHI:
3855 case HexagonISD::UMUL_LOHI:
3856 case HexagonISD::USMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3857
3858 case ISD::PARTIAL_REDUCE_SMLA:
3859 case ISD::PARTIAL_REDUCE_UMLA:
3860 case ISD::PARTIAL_REDUCE_SUMLA:
3861 return LowerHvxPartialReduceMLA(Op, DAG);
3862 // clang-format on
3863 }
3864#ifndef NDEBUG
3865 Op.dumpr(&DAG);
3866#endif
3867 llvm_unreachable("Unhandled HVX operation");
3868}
3869
3870SDValue
3871HexagonTargetLowering::ExpandHvxResizeIntoSteps(SDValue Op, SelectionDAG &DAG)
3872 const {
3873 // Rewrite the extension/truncation/saturation op into steps where each
3874 // step changes the type widths by a factor of 2.
3875 // E.g. i8 -> i16 remains unchanged, but i8 -> i32 ==> i8 -> i16 -> i32.
3876 //
3877 // Some of the vector types in Op may not be legal.
3878
3879 unsigned Opc = Op.getOpcode();
3880 switch (Opc) {
3881 case HexagonISD::SSAT:
3882 case HexagonISD::USAT:
3883 case HexagonISD::TL_EXTEND:
3884 case HexagonISD::TL_TRUNCATE:
3885 break;
3886 case ISD::ANY_EXTEND:
3887 case ISD::ZERO_EXTEND:
3888 case ISD::SIGN_EXTEND:
3889 case ISD::TRUNCATE:
3890 llvm_unreachable("ISD:: ops will be auto-folded");
3891 break;
3892#ifndef NDEBUG
3893 Op.dump(&DAG);
3894#endif
3895 llvm_unreachable("Unexpected operation");
3896 }
3897
3898 SDValue Inp = Op.getOperand(i: 0);
3899 MVT InpTy = ty(Op: Inp);
3900 MVT ResTy = ty(Op);
3901
3902 unsigned InpWidth = InpTy.getVectorElementType().getSizeInBits();
3903 unsigned ResWidth = ResTy.getVectorElementType().getSizeInBits();
3904 assert(InpWidth != ResWidth);
3905
3906 if (InpWidth == 2 * ResWidth || ResWidth == 2 * InpWidth)
3907 return Op;
3908
3909 const SDLoc &dl(Op);
3910 unsigned NumElems = InpTy.getVectorNumElements();
3911 assert(NumElems == ResTy.getVectorNumElements());
3912
3913 auto repeatOp = [&](unsigned NewWidth, SDValue Arg) {
3914 MVT Ty = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: NewWidth), NumElements: NumElems);
3915 switch (Opc) {
3916 case HexagonISD::SSAT:
3917 case HexagonISD::USAT:
3918 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, DAG.getValueType(Ty)});
3919 case HexagonISD::TL_EXTEND:
3920 case HexagonISD::TL_TRUNCATE:
3921 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, Op.getOperand(i: 1), Op.getOperand(i: 2)});
3922 default:
3923 llvm_unreachable("Unexpected opcode");
3924 }
3925 };
3926
3927 SDValue S = Inp;
3928 if (InpWidth < ResWidth) {
3929 assert(ResWidth % InpWidth == 0 && isPowerOf2_32(ResWidth / InpWidth));
3930 while (InpWidth * 2 <= ResWidth)
3931 S = repeatOp(InpWidth *= 2, S);
3932 } else {
3933 // InpWidth > ResWidth
3934 assert(InpWidth % ResWidth == 0 && isPowerOf2_32(InpWidth / ResWidth));
3935 while (InpWidth / 2 >= ResWidth)
3936 S = repeatOp(InpWidth /= 2, S);
3937 }
3938 return S;
3939}
3940
3941SDValue
3942HexagonTargetLowering::LegalizeHvxResize(SDValue Op, SelectionDAG &DAG) const {
3943 SDValue Inp0 = Op.getOperand(i: 0);
3944 MVT InpTy = ty(Op: Inp0);
3945 MVT ResTy = ty(Op);
3946 unsigned InpWidth = InpTy.getSizeInBits();
3947 unsigned ResWidth = ResTy.getSizeInBits();
3948 unsigned Opc = Op.getOpcode();
3949
3950 if (shouldWidenToHvx(Ty: InpTy, DAG) || shouldWidenToHvx(Ty: ResTy, DAG)) {
3951 // First, make sure that the narrower type is widened to HVX.
3952 // This may cause the result to be wider than what the legalizer
3953 // expects, so insert EXTRACT_SUBVECTOR to bring it back to the
3954 // desired type.
3955 auto [WInpTy, WResTy] =
3956 InpWidth < ResWidth ? typeWidenToWider(Ty0: typeWidenToHvx(Ty: InpTy), Ty1: ResTy)
3957 : typeWidenToWider(Ty0: InpTy, Ty1: typeWidenToHvx(Ty: ResTy));
3958 SDValue W = appendUndef(Val: Inp0, ResTy: WInpTy, DAG);
3959 SDValue S;
3960 if (Opc == HexagonISD::TL_EXTEND || Opc == HexagonISD::TL_TRUNCATE) {
3961 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: Op.getOperand(i: 1),
3962 N3: Op.getOperand(i: 2));
3963 } else {
3964 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: DAG.getValueType(WResTy));
3965 }
3966 SDValue T = ExpandHvxResizeIntoSteps(Op: S, DAG);
3967 return extractSubvector(Vec: T, SubTy: typeLegalize(Ty: ResTy, DAG), SubIdx: 0, DAG);
3968 } else if (shouldSplitToHvx(Ty: InpWidth < ResWidth ? ResTy : InpTy, DAG)) {
3969 // For multi-step extends/truncates (e.g., i8->i32), expand into
3970 // single-step operations first. Splitting a multi-step TL_EXTEND
3971 // would halve the operand type to a sub-HVX size (e.g., v128i8 ->
3972 // v64i8), creating illegal types that cause issues in the type
3973 // legalizer's map tracking. Single-step operations (e.g., i16->i32)
3974 // are safe to split because their halved operand types remain legal.
3975 SDValue T = ExpandHvxResizeIntoSteps(Op, DAG);
3976 if (T != Op)
3977 return T;
3978 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3979 } else {
3980 assert(isTypeLegal(InpTy) && isTypeLegal(ResTy));
3981 return RemoveTLWrapper(Op, DAG);
3982 }
3983 llvm_unreachable("Unexpected situation");
3984}
3985
3986void
3987HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
3988 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
3989 unsigned Opc = N->getOpcode();
3990 SDValue Op(N, 0);
3991 SDValue Inp0; // Optional first argument.
3992 if (N->getNumOperands() > 0)
3993 Inp0 = Op.getOperand(i: 0);
3994
3995 switch (Opc) {
3996 case ISD::ANY_EXTEND:
3997 case ISD::SIGN_EXTEND:
3998 case ISD::ZERO_EXTEND:
3999 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4000 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4001 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4002 }
4003 break;
4004 case ISD::TRUNCATE:
4005 // Handle truncate to boolean vector when the input is not a
4006 // standard HVX vector type (single or pair). This covers cases
4007 // where the input needs widening (e.g., v64i8 -> v64i1 in
4008 // 128-byte mode) and cases where the result boolean type itself
4009 // needs widening (e.g., v16i32 -> v16i1). When the input is
4010 // already an HVX type, tablegen patterns handle the truncation
4011 // directly (e.g., v64i16 -> v64i1 via V6_vandvrt).
4012 if (ty(Op).getVectorElementType() == MVT::i1 &&
4013 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
4014 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
4015 Results.push_back(Elt: T);
4016 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4017 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4018 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4019 }
4020 break;
4021 case ISD::SETCC:
4022 if (shouldWidenToHvx(Ty: ty(Op: Inp0), DAG)) {
4023 if (SDValue T = WidenHvxSetCC(Op, DAG))
4024 Results.push_back(Elt: T);
4025 }
4026 break;
4027 case ISD::STORE: {
4028 if (shouldWidenToHvx(Ty: ty(Op: cast<StoreSDNode>(Val: N)->getValue()), DAG)) {
4029 SDValue Store = WidenHvxStore(Op, DAG);
4030 Results.push_back(Elt: Store);
4031 }
4032 break;
4033 }
4034 case ISD::MLOAD:
4035 if (isHvxPairTy(Ty: ty(Op))) {
4036 SDValue S = SplitHvxMemOp(Op, DAG);
4037 assert(S->getOpcode() == ISD::MERGE_VALUES);
4038 Results.push_back(Elt: S.getOperand(i: 0));
4039 Results.push_back(Elt: S.getOperand(i: 1));
4040 }
4041 break;
4042 case ISD::MSTORE:
4043 if (isHvxPairTy(Ty: ty(Op: Op->getOperand(Num: 1)))) { // Stored value
4044 SDValue S = SplitHvxMemOp(Op, DAG);
4045 Results.push_back(Elt: S);
4046 }
4047 break;
4048 case ISD::SINT_TO_FP:
4049 case ISD::UINT_TO_FP:
4050 case ISD::FP_TO_SINT:
4051 case ISD::FP_TO_UINT:
4052 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4053 SDValue T = EqualizeFpIntConversion(Op, DAG);
4054 Results.push_back(Elt: T);
4055 }
4056 break;
4057 case HexagonISD::SSAT:
4058 case HexagonISD::USAT:
4059 case HexagonISD::TL_EXTEND:
4060 case HexagonISD::TL_TRUNCATE:
4061 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4062 break;
4063 default:
4064 break;
4065 }
4066}
4067
4068void
4069HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
4070 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
4071 unsigned Opc = N->getOpcode();
4072 SDValue Op(N, 0);
4073 SDValue Inp0; // Optional first argument.
4074 if (N->getNumOperands() > 0)
4075 Inp0 = Op.getOperand(i: 0);
4076
4077 switch (Opc) {
4078 case ISD::ANY_EXTEND:
4079 case ISD::SIGN_EXTEND:
4080 case ISD::ZERO_EXTEND:
4081 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4082 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4083 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4084 }
4085 break;
4086 case ISD::TRUNCATE:
4087 // Handle truncate to boolean vector when the input is not a
4088 // standard HVX vector type. See comment in LowerHvxOperationWrapper.
4089 if (ty(Op).getVectorElementType() == MVT::i1 &&
4090 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
4091 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
4092 Results.push_back(Elt: T);
4093 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4094 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4095 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4096 }
4097 break;
4098 case ISD::SETCC:
4099 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4100 if (SDValue T = WidenHvxSetCC(Op, DAG))
4101 Results.push_back(Elt: T);
4102 }
4103 break;
4104 case ISD::LOAD: {
4105 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4106 SDValue Load = WidenHvxLoad(Op, DAG);
4107 assert(Load->getOpcode() == ISD::MERGE_VALUES);
4108 Results.push_back(Elt: Load.getOperand(i: 0));
4109 Results.push_back(Elt: Load.getOperand(i: 1));
4110 }
4111 break;
4112 }
4113 case ISD::BITCAST:
4114 if (isHvxBoolTy(Ty: ty(Op: Inp0))) {
4115 SDValue C = LowerHvxBitcast(Op, DAG);
4116 Results.push_back(Elt: C);
4117 }
4118 break;
4119 case ISD::FP_TO_SINT:
4120 case ISD::FP_TO_UINT:
4121 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4122 SDValue T = EqualizeFpIntConversion(Op, DAG);
4123 Results.push_back(Elt: T);
4124 }
4125 break;
4126 case HexagonISD::SSAT:
4127 case HexagonISD::USAT:
4128 case HexagonISD::TL_EXTEND:
4129 case HexagonISD::TL_TRUNCATE:
4130 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4131 break;
4132 default:
4133 break;
4134 }
4135}
4136
4137SDValue
4138HexagonTargetLowering::combineTruncateBeforeLegal(SDValue Op,
4139 DAGCombinerInfo &DCI) const {
4140 // Simplify V:v2NiB --(bitcast)--> vNi2B --(truncate)--> vNiB
4141 // to extract-subvector (shuffle V, pick even, pick odd)
4142
4143 assert(Op.getOpcode() == ISD::TRUNCATE);
4144 SelectionDAG &DAG = DCI.DAG;
4145 const SDLoc &dl(Op);
4146
4147 if (Op.getOperand(i: 0).getOpcode() == ISD::BITCAST)
4148 return SDValue();
4149 SDValue Cast = Op.getOperand(i: 0);
4150 SDValue Src = Cast.getOperand(i: 0);
4151
4152 EVT TruncTy = Op.getValueType();
4153 EVT CastTy = Cast.getValueType();
4154 EVT SrcTy = Src.getValueType();
4155 if (SrcTy.isSimple())
4156 return SDValue();
4157 if (SrcTy.getVectorElementType() != TruncTy.getVectorElementType())
4158 return SDValue();
4159 unsigned SrcLen = SrcTy.getVectorNumElements();
4160 unsigned CastLen = CastTy.getVectorNumElements();
4161 if (2 * CastLen != SrcLen)
4162 return SDValue();
4163
4164 SmallVector<int, 128> Mask(SrcLen);
4165 for (int i = 0; i != static_cast<int>(CastLen); ++i) {
4166 Mask[i] = 2 * i;
4167 Mask[i + CastLen] = 2 * i + 1;
4168 }
4169 SDValue Deal =
4170 DAG.getVectorShuffle(VT: SrcTy, dl, N1: Src, N2: DAG.getUNDEF(VT: SrcTy), Mask);
4171 return opSplit(Vec: Deal, dl, DAG).first;
4172}
4173
4174SDValue
4175HexagonTargetLowering::combineConcatOfShuffles(SDValue Op,
4176 SelectionDAG &DAG) const {
4177 // Fold
4178 // concat (shuffle x, y, m1), (shuffle x, y, m2)
4179 // into
4180 // shuffle (concat x, y), undef, m3
4181 if (Op.getNumOperands() != 2)
4182 return SDValue();
4183
4184 const SDLoc &dl(Op);
4185 SDValue V0 = Op.getOperand(i: 0);
4186 SDValue V1 = Op.getOperand(i: 1);
4187
4188 if (V0.getOpcode() != ISD::VECTOR_SHUFFLE)
4189 return SDValue();
4190 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE)
4191 return SDValue();
4192
4193 SetVector<SDValue> Order;
4194 Order.insert(X: V0.getOperand(i: 0));
4195 Order.insert(X: V0.getOperand(i: 1));
4196 Order.insert(X: V1.getOperand(i: 0));
4197 Order.insert(X: V1.getOperand(i: 1));
4198
4199 if (Order.size() > 2)
4200 return SDValue();
4201
4202 // In ISD::VECTOR_SHUFFLE, the types of each input and the type of the
4203 // result must be the same.
4204 EVT InpTy = V0.getValueType();
4205 assert(InpTy.isVector());
4206 unsigned InpLen = InpTy.getVectorNumElements();
4207
4208 SmallVector<int, 128> LongMask;
4209 auto AppendToMask = [&](SDValue Shuffle) {
4210 auto *SV = cast<ShuffleVectorSDNode>(Val: Shuffle.getNode());
4211 ArrayRef<int> Mask = SV->getMask();
4212 SDValue X = Shuffle.getOperand(i: 0);
4213 SDValue Y = Shuffle.getOperand(i: 1);
4214 for (int M : Mask) {
4215 if (M == -1) {
4216 LongMask.push_back(Elt: M);
4217 continue;
4218 }
4219 SDValue Src = static_cast<unsigned>(M) < InpLen ? X : Y;
4220 if (static_cast<unsigned>(M) >= InpLen)
4221 M -= InpLen;
4222
4223 int OutOffset = Order[0] == Src ? 0 : InpLen;
4224 LongMask.push_back(Elt: M + OutOffset);
4225 }
4226 };
4227
4228 AppendToMask(V0);
4229 AppendToMask(V1);
4230
4231 SDValue C0 = Order.front();
4232 SDValue C1 = Order.back(); // Can be same as front
4233 EVT LongTy = InpTy.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4234
4235 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LongTy, Ops: {C0, C1});
4236 return DAG.getVectorShuffle(VT: LongTy, dl, N1: Cat, N2: DAG.getUNDEF(VT: LongTy), Mask: LongMask);
4237}
4238
4239// Reassociate concat(p1, p2, ...) into
4240// concat(concat(p1, ...), concat(pi, ...), ...)
4241// where each inner concat produces a predicate where each bit corresponds
4242// to at most BitBytes bytes.
4243// Concatenating predicates decreases the number of bytes per each predicate
4244// bit.
4245SDValue
4246HexagonTargetLowering::combineConcatOfScalarPreds(SDValue Op, unsigned BitBytes,
4247 SelectionDAG &DAG) const {
4248 const SDLoc &dl(Op);
4249 SmallVector<SDValue> Ops(Op->ops());
4250 MVT ResTy = ty(Op);
4251 MVT InpTy = ty(Op: Ops[0]);
4252 unsigned InpLen = InpTy.getVectorNumElements(); // Scalar predicate
4253 unsigned ResLen = ResTy.getVectorNumElements(); // HVX vector predicate
4254 assert(InpLen <= 8 && "Too long for scalar predicate");
4255 assert(ResLen > 8 && "Too short for HVX vector predicate");
4256
4257 unsigned Bytes = 8 / InpLen; // Bytes-per-bit in input
4258
4259 // Already in the right form?
4260 if (Bytes <= BitBytes)
4261 return Op;
4262
4263 ArrayRef<SDValue> Inputs(Ops);
4264 unsigned SliceLen = Bytes / BitBytes;
4265
4266 SmallVector<SDValue> Cats;
4267 // (8 / BitBytes) is the desired length of the result of the inner concat.
4268 MVT InnerTy = MVT::getVectorVT(VT: MVT::i1, NumElements: 8 / BitBytes);
4269 for (unsigned i = 0; i != ResLen / (8 / BitBytes); ++i) {
4270 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: InnerTy,
4271 Ops: Inputs.slice(N: SliceLen * i, M: SliceLen));
4272 Cats.push_back(Elt: Cat);
4273 }
4274
4275 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Cats);
4276}
4277
4278SDValue HexagonTargetLowering::combineConcatVectorsBeforeLegal(
4279 SDValue Op, DAGCombinerInfo &DCI) const {
4280 MVT ResTy = ty(Op);
4281 MVT ElemTy = ResTy.getVectorElementType();
4282
4283 if (ElemTy != MVT::i1) {
4284 return combineConcatOfShuffles(Op, DAG&: DCI.DAG);
4285 }
4286 return SDValue();
4287}
4288
4289// Create the inner partial reduction MLA that can be efficiently lowered. This
4290// function is used by partial and full reductions.
4291SDValue HexagonTargetLowering::createExtendingPartialReduceMLA(
4292 unsigned Opcode, EVT AccEltType, unsigned AccNumElements, EVT InputType,
4293 const SDValue &A, const SDValue &B, unsigned &RemainingReductionRatio,
4294 const SDLoc &DL, SelectionDAG &DAG) const {
4295 const auto &Subtarget = DAG.getSubtarget<HexagonSubtarget>();
4296 if (!Subtarget.useHVXOps())
4297 return SDValue();
4298
4299 EVT InputEltType = InputType.getVectorElementType();
4300
4301 // Find if an optimized instruction for the sub-reduction is available.
4302 unsigned NativeRatio;
4303 if (AccEltType == MVT::i32 && InputEltType == MVT::i8)
4304 NativeRatio = 4;
4305 else
4306 return SDValue();
4307
4308 // We only handle the case when additional reduction will be needed, i.e.
4309 // input is longer by a larger factor than the result.
4310 ElementCount InputEC = InputType.getVectorElementCount();
4311 if (!InputEC.isKnownMultipleOf(RHS: AccNumElements * NativeRatio))
4312 return SDValue();
4313
4314 unsigned InputNumElements = InputEC.getFixedValue();
4315 RemainingReductionRatio = InputNumElements / (AccNumElements * NativeRatio);
4316 if (RemainingReductionRatio == 1)
4317 return SDValue();
4318
4319 // Create a reduction by the natively supported factor.
4320 EVT IntermediateType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType,
4321 NumElements: InputNumElements / NativeRatio);
4322
4323 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: IntermediateType);
4324 return DAG.getNode(Opcode, DL, VT: IntermediateType, N1: Zero, N2: A, N3: B);
4325}
4326
4327static bool DetectExtendingMultiply(const SDValue &N, EVT ScalarType,
4328 unsigned &Opcode, SDValue &A, SDValue &B) {
4329 SDValue Mul = N;
4330 EVT AccType = Mul.getValueType(); // Vector input type after extension.
4331 if (ScalarType != AccType.getVectorElementType())
4332 return false;
4333 bool swap = false;
4334 if (Mul->getOpcode() != ISD::MUL)
4335 return false;
4336 A = Mul->getOperand(Num: 0);
4337 B = Mul->getOperand(Num: 1);
4338 if (A.getOpcode() == ISD::ZERO_EXTEND) {
4339 if (B.getOpcode() == ISD::ZERO_EXTEND)
4340 Opcode = ISD::PARTIAL_REDUCE_UMLA;
4341 else if (B.getOpcode() == ISD::SIGN_EXTEND) {
4342 swap = true;
4343 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4344 } else
4345 return false;
4346 } else if (A.getOpcode() == ISD::SIGN_EXTEND) {
4347 if (B.getOpcode() == ISD::ZERO_EXTEND)
4348 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4349 else if (B.getOpcode() == ISD::SIGN_EXTEND)
4350 Opcode = ISD::PARTIAL_REDUCE_SMLA;
4351 else
4352 return false;
4353 } else
4354 return false;
4355
4356 // Get multiplication arguments before extension.
4357 A = A->getOperand(Num: 0);
4358 B = B->getOperand(Num: 0);
4359 if (A.getValueType() != B.getValueType())
4360 return false;
4361
4362 if (swap)
4363 std::swap(a&: A, b&: B);
4364
4365 return true;
4366}
4367
4368SDValue HexagonTargetLowering::splitVecReduceAdd(SDNode *N,
4369 SelectionDAG &DAG) const {
4370 if (!Subtarget.useHVXOps())
4371 return SDValue();
4372
4373 EVT ScalarType = N->getValueType(ResNo: 0);
4374 unsigned Opcode;
4375 SDValue A, B;
4376 if (!DetectExtendingMultiply(N: N->getOperand(Num: 0), ScalarType, Opcode, A, B))
4377 return SDValue();
4378
4379 SDLoc DL(N);
4380 unsigned RemainingReductionRatio;
4381 SDValue Partial =
4382 createExtendingPartialReduceMLA(Opcode, AccEltType: ScalarType, AccNumElements: 1, InputType: A.getValueType(),
4383 A, B, RemainingReductionRatio, DL, DAG);
4384 if (!Partial)
4385 return SDValue();
4386
4387 // We could have inserted a trivial MLA and rely on the folding action,
4388 // similar to how vector_partial_reduce_add is lowered to an MLA in
4389 // SelectionDAGBuilder. However, we just replace the final result since we
4390 // have analyzed the input completely.
4391 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: ScalarType, Operand: Partial);
4392}
4393
4394// When possible, separate an MLA reduction with extended operands but
4395// unsupported reduction factor into an extending partial reduction that
4396// can be efficiently lowered, and a follow-up partial reduction.
4397// partial_reduce_mla(a, x, y) ->
4398// partial_reduce_mla(a, partial_reduce_mla(0, x, y), 1)
4399SDValue
4400HexagonTargetLowering::splitExtendingPartialReduceMLA(SDNode *N,
4401 SelectionDAG &DAG) const {
4402 if (!Subtarget.useHVXOps())
4403 return SDValue();
4404
4405 SDValue Acc = N->getOperand(Num: 0);
4406 SDValue A = N->getOperand(Num: 1);
4407 SDValue B = N->getOperand(Num: 2);
4408 if (A.getValueType() != B.getValueType())
4409 return SDValue();
4410
4411 // The types should be declared as custom, but do not split already legal
4412 // operation.
4413 EVT AccType = Acc.getValueType();
4414 EVT InputType = A.getValueType();
4415 if (getPartialReduceMLAAction(Opc: N->getOpcode(), AccVT: AccType, InputVT: InputType) != Custom)
4416 return SDValue();
4417
4418 SDLoc DL(N);
4419 unsigned RemainingReductionRatio;
4420 SDValue Partial = createExtendingPartialReduceMLA(
4421 Opcode: N->getOpcode(), AccEltType: AccType.getVectorElementType(),
4422 AccNumElements: AccType.getVectorNumElements(), InputType, A, B, RemainingReductionRatio,
4423 DL, DAG);
4424 if (!Partial)
4425 return SDValue();
4426 assert(RemainingReductionRatio <= MaxExpandMLA);
4427
4428 // Create the reduction for the remaining ratio.
4429 EVT IntermediateType = Partial->getOperand(Num: 0).getValueType();
4430 SDValue One = DAG.getConstant(Val: 1, DL, VT: IntermediateType);
4431 return DAG.getNode(Opcode: N->getOpcode() == ISD::PARTIAL_REDUCE_UMLA
4432 ? ISD::PARTIAL_REDUCE_UMLA
4433 : ISD::PARTIAL_REDUCE_SUMLA,
4434 DL, VT: AccType, N1: Acc, N2: Partial, N3: One);
4435}
4436
4437SDValue
4438HexagonTargetLowering::LowerHvxPartialReduceMLA(SDValue Op,
4439 SelectionDAG &DAG) const {
4440 const SDLoc &DL(Op);
4441 SDValue Acc = Op.getOperand(i: 0);
4442 SDValue A = Op.getOperand(i: 1);
4443 SDValue B = Op.getOperand(i: 2);
4444
4445 // Split the input vectors into units of one HVX vector length.
4446 unsigned HwVectorSizeInBits = Subtarget.getVectorLength() * 8;
4447
4448 EVT AccType = Acc.getValueType();
4449 EVT AccEltType = AccType.getVectorElementType();
4450 unsigned AccSubvectorNumElements =
4451 HwVectorSizeInBits / AccEltType.getSizeInBits();
4452 EVT AccSubvectorType =
4453 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType, NumElements: AccSubvectorNumElements);
4454
4455 EVT InputType = A.getValueType();
4456 assert(InputType.getSizeInBits() % HwVectorSizeInBits == 0);
4457 EVT InputEltType = InputType.getVectorElementType();
4458 unsigned InputSubvectorNumElements =
4459 HwVectorSizeInBits / InputEltType.getSizeInBits();
4460 EVT InputSubvectorType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InputEltType,
4461 NumElements: InputSubvectorNumElements);
4462
4463 unsigned SubvectorNum = InputType.getFixedSizeInBits() / HwVectorSizeInBits;
4464 SmallVector<SDValue, MaxExpandMLA> Subvectors;
4465
4466 for (unsigned I = 0; I != SubvectorNum; ++I) {
4467 SDValue SubvectorAcc = DAG.getExtractSubvector(DL, VT: AccSubvectorType, Vec: Acc,
4468 Idx: I * AccSubvectorNumElements);
4469 SDValue SubvectorA = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: A,
4470 Idx: I * InputSubvectorNumElements);
4471 SDValue SubvectorB = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: B,
4472 Idx: I * InputSubvectorNumElements);
4473 SDValue SubvectorMLA = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: AccSubvectorType,
4474 N1: SubvectorAcc, N2: SubvectorA, N3: SubvectorB);
4475 Subvectors.push_back(Elt: SubvectorMLA);
4476 }
4477
4478 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: AccType, Ops: Subvectors);
4479}
4480
4481// Lower fcmp oeq on HVX float vectors for architectures before v81, which
4482// lack a dedicated floating-point equality instruction.
4483//
4484// Correct IEEE-754 semantics: oeq(a,b) is true iff a==b and neither is NaN.
4485// We use the available float-GT instruction (V6_vgtsf/V6_vgthf) for the
4486// inequality check and bit manipulation for NaN detection:
4487//
4488// oeq(a, b) = NOT(ogt(a,b) OR ogt(b,a) OR isNaN(a) OR isNaN(b))
4489//
4490// where isNaN(x) = ((int_bits(x) & AbsMask) > NaNThreshold)
4491// f32: AbsMask=0x7FFFFFFF, NaNThreshold=0x7F800000
4492// f16: AbsMask=0x7FFF, NaNThreshold=0x7C00
4493//
4494// This handles +0/-0 correctly because float-GT treats them as equal, so
4495// neither ogt(+0,-0) nor ogt(-0,+0) is ever true.
4496//
4497// Example f32 assembly (no NaNs case):
4498// q0 = vcmp.eq(v0.w, v1.w) // bitwise comparison should just work
4499//
4500// Example f32 assembly (NaN-present case):
4501// q0 = vcmp.gt(v0.sf, v1.sf) // ogt(a,b)
4502// q0 |= vcmp.gt(v1.sf, v0.sf) // |= ogt(b,a)
4503// r0 = ##0x7FFFFFFF
4504// v2 = vsplat(r0) // AbsMask broadcast
4505// r1 = ##0x7F800000
4506// v3 = vsplat(r1) // NaNThresh broadcast
4507// v4 = vand(v0, v2) // int_bits(a) & AbsMask
4508// v5 = vand(v1, v2) // int_bits(b) & AbsMask
4509// q0 |= vcmp.gt(v4.w, v3.w) // |= isNaN(a)
4510// q0 |= vcmp.gt(v5.w, v3.w) // |= isNaN(b)
4511// // q0 now holds AnyFalse; result = XOR(q0, allones) = oeq
4512SDValue HexagonTargetLowering::LowerHvxFpSetoeq(SDValue Op,
4513 SelectionDAG &DAG) const {
4514 auto ResTy = ty(Op);
4515 auto A = Op.getOperand(i: 0), B = Op->getOperand(Num: 1);
4516 MVT FloatTy = ty(Op: A);
4517 MVT ElemTy = FloatTy.getVectorElementType();
4518 bool IsF32 = (ElemTy == MVT::f32);
4519 if (!IsF32) {
4520 assert((ElemTy == MVT::f16));
4521 }
4522 const SDLoc &DL(Op);
4523 MVT IntElemTy = IsF32 ? MVT::i32 : MVT::i16;
4524 MVT IntVecTy = tyVector(Ty: FloatTy, ElemTy: IntElemTy);
4525
4526 // Under nnan semantics NaN cannot appear, so integer equality is both
4527 // correct and cheaper (one instruction vs the float-GT sequence).
4528 bool NoNaN = Op->getFlags().hasNoNaNs();
4529 if (NoNaN) {
4530 SDValue IA = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: A);
4531 SDValue IB = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: B);
4532 return DAG.getSetCC(DL, VT: ResTy, LHS: IA, RHS: IB, Cond: ISD::SETEQ);
4533 }
4534
4535 // Float GT comparisons (IEEE-754: false whenever either operand is NaN).
4536 SDValue QAgtB = DAG.getSetCC(DL, VT: ResTy, LHS: A, RHS: B, Cond: ISD::SETOGT);
4537 SDValue QBgtA = DAG.getSetCC(DL, VT: ResTy, LHS: B, RHS: A, Cond: ISD::SETOGT);
4538
4539 // OR all "false" conditions together, then invert.
4540 SDValue AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: QAgtB, N2: QBgtA);
4541
4542 // Detect NaN by checking whether the unbiased exponent/mantissa field
4543 // exceeds the largest finite value.
4544 // f32: (bits & 0x7FFFFFFF) > 0x7F800000
4545 // f16: (bits & 0x7FFF) > 0x7C00
4546 uint64_t AbsMask = IsF32 ? 0x7FFFFFFFull : 0x7FFFull;
4547 uint64_t NaNThresh = IsF32 ? 0x7F800000ull : 0x7C00ull;
4548
4549 SDValue IA = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: A);
4550 SDValue IB = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecTy, Operand: B);
4551 SDValue MaskVec = DAG.getConstant(Val: AbsMask, DL, VT: IntVecTy);
4552 SDValue ThreshVec = DAG.getConstant(Val: NaNThresh, DL, VT: IntVecTy);
4553 SDValue QNanA =
4554 DAG.getSetCC(DL, VT: ResTy, LHS: DAG.getNode(Opcode: ISD::AND, DL, VT: IntVecTy, N1: IA, N2: MaskVec),
4555 RHS: ThreshVec, Cond: ISD::SETGT);
4556 SDValue QNanB =
4557 DAG.getSetCC(DL, VT: ResTy, LHS: DAG.getNode(Opcode: ISD::AND, DL, VT: IntVecTy, N1: IB, N2: MaskVec),
4558 RHS: ThreshVec, Cond: ISD::SETGT);
4559 AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: AnyFalse, N2: QNanA);
4560 AnyFalse = DAG.getNode(Opcode: ISD::OR, DL, VT: ResTy, N1: AnyFalse, N2: QNanB);
4561
4562 // Result = NOT(<Is A gt B>, <IS B gt A>, <IS A NaN>, <IS B NaN>)
4563 // Use XOR with ones to simulate logical not.
4564 return DAG.getNode(Opcode: ISD::XOR, DL, VT: ResTy, N1: AnyFalse,
4565 N2: DAG.getConstant(Val: 1, DL, VT: ResTy));
4566}
4567
4568SDValue
4569HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
4570 const {
4571 const SDLoc &dl(N);
4572 SelectionDAG &DAG = DCI.DAG;
4573 SDValue Op(N, 0);
4574 unsigned Opc = Op.getOpcode();
4575
4576 SmallVector<SDValue, 4> Ops(N->ops());
4577
4578 if (Opc == ISD::TRUNCATE)
4579 return combineTruncateBeforeLegal(Op, DCI);
4580 if (Opc == ISD::CONCAT_VECTORS)
4581 return combineConcatVectorsBeforeLegal(Op, DCI);
4582
4583 if (DCI.isBeforeLegalizeOps())
4584 return SDValue();
4585
4586 switch (Opc) {
4587 case HexagonISD::V2Q:
4588 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) {
4589 if (const auto *C = dyn_cast<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
4590 return C->isZero() ? DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: ty(Op))
4591 : DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: ty(Op));
4592 }
4593 break;
4594 case HexagonISD::Q2V:
4595 if (Ops[0].getOpcode() == HexagonISD::QTRUE)
4596 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ty(Op),
4597 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
4598 if (Ops[0].getOpcode() == HexagonISD::QFALSE)
4599 return getZero(dl, Ty: ty(Op), DAG);
4600 break;
4601 case HexagonISD::VINSERTW0:
4602 if (isUndef(Op: Ops[1]))
4603 return Ops[0];
4604 break;
4605 case HexagonISD::VROR: {
4606 if (Ops[0].getOpcode() == HexagonISD::VROR) {
4607 SDValue Vec = Ops[0].getOperand(i: 0);
4608 SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(i: 1);
4609 SDValue Rot = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ty(Op: Rot0), Ops: {Rot0, Rot1});
4610 return DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ty(Op), Ops: {Vec, Rot});
4611 }
4612 break;
4613 }
4614 }
4615
4616 return SDValue();
4617}
4618
4619bool
4620HexagonTargetLowering::shouldSplitToHvx(MVT Ty, SelectionDAG &DAG) const {
4621 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4622 return false;
4623 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4624 if (Action == TargetLoweringBase::TypeSplitVector)
4625 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4626 return false;
4627}
4628
4629bool
4630HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const {
4631 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4632 return false;
4633 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4634 if (Action == TargetLoweringBase::TypeWidenVector)
4635 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4636 return false;
4637}
4638
4639bool
4640HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const {
4641 if (!Subtarget.useHVXOps())
4642 return false;
4643 // If the type of any result, or any operand type are HVX vector types,
4644 // this is an HVX operation.
4645 auto IsHvxTy = [this](EVT Ty) {
4646 return Ty.isSimple() && Subtarget.isHVXVectorType(VecTy: Ty.getSimpleVT(), IncludeBool: true);
4647 };
4648 auto IsHvxOp = [this](SDValue Op) {
4649 return Op.getValueType().isSimple() &&
4650 Subtarget.isHVXVectorType(VecTy: ty(Op), IncludeBool: true);
4651 };
4652 if (llvm::any_of(Range: N->values(), P: IsHvxTy) || llvm::any_of(Range: N->ops(), P: IsHvxOp))
4653 return true;
4654
4655 // Check if this could be an HVX operation after type widening.
4656 auto IsWidenedToHvx = [this, &DAG](SDValue Op) {
4657 if (!Op.getValueType().isSimple())
4658 return false;
4659 MVT ValTy = ty(Op);
4660 return ValTy.isVector() && shouldWidenToHvx(Ty: ValTy, DAG);
4661 };
4662
4663 for (int i = 0, e = N->getNumValues(); i != e; ++i) {
4664 if (IsWidenedToHvx(SDValue(N, i)))
4665 return true;
4666 }
4667 return llvm::any_of(Range: N->ops(), P: IsWidenedToHvx);
4668}
4669