1//===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "HexagonISelLowering.h"
10#include "HexagonRegisterInfo.h"
11#include "HexagonSubtarget.h"
12#include "llvm/ADT/SetVector.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/MemoryLocation.h"
15#include "llvm/CodeGen/MachineBasicBlock.h"
16#include "llvm/CodeGen/MachineFunction.h"
17#include "llvm/CodeGen/MachineInstr.h"
18#include "llvm/CodeGen/MachineOperand.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/TargetInstrInfo.h"
21#include "llvm/IR/IntrinsicsHexagon.h"
22#include "llvm/Support/CommandLine.h"
23
24#include <algorithm>
25#include <string>
26#include <utility>
27
28using namespace llvm;
29
30static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
31 cl::Hidden, cl::init(Val: 16),
32 cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
33
34static cl::opt<bool>
35 EnableFpFastConvert("hexagon-fp-fast-convert", cl::Hidden, cl::init(Val: false),
36 cl::desc("Enable FP fast conversion routine."));
37
38static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
39static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
40static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
41static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
42
43static const unsigned MaxExpandMLA = 8;
44
45static std::tuple<unsigned, unsigned, unsigned> getIEEEProperties(MVT Ty) {
46 // For a float scalar type, return (exp-bits, exp-bias, fraction-bits)
47 MVT ElemTy = Ty.getScalarType();
48 switch (ElemTy.SimpleTy) {
49 case MVT::f16:
50 return std::make_tuple(args: 5, args: 15, args: 10);
51 case MVT::f32:
52 return std::make_tuple(args: 8, args: 127, args: 23);
53 case MVT::f64:
54 return std::make_tuple(args: 11, args: 1023, args: 52);
55 default:
56 break;
57 }
58 llvm_unreachable(("Unexpected type: " + EVT(ElemTy).getEVTString()).c_str());
59}
60
61void
62HexagonTargetLowering::initializeHVXLowering() {
63 if (Subtarget.useHVX64BOps()) {
64 addRegisterClass(VT: MVT::v64i8, RC: &Hexagon::HvxVRRegClass);
65 addRegisterClass(VT: MVT::v32i16, RC: &Hexagon::HvxVRRegClass);
66 addRegisterClass(VT: MVT::v16i32, RC: &Hexagon::HvxVRRegClass);
67 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxWRRegClass);
68 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxWRRegClass);
69 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxWRRegClass);
70 // These "short" boolean vector types should be legal because
71 // they will appear as results of vector compares. If they were
72 // not legal, type legalization would try to make them legal
73 // and that would require using operations that do not use or
74 // produce such types. That, in turn, would imply using custom
75 // nodes, which would be unoptimizable by the DAG combiner.
76 // The idea is to rely on target-independent operations as much
77 // as possible.
78 addRegisterClass(VT: MVT::v16i1, RC: &Hexagon::HvxQRRegClass);
79 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
80 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
81 } else if (Subtarget.useHVX128BOps()) {
82 addRegisterClass(VT: MVT::v128i8, RC: &Hexagon::HvxVRRegClass);
83 addRegisterClass(VT: MVT::v64i16, RC: &Hexagon::HvxVRRegClass);
84 addRegisterClass(VT: MVT::v32i32, RC: &Hexagon::HvxVRRegClass);
85 addRegisterClass(VT: MVT::v256i8, RC: &Hexagon::HvxWRRegClass);
86 addRegisterClass(VT: MVT::v128i16, RC: &Hexagon::HvxWRRegClass);
87 addRegisterClass(VT: MVT::v64i32, RC: &Hexagon::HvxWRRegClass);
88 addRegisterClass(VT: MVT::v32i1, RC: &Hexagon::HvxQRRegClass);
89 addRegisterClass(VT: MVT::v64i1, RC: &Hexagon::HvxQRRegClass);
90 addRegisterClass(VT: MVT::v128i1, RC: &Hexagon::HvxQRRegClass);
91 if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) {
92 addRegisterClass(VT: MVT::v32f32, RC: &Hexagon::HvxVRRegClass);
93 addRegisterClass(VT: MVT::v64f16, RC: &Hexagon::HvxVRRegClass);
94 addRegisterClass(VT: MVT::v64f32, RC: &Hexagon::HvxWRRegClass);
95 addRegisterClass(VT: MVT::v128f16, RC: &Hexagon::HvxWRRegClass);
96 }
97 if (Subtarget.useHVXV81Ops()) {
98 addRegisterClass(VT: MVT::v64bf16, RC: &Hexagon::HvxVRRegClass);
99 addRegisterClass(VT: MVT::v128bf16, RC: &Hexagon::HvxWRRegClass);
100 }
101 }
102
103 // Set up operation actions.
104
105 bool Use64b = Subtarget.useHVX64BOps();
106 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
107 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
108 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
109 MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32;
110 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
111
112 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
113 setOperationAction(Op: Opc, VT: FromTy, Action: Promote);
114 AddPromotedToType(Opc, OrigVT: FromTy, DestVT: ToTy);
115 };
116
117 // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
118 // Note: v16i1 -> i16 is handled in type legalization instead of op
119 // legalization.
120 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
121 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
122 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
123 setOperationAction(Op: ISD::BITCAST, VT: MVT::v16i1, Action: Custom);
124 setOperationAction(Op: ISD::BITCAST, VT: MVT::v128i1, Action: Custom);
125 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
126 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteV, Action: Legal);
127 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: ByteW, Action: Legal);
128 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
129
130 if (Subtarget.useHVX128BOps()) {
131 setOperationAction(Op: ISD::BITCAST, VT: MVT::v32i1, Action: Custom);
132 setOperationAction(Op: ISD::BITCAST, VT: MVT::v64i1, Action: Custom);
133 setOperationAction(Op: ISD::STORE, VT: MVT::v32i1, Action: Custom);
134 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i1, Action: Custom);
135 setOperationAction(Op: ISD::STORE, VT: MVT::v64i1, Action: Custom);
136 setOperationAction(Op: ISD::LOAD, VT: MVT::v64i1, Action: Custom);
137 setOperationAction(Op: ISD::STORE, VT: MVT::v128i1, Action: Custom);
138 setOperationAction(Op: ISD::LOAD, VT: MVT::v128i1, Action: Custom);
139 }
140 if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() &&
141 Subtarget.useHVXFloatingPoint()) {
142
143 static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 };
144 static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 };
145
146 for (MVT T : FloatV) {
147 setOperationAction(Op: ISD::FADD, VT: T, Action: Legal);
148 setOperationAction(Op: ISD::FSUB, VT: T, Action: Legal);
149 setOperationAction(Op: ISD::FMUL, VT: T, Action: Legal);
150 setOperationAction(Op: ISD::FMINIMUMNUM, VT: T, Action: Legal);
151 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: T, Action: Legal);
152 setOperationAction(Op: ISD::FMINNUM, VT: T, Action: Legal);
153 setOperationAction(Op: ISD::FMAXNUM, VT: T, Action: Legal);
154
155 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
156 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
157
158 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
159 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
160
161 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
162 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
163 // Custom-lower BUILD_VECTOR. The standard (target-independent)
164 // handling of it would convert it to a load, which is not always
165 // the optimal choice.
166 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
167 }
168
169
170 // BUILD_VECTOR with f16 operands cannot be promoted without
171 // promoting the result, so lower the node to vsplat or constant pool
172 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
173 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::f16, Action: Custom);
174 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::f16, Action: Custom);
175
176 // Vector shuffle is always promoted to ByteV and a bitcast to f16 is
177 // generated.
178 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128f16, ByteW);
179 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f16, ByteV);
180 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f32, ByteW);
181 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v32f32, ByteV);
182
183 if (Subtarget.useHVXV81Ops()) {
184 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128bf16, ByteW);
185 setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64bf16, ByteV);
186 setPromoteTo(ISD::SETCC, MVT::v64bf16, MVT::v64f32);
187 setPromoteTo(ISD::FADD, MVT::v64bf16, MVT::v64f32);
188 setPromoteTo(ISD::FSUB, MVT::v64bf16, MVT::v64f32);
189 setPromoteTo(ISD::FMUL, MVT::v64bf16, MVT::v64f32);
190 setPromoteTo(ISD::FMINNUM, MVT::v64bf16, MVT::v64f32);
191 setPromoteTo(ISD::FMAXNUM, MVT::v64bf16, MVT::v64f32);
192
193 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::v64bf16, Action: Legal);
194 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
195 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: MVT::v64bf16, Action: Custom);
196
197 setOperationAction(Op: ISD::LOAD, VT: MVT::v128bf16, Action: Custom);
198 setOperationAction(Op: ISD::STORE, VT: MVT::v128bf16, Action: Custom);
199
200 setOperationAction(Op: ISD::MLOAD, VT: MVT::v64bf16, Action: Custom);
201 setOperationAction(Op: ISD::MSTORE, VT: MVT::v64bf16, Action: Custom);
202 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v64bf16, Action: Custom);
203 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v64bf16, Action: Custom);
204
205 setOperationAction(Op: ISD::MLOAD, VT: MVT::v128bf16, Action: Custom);
206 setOperationAction(Op: ISD::MSTORE, VT: MVT::v128bf16, Action: Custom);
207 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v128bf16, Action: Custom);
208 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: MVT::v128bf16, Action: Custom);
209
210 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: MVT::bf16, Action: Custom);
211 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::bf16, Action: Custom);
212 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::bf16, Action: Custom);
213 }
214
215 for (MVT P : FloatW) {
216 setOperationAction(Op: ISD::LOAD, VT: P, Action: Custom);
217 setOperationAction(Op: ISD::STORE, VT: P, Action: Custom);
218 setOperationAction(Op: ISD::FADD, VT: P, Action: Custom);
219 setOperationAction(Op: ISD::FSUB, VT: P, Action: Custom);
220 setOperationAction(Op: ISD::FMUL, VT: P, Action: Custom);
221 setOperationAction(Op: ISD::FMINIMUMNUM, VT: P, Action: Custom);
222 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: P, Action: Custom);
223 setOperationAction(Op: ISD::FMINNUM, VT: P, Action: Custom);
224 setOperationAction(Op: ISD::FMAXNUM, VT: P, Action: Custom);
225 setOperationAction(Op: ISD::SETCC, VT: P, Action: Custom);
226 setOperationAction(Op: ISD::VSELECT, VT: P, Action: Custom);
227
228 // Custom-lower BUILD_VECTOR. The standard (target-independent)
229 // handling of it would convert it to a load, which is not always
230 // the optimal choice.
231 setOperationAction(Op: ISD::BUILD_VECTOR, VT: P, Action: Custom);
232 // Make concat-vectors custom to handle concats of more than 2 vectors.
233 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: P, Action: Custom);
234
235 setOperationAction(Op: ISD::MLOAD, VT: P, Action: Custom);
236 setOperationAction(Op: ISD::MSTORE, VT: P, Action: Custom);
237 }
238
239 if (Subtarget.useHVXQFloatOps()) {
240 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Custom);
241 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
242 } else if (Subtarget.useHVXIEEEFPOps()) {
243 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v64f32, Action: Legal);
244 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v64f16, Action: Legal);
245 }
246 }
247
248 for (MVT T : LegalV) {
249 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
250 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: T, Action: Legal);
251
252 setOperationAction(Op: ISD::ABS, VT: T, Action: Legal);
253 setOperationAction(Op: ISD::AND, VT: T, Action: Legal);
254 setOperationAction(Op: ISD::OR, VT: T, Action: Legal);
255 setOperationAction(Op: ISD::XOR, VT: T, Action: Legal);
256 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
257 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
258 setOperationAction(Op: ISD::MUL, VT: T, Action: Legal);
259 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Legal);
260 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Legal);
261 setOperationAction(Op: ISD::SELECT, VT: T, Action: Legal);
262 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
263 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
264 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
265 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
266 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
267 if (T != ByteV) {
268 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
269 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
270 setOperationAction(Op: ISD::BSWAP, VT: T, Action: Legal);
271 }
272
273 setOperationAction(Op: ISD::SMIN, VT: T, Action: Legal);
274 setOperationAction(Op: ISD::SMAX, VT: T, Action: Legal);
275 if (T.getScalarType() != MVT::i32) {
276 setOperationAction(Op: ISD::UMIN, VT: T, Action: Legal);
277 setOperationAction(Op: ISD::UMAX, VT: T, Action: Legal);
278 }
279
280 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
281 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
282 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
283 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
284 if (T.getScalarType() != MVT::i32) {
285 setOperationAction(Op: ISD::MULHS, VT: T, Action: Legal);
286 setOperationAction(Op: ISD::MULHU, VT: T, Action: Legal);
287 }
288
289 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
290 // Make concat-vectors custom to handle concats of more than 2 vectors.
291 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
292 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: T, Action: Custom);
293 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: T, Action: Custom);
294 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: T, Action: Custom);
295 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: T, Action: Custom);
296 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
297 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
298 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
299 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
300 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
301 if (T != ByteV) {
302 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
303 // HVX only has shifts of words and halfwords.
304 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
305 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
306 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
307
308 // Promote all shuffles to operate on vectors of bytes.
309 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
310 }
311
312 if (Subtarget.useHVXFloatingPoint()) {
313 // Same action for both QFloat and IEEE.
314 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
315 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
316 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
317 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
318 }
319
320 setCondCodeAction(CCs: ISD::SETNE, VT: T, Action: Expand);
321 setCondCodeAction(CCs: ISD::SETLE, VT: T, Action: Expand);
322 setCondCodeAction(CCs: ISD::SETGE, VT: T, Action: Expand);
323 setCondCodeAction(CCs: ISD::SETLT, VT: T, Action: Expand);
324 setCondCodeAction(CCs: ISD::SETULE, VT: T, Action: Expand);
325 setCondCodeAction(CCs: ISD::SETUGE, VT: T, Action: Expand);
326 setCondCodeAction(CCs: ISD::SETULT, VT: T, Action: Expand);
327 }
328
329 for (MVT T : LegalW) {
330 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
331 // independent) handling of it would convert it to a load, which is
332 // not always the optimal choice.
333 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
334 // Make concat-vectors custom to handle concats of more than 2 vectors.
335 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: T, Action: Custom);
336
337 // Custom-lower these operations for pairs. Expand them into a concat
338 // of the corresponding operations on individual vectors.
339 setOperationAction(Op: ISD::ANY_EXTEND, VT: T, Action: Custom);
340 setOperationAction(Op: ISD::SIGN_EXTEND, VT: T, Action: Custom);
341 setOperationAction(Op: ISD::ZERO_EXTEND, VT: T, Action: Custom);
342 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Custom);
343 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
344 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
345 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Legal);
346 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Custom);
347
348 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
349 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
350 setOperationAction(Op: ISD::MLOAD, VT: T, Action: Custom);
351 setOperationAction(Op: ISD::MSTORE, VT: T, Action: Custom);
352 setOperationAction(Op: ISD::ABS, VT: T, Action: Custom);
353 setOperationAction(Op: ISD::CTLZ, VT: T, Action: Custom);
354 setOperationAction(Op: ISD::CTTZ, VT: T, Action: Custom);
355 setOperationAction(Op: ISD::CTPOP, VT: T, Action: Custom);
356
357 setOperationAction(Op: ISD::ADD, VT: T, Action: Legal);
358 setOperationAction(Op: ISD::UADDSAT, VT: T, Action: Legal);
359 setOperationAction(Op: ISD::SADDSAT, VT: T, Action: Legal);
360 setOperationAction(Op: ISD::SUB, VT: T, Action: Legal);
361 setOperationAction(Op: ISD::USUBSAT, VT: T, Action: Legal);
362 setOperationAction(Op: ISD::SSUBSAT, VT: T, Action: Legal);
363 setOperationAction(Op: ISD::MUL, VT: T, Action: Custom);
364 setOperationAction(Op: ISD::MULHS, VT: T, Action: Custom);
365 setOperationAction(Op: ISD::MULHU, VT: T, Action: Custom);
366 setOperationAction(Op: ISD::AND, VT: T, Action: Custom);
367 setOperationAction(Op: ISD::OR, VT: T, Action: Custom);
368 setOperationAction(Op: ISD::XOR, VT: T, Action: Custom);
369 setOperationAction(Op: ISD::SETCC, VT: T, Action: Custom);
370 setOperationAction(Op: ISD::VSELECT, VT: T, Action: Custom);
371 if (T != ByteW) {
372 setOperationAction(Op: ISD::SRA, VT: T, Action: Custom);
373 setOperationAction(Op: ISD::SHL, VT: T, Action: Custom);
374 setOperationAction(Op: ISD::SRL, VT: T, Action: Custom);
375
376 // Promote all shuffles to operate on vectors of bytes.
377 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
378 }
379 setOperationAction(Op: ISD::FSHL, VT: T, Action: Custom);
380 setOperationAction(Op: ISD::FSHR, VT: T, Action: Custom);
381
382 setOperationAction(Op: ISD::SMIN, VT: T, Action: Custom);
383 setOperationAction(Op: ISD::SMAX, VT: T, Action: Custom);
384 if (T.getScalarType() != MVT::i32) {
385 setOperationAction(Op: ISD::UMIN, VT: T, Action: Custom);
386 setOperationAction(Op: ISD::UMAX, VT: T, Action: Custom);
387 }
388
389 if (Subtarget.useHVXFloatingPoint()) {
390 // Same action for both QFloat and IEEE.
391 setOperationAction(Op: ISD::SINT_TO_FP, VT: T, Action: Custom);
392 setOperationAction(Op: ISD::UINT_TO_FP, VT: T, Action: Custom);
393 setOperationAction(Op: ISD::FP_TO_SINT, VT: T, Action: Custom);
394 setOperationAction(Op: ISD::FP_TO_UINT, VT: T, Action: Custom);
395 }
396 }
397
398 // Legalize all of these to HexagonISD::[SU]MUL_LOHI.
399 setOperationAction(Op: ISD::MULHS, VT: WordV, Action: Custom); // -> _LOHI
400 setOperationAction(Op: ISD::MULHU, VT: WordV, Action: Custom); // -> _LOHI
401 setOperationAction(Op: ISD::SMUL_LOHI, VT: WordV, Action: Custom);
402 setOperationAction(Op: ISD::UMUL_LOHI, VT: WordV, Action: Custom);
403
404 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v64f16, Action: Expand);
405 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v64f16, Action: Expand);
406 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v64f16, Action: Expand);
407 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v64f16, Action: Expand);
408 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v64f16, Action: Expand);
409 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v64f16, Action: Expand);
410 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v64f16, Action: Expand);
411 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v64f16, Action: Expand);
412 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v64f16, Action: Expand);
413 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v64f16, Action: Expand);
414 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v64f16, Action: Expand);
415 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v64f16, Action: Expand);
416 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v64f16, Action: Expand);
417 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v64f16, Action: Expand);
418
419 setCondCodeAction(CCs: ISD::SETNE, VT: MVT::v32f32, Action: Expand);
420 setCondCodeAction(CCs: ISD::SETLE, VT: MVT::v32f32, Action: Expand);
421 setCondCodeAction(CCs: ISD::SETGE, VT: MVT::v32f32, Action: Expand);
422 setCondCodeAction(CCs: ISD::SETLT, VT: MVT::v32f32, Action: Expand);
423 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v32f32, Action: Expand);
424 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::v32f32, Action: Expand);
425 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::v32f32, Action: Expand);
426 setCondCodeAction(CCs: ISD::SETOLT, VT: MVT::v32f32, Action: Expand);
427 setCondCodeAction(CCs: ISD::SETUNE, VT: MVT::v32f32, Action: Expand);
428 setCondCodeAction(CCs: ISD::SETULE, VT: MVT::v32f32, Action: Expand);
429 setCondCodeAction(CCs: ISD::SETUGE, VT: MVT::v32f32, Action: Expand);
430 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::v32f32, Action: Expand);
431 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v32f32, Action: Expand);
432 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v32f32, Action: Expand);
433
434 // Boolean vectors.
435
436 for (MVT T : LegalW) {
437 // Boolean types for vector pairs will overlap with the boolean
438 // types for single vectors, e.g.
439 // v64i8 -> v64i1 (single)
440 // v64i16 -> v64i1 (pair)
441 // Set these actions first, and allow the single actions to overwrite
442 // any duplicates.
443 MVT BoolW = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
444 setOperationAction(Op: ISD::SETCC, VT: BoolW, Action: Custom);
445 setOperationAction(Op: ISD::AND, VT: BoolW, Action: Custom);
446 setOperationAction(Op: ISD::OR, VT: BoolW, Action: Custom);
447 setOperationAction(Op: ISD::XOR, VT: BoolW, Action: Custom);
448 // Masked load/store takes a mask that may need splitting.
449 setOperationAction(Op: ISD::MLOAD, VT: BoolW, Action: Custom);
450 setOperationAction(Op: ISD::MSTORE, VT: BoolW, Action: Custom);
451 }
452
453 for (MVT T : LegalV) {
454 MVT BoolV = MVT::getVectorVT(VT: MVT::i1, NumElements: T.getVectorNumElements());
455 setOperationAction(Op: ISD::BUILD_VECTOR, VT: BoolV, Action: Custom);
456 setOperationAction(Op: ISD::CONCAT_VECTORS, VT: BoolV, Action: Custom);
457 setOperationAction(Op: ISD::INSERT_SUBVECTOR, VT: BoolV, Action: Custom);
458 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: BoolV, Action: Custom);
459 setOperationAction(Op: ISD::EXTRACT_SUBVECTOR, VT: BoolV, Action: Custom);
460 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: BoolV, Action: Custom);
461 setOperationAction(Op: ISD::SELECT, VT: BoolV, Action: Custom);
462 setOperationAction(Op: ISD::AND, VT: BoolV, Action: Legal);
463 setOperationAction(Op: ISD::OR, VT: BoolV, Action: Legal);
464 setOperationAction(Op: ISD::XOR, VT: BoolV, Action: Legal);
465 }
466
467 if (Use64b) {
468 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
469 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
470 } else {
471 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
472 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Legal);
473 }
474
475 // Handle store widening for short vectors.
476 unsigned HwLen = Subtarget.getVectorLength();
477 for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
478 if (ElemTy == MVT::i1)
479 continue;
480 int ElemWidth = ElemTy.getFixedSizeInBits();
481 int MaxElems = (8*HwLen) / ElemWidth;
482 for (int N = 2; N < MaxElems; N *= 2) {
483 MVT VecTy = MVT::getVectorVT(VT: ElemTy, NumElements: N);
484 auto Action = getPreferredVectorAction(VT: VecTy);
485 if (Action == TargetLoweringBase::TypeWidenVector) {
486 setOperationAction(Op: ISD::LOAD, VT: VecTy, Action: Custom);
487 setOperationAction(Op: ISD::STORE, VT: VecTy, Action: Custom);
488 setOperationAction(Op: ISD::SETCC, VT: VecTy, Action: Custom);
489 setOperationAction(Op: ISD::TRUNCATE, VT: VecTy, Action: Custom);
490 setOperationAction(Op: ISD::ANY_EXTEND, VT: VecTy, Action: Custom);
491 setOperationAction(Op: ISD::SIGN_EXTEND, VT: VecTy, Action: Custom);
492 setOperationAction(Op: ISD::ZERO_EXTEND, VT: VecTy, Action: Custom);
493 if (Subtarget.useHVXFloatingPoint()) {
494 setOperationAction(Op: ISD::FP_TO_SINT, VT: VecTy, Action: Custom);
495 setOperationAction(Op: ISD::FP_TO_UINT, VT: VecTy, Action: Custom);
496 setOperationAction(Op: ISD::SINT_TO_FP, VT: VecTy, Action: Custom);
497 setOperationAction(Op: ISD::UINT_TO_FP, VT: VecTy, Action: Custom);
498 }
499
500 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: N);
501 if (!isTypeLegal(VT: BoolTy))
502 setOperationAction(Op: ISD::SETCC, VT: BoolTy, Action: Custom);
503 }
504 }
505 }
506
507 // Include cases which are not hander earlier
508 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v32i1, Action: Custom);
509 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v64i1, Action: Custom);
510 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v32i1, Action: Custom);
511
512 setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT});
513
514 setTargetDAGCombine({ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA,
515 ISD::PARTIAL_REDUCE_SUMLA});
516
517 // Partial MLA reductions.
518 {
519 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
520 ISD::PARTIAL_REDUCE_UMLA,
521 ISD::PARTIAL_REDUCE_SUMLA};
522
523 auto HvxType = [=](MVT ScalarT, unsigned Factor = 1) {
524 return MVT::getVectorVT(VT: ScalarT, NumElements: Subtarget.getVectorLength() * Factor *
525 8 / ScalarT.getSizeInBits());
526 };
527
528 // Tuple of (Acc element type, input element type, vector pair).
529 // The assumption is both the input and reduction result are of the same
530 // size so the reduction ratio is the same as the ratio of element type
531 // sizes. This may not hold for all available instructions.
532 typedef std::tuple<MVT, MVT, bool> ReductionSignature;
533
534 static const std::vector<ReductionSignature> NativeReductions = {
535 {MVT::i32, MVT::i8, false},
536 };
537
538 for (const auto &R : NativeReductions) {
539
540 MVT AccType = std::get<0>(t: R);
541 MVT InputType = std::get<1>(t: R);
542 unsigned Factor = std::get<2>(t: R) ? 2 : 1;
543
544 // The native size is legal.
545 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: HvxType(AccType), InputVT: HvxType(InputType),
546 Action: Legal);
547
548 // Allow custom partial MLA reductions on larger vectors than legally
549 // supported. These reduction must be declared as Custom (or Legal)
550 // for foldPartialReduceMLAMulOp() to fold the multiply by one pattern
551 // inserted when the partial reduction intrinsic is converted to
552 // PARTIAL_REDUCE_U/S/SUMLA. Otherwise, the Split action will apply
553 // on the original pattern, including the extensions and multiplies,
554 // which will make it impossible to match.
555 // There are two independent ways to extend the
556 // input size: 1. to concatenate the result - output vector is
557 // proportionally extended, 2) to reduce the result - the output vector
558 // size stays the same. We limit allowed combinations so that the total
559 // number of generated reduction instructions is limited by a constant
560 // number. This limit is arbitrary and can be revised. On one hand, it is
561 // convenient to have more choices; on the other hand, there is a
562 // diminishing benefit of very long sequences, which should probably be
563 // written as loops instead.
564 for (unsigned ConcatFactor = 1; ConcatFactor <= MaxExpandMLA;
565 ConcatFactor <<= 1)
566 for (unsigned ReductionFactor = 1; ReductionFactor <= MaxExpandMLA;
567 ReductionFactor <<= 1)
568 if (ConcatFactor * ReductionFactor != 1 &&
569 ConcatFactor * ReductionFactor <= MaxExpandMLA)
570 setPartialReduceMLAAction(
571 Opcodes: MLAOps, AccVT: HvxType(AccType, Factor * ConcatFactor),
572 InputVT: HvxType(InputType, Factor * ConcatFactor * ReductionFactor),
573 Action: Custom);
574 }
575 }
576}
577
578unsigned
579HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
580 // Early exit for invalid input types
581 if (!VecTy.isVector())
582 return ~0u;
583
584 MVT ElemTy = VecTy.getVectorElementType();
585 unsigned VecLen = VecTy.getVectorNumElements();
586 unsigned HwLen = Subtarget.getVectorLength();
587
588 // Split vectors of i1 that exceed byte vector length.
589 if (ElemTy == MVT::i1 && VecLen > HwLen)
590 return TargetLoweringBase::TypeSplitVector;
591
592 ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
593 // For shorter vectors of i1, widen them if any of the corresponding
594 // vectors of integers needs to be widened.
595 if (ElemTy == MVT::i1) {
596 for (MVT T : Tys) {
597 assert(T != MVT::i1);
598 auto A = getPreferredHvxVectorAction(VecTy: MVT::getVectorVT(VT: T, NumElements: VecLen));
599 if (A != ~0u)
600 return A;
601 }
602 return ~0u;
603 }
604
605 // If the size of VecTy is at least half of the vector length,
606 // widen the vector. Note: the threshold was not selected in
607 // any scientific way.
608 if (llvm::is_contained(Range&: Tys, Element: ElemTy)) {
609 unsigned VecWidth = VecTy.getSizeInBits();
610 unsigned HwWidth = 8*HwLen;
611 if (VecWidth > 2*HwWidth)
612 return TargetLoweringBase::TypeSplitVector;
613
614 bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0;
615 if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth)
616 return TargetLoweringBase::TypeWidenVector;
617 if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
618 return TargetLoweringBase::TypeWidenVector;
619 }
620
621 // Defer to default.
622 return ~0u;
623}
624
625unsigned
626HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const {
627 unsigned Opc = Op.getOpcode();
628 switch (Opc) {
629 case HexagonISD::SMUL_LOHI:
630 case HexagonISD::UMUL_LOHI:
631 case HexagonISD::USMUL_LOHI:
632 return TargetLoweringBase::Custom;
633 }
634 return TargetLoweringBase::Legal;
635}
636
637SDValue
638HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
639 const SDLoc &dl, SelectionDAG &DAG) const {
640 SmallVector<SDValue,4> IntOps;
641 IntOps.push_back(Elt: DAG.getConstant(Val: IntId, DL: dl, VT: MVT::i32));
642 append_range(C&: IntOps, R&: Ops);
643 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: ResTy, Ops: IntOps);
644}
645
646MVT
647HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
648 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
649
650 MVT ElemTy = Tys.first.getVectorElementType();
651 return MVT::getVectorVT(VT: ElemTy, NumElements: Tys.first.getVectorNumElements() +
652 Tys.second.getVectorNumElements());
653}
654
655HexagonTargetLowering::TypePair
656HexagonTargetLowering::typeSplit(MVT VecTy) const {
657 assert(VecTy.isVector());
658 unsigned NumElem = VecTy.getVectorNumElements();
659 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
660 MVT HalfTy = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: NumElem/2);
661 return { HalfTy, HalfTy };
662}
663
664MVT
665HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
666 MVT ElemTy = VecTy.getVectorElementType();
667 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() * Factor);
668 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
669}
670
671MVT
672HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
673 MVT ElemTy = VecTy.getVectorElementType();
674 MVT NewElemTy = MVT::getIntegerVT(BitWidth: ElemTy.getSizeInBits() / Factor);
675 return MVT::getVectorVT(VT: NewElemTy, NumElements: VecTy.getVectorNumElements());
676}
677
678SDValue
679HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
680 SelectionDAG &DAG) const {
681 if (ty(Op: Vec).getVectorElementType() == ElemTy)
682 return Vec;
683 MVT CastTy = tyVector(Ty: Vec.getValueType().getSimpleVT(), ElemTy);
684 return DAG.getBitcast(VT: CastTy, V: Vec);
685}
686
687SDValue
688HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
689 SelectionDAG &DAG) const {
690 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: typeJoin(Tys: ty(Ops)),
691 N1: Ops.first, N2: Ops.second);
692}
693
694HexagonTargetLowering::VectorPair
695HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
696 SelectionDAG &DAG) const {
697 TypePair Tys = typeSplit(VecTy: ty(Op: Vec));
698 if (Vec.getOpcode() == HexagonISD::QCAT)
699 return VectorPair(Vec.getOperand(i: 0), Vec.getOperand(i: 1));
700 return DAG.SplitVector(N: Vec, DL: dl, LoVT: Tys.first, HiVT: Tys.second);
701}
702
703bool
704HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
705 return Subtarget.isHVXVectorType(VecTy: Ty) &&
706 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
707}
708
709bool
710HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
711 return Subtarget.isHVXVectorType(VecTy: Ty) &&
712 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
713}
714
715bool
716HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
717 return Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true) &&
718 Ty.getVectorElementType() == MVT::i1;
719}
720
721bool HexagonTargetLowering::allowsHvxMemoryAccess(
722 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
723 // Bool vectors are excluded by default, but make it explicit to
724 // emphasize that bool vectors cannot be loaded or stored.
725 // Also, disallow double vector stores (to prevent unnecessary
726 // store widening in DAG combiner).
727 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
728 return false;
729 if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
730 return false;
731 if (Fast)
732 *Fast = 1;
733 return true;
734}
735
736bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
737 MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
738 if (!Subtarget.isHVXVectorType(VecTy))
739 return false;
740 // XXX Should this be false? vmemu are a bit slower than vmem.
741 if (Fast)
742 *Fast = 1;
743 return true;
744}
745
746void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection(
747 MachineInstr &MI, SDNode *Node) const {
748 unsigned Opc = MI.getOpcode();
749 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
750 MachineBasicBlock &MB = *MI.getParent();
751 MachineFunction &MF = *MB.getParent();
752 MachineRegisterInfo &MRI = MF.getRegInfo();
753 DebugLoc DL = MI.getDebugLoc();
754 auto At = MI.getIterator();
755
756 switch (Opc) {
757 case Hexagon::PS_vsplatib:
758 if (Subtarget.useHVXV62Ops()) {
759 // SplatV = A2_tfrsi #imm
760 // OutV = V6_lvsplatb SplatV
761 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
762 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
763 .add(MO: MI.getOperand(i: 1));
764 Register OutV = MI.getOperand(i: 0).getReg();
765 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
766 .addReg(RegNo: SplatV);
767 } else {
768 // SplatV = A2_tfrsi #imm:#imm:#imm:#imm
769 // OutV = V6_lvsplatw SplatV
770 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
771 const MachineOperand &InpOp = MI.getOperand(i: 1);
772 assert(InpOp.isImm());
773 uint32_t V = InpOp.getImm() & 0xFF;
774 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
775 .addImm(Val: V << 24 | V << 16 | V << 8 | V);
776 Register OutV = MI.getOperand(i: 0).getReg();
777 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
778 }
779 MB.erase(I: At);
780 break;
781 case Hexagon::PS_vsplatrb:
782 if (Subtarget.useHVXV62Ops()) {
783 // OutV = V6_lvsplatb Inp
784 Register OutV = MI.getOperand(i: 0).getReg();
785 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatb), DestReg: OutV)
786 .add(MO: MI.getOperand(i: 1));
787 } else {
788 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
789 const MachineOperand &InpOp = MI.getOperand(i: 1);
790 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::S2_vsplatrb), DestReg: SplatV)
791 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
792 Register OutV = MI.getOperand(i: 0).getReg();
793 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV)
794 .addReg(RegNo: SplatV);
795 }
796 MB.erase(I: At);
797 break;
798 case Hexagon::PS_vsplatih:
799 if (Subtarget.useHVXV62Ops()) {
800 // SplatV = A2_tfrsi #imm
801 // OutV = V6_lvsplath SplatV
802 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
803 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
804 .add(MO: MI.getOperand(i: 1));
805 Register OutV = MI.getOperand(i: 0).getReg();
806 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
807 .addReg(RegNo: SplatV);
808 } else {
809 // SplatV = A2_tfrsi #imm:#imm
810 // OutV = V6_lvsplatw SplatV
811 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
812 const MachineOperand &InpOp = MI.getOperand(i: 1);
813 assert(InpOp.isImm());
814 uint32_t V = InpOp.getImm() & 0xFFFF;
815 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
816 .addImm(Val: V << 16 | V);
817 Register OutV = MI.getOperand(i: 0).getReg();
818 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
819 }
820 MB.erase(I: At);
821 break;
822 case Hexagon::PS_vsplatrh:
823 if (Subtarget.useHVXV62Ops()) {
824 // OutV = V6_lvsplath Inp
825 Register OutV = MI.getOperand(i: 0).getReg();
826 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplath), DestReg: OutV)
827 .add(MO: MI.getOperand(i: 1));
828 } else {
829 // SplatV = A2_combine_ll Inp, Inp
830 // OutV = V6_lvsplatw SplatV
831 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
832 const MachineOperand &InpOp = MI.getOperand(i: 1);
833 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_combine_ll), DestReg: SplatV)
834 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg())
835 .addReg(RegNo: InpOp.getReg(), Flags: {}, SubReg: InpOp.getSubReg());
836 Register OutV = MI.getOperand(i: 0).getReg();
837 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::V6_lvsplatw), DestReg: OutV).addReg(RegNo: SplatV);
838 }
839 MB.erase(I: At);
840 break;
841 case Hexagon::PS_vsplatiw:
842 case Hexagon::PS_vsplatrw:
843 if (Opc == Hexagon::PS_vsplatiw) {
844 // SplatV = A2_tfrsi #imm
845 Register SplatV = MRI.createVirtualRegister(RegClass: &Hexagon::IntRegsRegClass);
846 BuildMI(BB&: MB, I: At, MIMD: DL, MCID: TII.get(Opcode: Hexagon::A2_tfrsi), DestReg: SplatV)
847 .add(MO: MI.getOperand(i: 1));
848 MI.getOperand(i: 1).ChangeToRegister(Reg: SplatV, isDef: false);
849 }
850 // OutV = V6_lvsplatw SplatV/Inp
851 MI.setDesc(TII.get(Opcode: Hexagon::V6_lvsplatw));
852 break;
853 }
854}
855
856SDValue
857HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
858 SelectionDAG &DAG) const {
859 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
860 ElemIdx = DAG.getBitcast(VT: MVT::i32, V: ElemIdx);
861
862 unsigned ElemWidth = ElemTy.getSizeInBits();
863 if (ElemWidth == 8)
864 return ElemIdx;
865
866 unsigned L = Log2_32(Value: ElemWidth/8);
867 const SDLoc &dl(ElemIdx);
868 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32,
869 Ops: {ElemIdx, DAG.getConstant(Val: L, DL: dl, VT: MVT::i32)});
870}
871
872SDValue
873HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
874 SelectionDAG &DAG) const {
875 unsigned ElemWidth = ElemTy.getSizeInBits();
876 assert(ElemWidth >= 8 && ElemWidth <= 32);
877 if (ElemWidth == 32)
878 return Idx;
879
880 if (ty(Op: Idx) != MVT::i32)
881 Idx = DAG.getBitcast(VT: MVT::i32, V: Idx);
882 const SDLoc &dl(Idx);
883 SDValue Mask = DAG.getConstant(Val: 32/ElemWidth - 1, DL: dl, VT: MVT::i32);
884 SDValue SubIdx = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, Ops: {Idx, Mask});
885 return SubIdx;
886}
887
888SDValue
889HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
890 SDValue Op1, ArrayRef<int> Mask,
891 SelectionDAG &DAG) const {
892 MVT OpTy = ty(Op: Op0);
893 assert(OpTy == ty(Op1));
894
895 MVT ElemTy = OpTy.getVectorElementType();
896 if (ElemTy == MVT::i8)
897 return DAG.getVectorShuffle(VT: OpTy, dl, N1: Op0, N2: Op1, Mask);
898 assert(ElemTy.getSizeInBits() >= 8);
899
900 MVT ResTy = tyVector(Ty: OpTy, ElemTy: MVT::i8);
901 unsigned ElemSize = ElemTy.getSizeInBits() / 8;
902
903 SmallVector<int,128> ByteMask;
904 for (int M : Mask) {
905 if (M < 0) {
906 for (unsigned I = 0; I != ElemSize; ++I)
907 ByteMask.push_back(Elt: -1);
908 } else {
909 int NewM = M*ElemSize;
910 for (unsigned I = 0; I != ElemSize; ++I)
911 ByteMask.push_back(Elt: NewM+I);
912 }
913 }
914 assert(ResTy.getVectorNumElements() == ByteMask.size());
915 return DAG.getVectorShuffle(VT: ResTy, dl, N1: opCastElem(Vec: Op0, ElemTy: MVT::i8, DAG),
916 N2: opCastElem(Vec: Op1, ElemTy: MVT::i8, DAG), Mask: ByteMask);
917}
918
919SDValue
920HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
921 const SDLoc &dl, MVT VecTy,
922 SelectionDAG &DAG) const {
923 unsigned VecLen = Values.size();
924 MachineFunction &MF = DAG.getMachineFunction();
925 MVT ElemTy = VecTy.getVectorElementType();
926 unsigned ElemWidth = ElemTy.getSizeInBits();
927 unsigned HwLen = Subtarget.getVectorLength();
928
929 unsigned ElemSize = ElemWidth / 8;
930 assert(ElemSize*VecLen == HwLen);
931 SmallVector<SDValue,32> Words;
932
933 if (VecTy.getVectorElementType() != MVT::i32 &&
934 !(Subtarget.useHVXFloatingPoint() &&
935 VecTy.getVectorElementType() == MVT::f32)) {
936 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
937 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
938 MVT PartVT = MVT::getVectorVT(VT: VecTy.getVectorElementType(), NumElements: OpsPerWord);
939 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
940 SDValue W = buildVector32(Elem: Values.slice(N: i, M: OpsPerWord), dl, VecTy: PartVT, DAG);
941 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: W));
942 }
943 } else {
944 for (SDValue V : Values)
945 Words.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V));
946 }
947 auto isSplat = [] (ArrayRef<SDValue> Values, SDValue &SplatV) {
948 unsigned NumValues = Values.size();
949 assert(NumValues > 0);
950 bool IsUndef = true;
951 for (unsigned i = 0; i != NumValues; ++i) {
952 if (Values[i].isUndef())
953 continue;
954 IsUndef = false;
955 if (!SplatV.getNode())
956 SplatV = Values[i];
957 else if (SplatV != Values[i])
958 return false;
959 }
960 if (IsUndef)
961 SplatV = Values[0];
962 return true;
963 };
964
965 unsigned NumWords = Words.size();
966 SDValue SplatV;
967 bool IsSplat = isSplat(Words, SplatV);
968 if (IsSplat && isUndef(Op: SplatV))
969 return DAG.getUNDEF(VT: VecTy);
970 if (IsSplat) {
971 assert(SplatV.getNode());
972 if (isNullConstant(V: SplatV))
973 return getZero(dl, Ty: VecTy, DAG);
974 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
975 SDValue S = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordTy, Operand: SplatV);
976 return DAG.getBitcast(VT: VecTy, V: S);
977 }
978
979 // Delay recognizing constant vectors until here, so that we can generate
980 // a vsplat.
981 SmallVector<ConstantInt*, 128> Consts(VecLen);
982 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
983 if (AllConst) {
984 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
985 (Constant**)Consts.end());
986 Constant *CV = ConstantVector::get(V: Tmp);
987 Align Alignment(HwLen);
988 SDValue CP = LowerConstantPool(
989 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
990 DAG);
991 return DAG.getLoad(VT: VecTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
992 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
993 }
994
995 // A special case is a situation where the vector is built entirely from
996 // elements extracted from another vector. This could be done via a shuffle
997 // more efficiently, but typically, the size of the source vector will not
998 // match the size of the vector being built (which precludes the use of a
999 // shuffle directly).
1000 // This only handles a single source vector, and the vector being built
1001 // should be of a sub-vector type of the source vector type.
1002 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
1003 SmallVectorImpl<int> &SrcIdx) {
1004 SDValue Vec;
1005 for (SDValue V : Values) {
1006 if (isUndef(Op: V)) {
1007 SrcIdx.push_back(Elt: -1);
1008 continue;
1009 }
1010 if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1011 return false;
1012 // All extracts should come from the same vector.
1013 SDValue T = V.getOperand(i: 0);
1014 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
1015 return false;
1016 Vec = T;
1017 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
1018 if (C == nullptr)
1019 return false;
1020 int I = C->getSExtValue();
1021 assert(I >= 0 && "Negative element index");
1022 SrcIdx.push_back(Elt: I);
1023 }
1024 SrcVec = Vec;
1025 return true;
1026 };
1027
1028 SmallVector<int,128> ExtIdx;
1029 SDValue ExtVec;
1030 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
1031 MVT ExtTy = ty(Op: ExtVec);
1032 unsigned ExtLen = ExtTy.getVectorNumElements();
1033 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
1034 // Construct a new shuffle mask that will produce a vector with the same
1035 // number of elements as the input vector, and such that the vector we
1036 // want will be the initial subvector of it.
1037 SmallVector<int,128> Mask;
1038 BitVector Used(ExtLen);
1039
1040 for (int M : ExtIdx) {
1041 Mask.push_back(Elt: M);
1042 if (M >= 0)
1043 Used.set(M);
1044 }
1045 // Fill the rest of the mask with the unused elements of ExtVec in hopes
1046 // that it will result in a permutation of ExtVec's elements. It's still
1047 // fine if it doesn't (e.g. if undefs are present, or elements are
1048 // repeated), but permutations can always be done efficiently via vdelta
1049 // and vrdelta.
1050 for (unsigned I = 0; I != ExtLen; ++I) {
1051 if (Mask.size() == ExtLen)
1052 break;
1053 if (!Used.test(Idx: I))
1054 Mask.push_back(Elt: I);
1055 }
1056
1057 SDValue S = DAG.getVectorShuffle(VT: ExtTy, dl, N1: ExtVec,
1058 N2: DAG.getUNDEF(VT: ExtTy), Mask);
1059 return ExtLen == VecLen ? S : LoHalf(V: S, DAG);
1060 }
1061 }
1062
1063 // Find most common element to initialize vector with. This is to avoid
1064 // unnecessary vinsert/valign for cases where the same value is present
1065 // many times. Creates a histogram of the vector's elements to find the
1066 // most common element n.
1067 assert(4*Words.size() == Subtarget.getVectorLength());
1068 int VecHist[32];
1069 int n = 0;
1070 for (unsigned i = 0; i != NumWords; ++i) {
1071 VecHist[i] = 0;
1072 if (Words[i].isUndef())
1073 continue;
1074 for (unsigned j = i; j != NumWords; ++j)
1075 if (Words[i] == Words[j])
1076 VecHist[i]++;
1077
1078 if (VecHist[i] > VecHist[n])
1079 n = i;
1080 }
1081
1082 SDValue HalfV = getZero(dl, Ty: VecTy, DAG);
1083 if (VecHist[n] > 1) {
1084 // Always splat at word (i32) granularity so that the SPLAT_VECTOR node
1085 // is selected as PS_vsplatrw (word broadcast) rather than PS_vsplatrb
1086 // (byte broadcast of the low byte only), which would corrupt multi-byte
1087 // element types.
1088 MVT WordVecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
1089 SDValue WordSplat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: WordVecTy, Operand: Words[n]);
1090 SDValue SplatV = DAG.getBitcast(VT: VecTy, V: WordSplat);
1091 HalfV = DAG.getNode(Opcode: HexagonISD::VALIGN, DL: dl, VT: VecTy,
1092 Ops: {HalfV, SplatV, DAG.getConstant(Val: HwLen/2, DL: dl, VT: MVT::i32)});
1093 }
1094 SDValue HalfV0 = HalfV;
1095 SDValue HalfV1 = HalfV;
1096
1097 // Construct two halves in parallel, then or them together. Rn and Rm count
1098 // number of rotations needed before the next element. One last rotation is
1099 // performed post-loop to position the last element.
1100 int Rn = 0, Rm = 0;
1101 SDValue Sn, Sm;
1102 SDValue N = HalfV0;
1103 SDValue M = HalfV1;
1104 for (unsigned i = 0; i != NumWords/2; ++i) {
1105 // Rotate by element count since last insertion.
1106 if (Words[i] != Words[n] || VecHist[n] <= 1) {
1107 Sn = DAG.getConstant(Val: Rn, DL: dl, VT: MVT::i32);
1108 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1109 N = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1110 Ops: {HalfV0, Words[i]});
1111 Rn = 0;
1112 }
1113 if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) {
1114 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1115 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1116 M = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy,
1117 Ops: {HalfV1, Words[i+NumWords/2]});
1118 Rm = 0;
1119 }
1120 Rn += 4;
1121 Rm += 4;
1122 }
1123 // Perform last rotation.
1124 Sn = DAG.getConstant(Val: Rn+HwLen/2, DL: dl, VT: MVT::i32);
1125 Sm = DAG.getConstant(Val: Rm, DL: dl, VT: MVT::i32);
1126 HalfV0 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {N, Sn});
1127 HalfV1 = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {M, Sm});
1128
1129 SDValue T0 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV0);
1130 SDValue T1 = DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i32), V: HalfV1);
1131
1132 SDValue DstV = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ty(Op: T0), Ops: {T0, T1});
1133
1134 SDValue OutV =
1135 DAG.getBitcast(VT: tyVector(Ty: ty(Op: DstV), ElemTy: VecTy.getVectorElementType()), V: DstV);
1136 return OutV;
1137}
1138
1139SDValue
1140HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
1141 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
1142 MVT PredTy = ty(Op: PredV);
1143 unsigned HwLen = Subtarget.getVectorLength();
1144 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1145
1146 if (Subtarget.isHVXVectorType(VecTy: PredTy, IncludeBool: true)) {
1147 // Move the vector predicate SubV to a vector register, and scale it
1148 // down to match the representation (bytes per type element) that VecV
1149 // uses. The scaling down will pick every 2nd or 4th (every Scale-th
1150 // in general) element and put them at the front of the resulting
1151 // vector. This subvector will then be inserted into the Q2V of VecV.
1152 // To avoid having an operation that generates an illegal type (short
1153 // vector), generate a full size vector.
1154 //
1155 SDValue T = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: PredV);
1156 SmallVector<int,128> Mask(HwLen);
1157 // Scale = BitBytes(PredV) / Given BitBytes.
1158 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
1159 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
1160
1161 for (unsigned i = 0; i != HwLen; ++i) {
1162 unsigned Num = i % Scale;
1163 unsigned Off = i / Scale;
1164 Mask[BlockLen*Num + Off] = i;
1165 }
1166 SDValue S = DAG.getVectorShuffle(VT: ByteTy, dl, N1: T, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1167 if (!ZeroFill)
1168 return S;
1169 // Fill the bytes beyond BlockLen with 0s.
1170 // V6_pred_scalar2 cannot fill the entire predicate, so it only works
1171 // when BlockLen < HwLen.
1172 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1173 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1174 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1175 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1176 SDValue M = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Q);
1177 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ByteTy, N1: S, N2: M);
1178 }
1179
1180 // Make sure that this is a valid scalar predicate.
1181 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
1182
1183 unsigned Bytes = 8 / PredTy.getVectorNumElements();
1184 SmallVector<SDValue,4> Words[2];
1185 unsigned IdxW = 0;
1186
1187 SDValue W0 = isUndef(Op: PredV)
1188 ? DAG.getUNDEF(VT: MVT::i64)
1189 : DAG.getNode(Opcode: HexagonISD::P2D, DL: dl, VT: MVT::i64, Operand: PredV);
1190 Words[IdxW].push_back(Elt: HiHalf(V: W0, DAG));
1191 Words[IdxW].push_back(Elt: LoHalf(V: W0, DAG));
1192
1193 while (Bytes < BitBytes) {
1194 IdxW ^= 1;
1195 Words[IdxW].clear();
1196
1197 if (Bytes < 4) {
1198 for (const SDValue &W : Words[IdxW ^ 1]) {
1199 SDValue T = expandPredicate(Vec32: W, dl, DAG);
1200 Words[IdxW].push_back(Elt: HiHalf(V: T, DAG));
1201 Words[IdxW].push_back(Elt: LoHalf(V: T, DAG));
1202 }
1203 } else {
1204 for (const SDValue &W : Words[IdxW ^ 1]) {
1205 Words[IdxW].push_back(Elt: W);
1206 Words[IdxW].push_back(Elt: W);
1207 }
1208 }
1209 Bytes *= 2;
1210 }
1211
1212 assert(Bytes == BitBytes);
1213 SDValue Vec = ZeroFill ? getZero(dl, Ty: ByteTy, DAG) : DAG.getUNDEF(VT: ByteTy);
1214 SDValue S4 = DAG.getConstant(Val: HwLen-4, DL: dl, VT: MVT::i32);
1215 for (const SDValue &W : Words[IdxW]) {
1216 Vec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Vec, N2: S4);
1217 Vec = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: ByteTy, N1: Vec, N2: W);
1218 }
1219
1220 return Vec;
1221}
1222
1223SDValue
1224HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
1225 const SDLoc &dl, MVT VecTy,
1226 SelectionDAG &DAG) const {
1227 // Construct a vector V of bytes, such that a comparison V >u 0 would
1228 // produce the required vector predicate.
1229 unsigned VecLen = Values.size();
1230 unsigned HwLen = Subtarget.getVectorLength();
1231 assert(VecLen <= HwLen || VecLen == 8*HwLen);
1232 SmallVector<SDValue,128> Bytes;
1233 bool AllT = true, AllF = true;
1234
1235 auto IsTrue = [] (SDValue V) {
1236 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1237 return !N->isZero();
1238 return false;
1239 };
1240 auto IsFalse = [] (SDValue V) {
1241 if (const auto *N = dyn_cast<ConstantSDNode>(Val: V.getNode()))
1242 return N->isZero();
1243 return false;
1244 };
1245
1246 if (VecLen <= HwLen) {
1247 // In the hardware, each bit of a vector predicate corresponds to a byte
1248 // of a vector register. Calculate how many bytes does a bit of VecTy
1249 // correspond to.
1250 assert(HwLen % VecLen == 0);
1251 unsigned BitBytes = HwLen / VecLen;
1252 for (SDValue V : Values) {
1253 AllT &= IsTrue(V);
1254 AllF &= IsFalse(V);
1255
1256 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(Op: V, DL: dl, VT: MVT::i8)
1257 : DAG.getUNDEF(VT: MVT::i8);
1258 for (unsigned B = 0; B != BitBytes; ++B)
1259 Bytes.push_back(Elt: Ext);
1260 }
1261 } else {
1262 // There are as many i1 values, as there are bits in a vector register.
1263 // Divide the values into groups of 8 and check that each group consists
1264 // of the same value (ignoring undefs).
1265 for (unsigned I = 0; I != VecLen; I += 8) {
1266 unsigned B = 0;
1267 // Find the first non-undef value in this group.
1268 for (; B != 8; ++B) {
1269 if (!Values[I+B].isUndef())
1270 break;
1271 }
1272 SDValue F = Values[I+B];
1273 AllT &= IsTrue(F);
1274 AllF &= IsFalse(F);
1275
1276 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(Op: F, DL: dl, VT: MVT::i8)
1277 : DAG.getUNDEF(VT: MVT::i8);
1278 Bytes.push_back(Elt: Ext);
1279 // Verify that the rest of values in the group are the same as the
1280 // first.
1281 for (; B != 8; ++B)
1282 assert(Values[I+B].isUndef() || Values[I+B] == F);
1283 }
1284 }
1285
1286 if (AllT)
1287 return DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: VecTy);
1288 if (AllF)
1289 return DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: VecTy);
1290
1291 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1292 SDValue ByteVec = buildHvxVectorReg(Values: Bytes, dl, VecTy: ByteTy, DAG);
1293 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1294}
1295
1296SDValue
1297HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
1298 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1299 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1300
1301 unsigned ElemWidth = ElemTy.getSizeInBits();
1302 assert(ElemWidth >= 8 && ElemWidth <= 32);
1303 (void)ElemWidth;
1304
1305 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1306 SDValue ExWord = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1307 Ops: {VecV, ByteIdx});
1308 if (ElemTy == MVT::i32)
1309 return ExWord;
1310
1311 // Have an extracted word, need to extract the smaller element out of it.
1312 // 1. Extract the bits of (the original) IdxV that correspond to the index
1313 // of the desired element in the 32-bit word.
1314 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1315 // 2. Extract the element from the word.
1316 SDValue ExVec = DAG.getBitcast(VT: tyVector(Ty: ty(Op: ExWord), ElemTy), V: ExWord);
1317 return extractVector(VecV: ExVec, IdxV: SubIdx, dl, ValTy: ElemTy, ResTy: MVT::i32, DAG);
1318}
1319
1320SDValue
1321HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
1322 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1323 // Implement other return types if necessary.
1324 assert(ResTy == MVT::i1);
1325
1326 unsigned HwLen = Subtarget.getVectorLength();
1327 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1328 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1329
1330 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1331 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1332 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1333
1334 SDValue ExtB = extractHvxElementReg(VecV: ByteVec, IdxV, dl, ResTy: MVT::i32, DAG);
1335 SDValue Zero = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1336 return getInstr(MachineOpc: Hexagon::C2_cmpgtui, dl, Ty: MVT::i1, Ops: {ExtB, Zero}, DAG);
1337}
1338
1339SDValue
1340HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
1341 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1342 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1343
1344 unsigned ElemWidth = ElemTy.getSizeInBits();
1345 assert(ElemWidth >= 8 && ElemWidth <= 32);
1346 (void)ElemWidth;
1347
1348 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
1349 SDValue ByteIdxV) {
1350 MVT VecTy = ty(Op: VecV);
1351 unsigned HwLen = Subtarget.getVectorLength();
1352 SDValue MaskV =
1353 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
1354 Ops: {ByteIdxV, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)});
1355 SDValue RotV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {VecV, MaskV});
1356 SDValue InsV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: VecTy, Ops: {RotV, ValV});
1357 SDValue SubV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1358 Ops: {DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32), MaskV});
1359 SDValue TorV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: VecTy, Ops: {InsV, SubV});
1360 return TorV;
1361 };
1362
1363 SDValue ByteIdx = convertToByteIndex(ElemIdx: IdxV, ElemTy, DAG);
1364 if (ElemTy == MVT::i32)
1365 return InsertWord(VecV, ValV, ByteIdx);
1366
1367 // If this is not inserting a 32-bit word, convert it into such a thing.
1368 // 1. Extract the existing word from the target vector.
1369 SDValue WordIdx = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
1370 Ops: {ByteIdx, DAG.getConstant(Val: 2, DL: dl, VT: MVT::i32)});
1371 SDValue Ext = extractHvxElementReg(VecV: opCastElem(Vec: VecV, ElemTy: MVT::i32, DAG), IdxV: WordIdx,
1372 dl, ResTy: MVT::i32, DAG);
1373
1374 // 2. Treating the extracted word as a 32-bit vector, insert the given
1375 // value into it.
1376 SDValue SubIdx = getIndexInWord32(Idx: IdxV, ElemTy, DAG);
1377 MVT SubVecTy = tyVector(Ty: ty(Op: Ext), ElemTy);
1378 SDValue Ins = insertVector(VecV: DAG.getBitcast(VT: SubVecTy, V: Ext),
1379 ValV, IdxV: SubIdx, dl, ValTy: ElemTy, DAG);
1380
1381 // 3. Insert the 32-bit word back into the original vector.
1382 return InsertWord(VecV, Ins, ByteIdx);
1383}
1384
1385SDValue
1386HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
1387 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1388 unsigned HwLen = Subtarget.getVectorLength();
1389 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1390 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1391
1392 unsigned Scale = HwLen / ty(Op: VecV).getVectorNumElements();
1393 SDValue ScV = DAG.getConstant(Val: Scale, DL: dl, VT: MVT::i32);
1394 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV, N2: ScV);
1395 ValV = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i32, Operand: ValV);
1396
1397 SDValue InsV = insertHvxElementReg(VecV: ByteVec, IdxV, ValV, dl, DAG);
1398 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ty(Op: VecV), Operand: InsV);
1399}
1400
1401SDValue
1402HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV,
1403 SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1404 MVT VecTy = ty(Op: VecV);
1405 unsigned HwLen = Subtarget.getVectorLength();
1406 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1407 MVT ElemTy = VecTy.getVectorElementType();
1408 unsigned ElemWidth = ElemTy.getSizeInBits();
1409
1410 // If the source vector is a vector pair, get the single vector containing
1411 // the subvector of interest. The subvector will never overlap two single
1412 // vectors.
1413 if (isHvxPairTy(Ty: VecTy)) {
1414 unsigned SubIdx = Hexagon::vsub_lo;
1415 if (Idx * ElemWidth >= 8 * HwLen) {
1416 SubIdx = Hexagon::vsub_hi;
1417 Idx -= VecTy.getVectorNumElements() / 2;
1418 }
1419
1420 VecTy = typeSplit(VecTy).first;
1421 VecV = DAG.getTargetExtractSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV);
1422 if (VecTy == ResTy)
1423 return VecV;
1424 }
1425
1426 // The only meaningful subvectors of a single HVX vector are those that
1427 // fit in a scalar register.
1428 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
1429
1430 MVT WordTy = tyVector(Ty: VecTy, ElemTy: MVT::i32);
1431 SDValue WordVec = DAG.getBitcast(VT: WordTy, V: VecV);
1432 unsigned WordIdx = (Idx*ElemWidth) / 32;
1433
1434 SDValue W0Idx = DAG.getConstant(Val: WordIdx, DL: dl, VT: MVT::i32);
1435 SDValue W0 = extractHvxElementReg(VecV: WordVec, IdxV: W0Idx, dl, ResTy: MVT::i32, DAG);
1436 if (ResTy.getSizeInBits() == 32)
1437 return DAG.getBitcast(VT: ResTy, V: W0);
1438
1439 SDValue W1Idx = DAG.getConstant(Val: WordIdx+1, DL: dl, VT: MVT::i32);
1440 SDValue W1 = extractHvxElementReg(VecV: WordVec, IdxV: W1Idx, dl, ResTy: MVT::i32, DAG);
1441 SDValue WW = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::i64, DAG);
1442 return DAG.getBitcast(VT: ResTy, V: WW);
1443}
1444
1445SDValue
1446HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
1447 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1448 MVT VecTy = ty(Op: VecV);
1449 unsigned HwLen = Subtarget.getVectorLength();
1450 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1451 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1452 // IdxV is required to be a constant.
1453 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1454
1455 unsigned ResLen = ResTy.getVectorNumElements();
1456 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1457 unsigned Offset = Idx * BitBytes;
1458 SDValue Undef = DAG.getUNDEF(VT: ByteTy);
1459 SmallVector<int,128> Mask;
1460
1461 if (Subtarget.isHVXVectorType(VecTy: ResTy, IncludeBool: true)) {
1462 // Converting between two vector predicates. Since the result is shorter
1463 // than the source, it will correspond to a vector predicate with the
1464 // relevant bits replicated. The replication count is the ratio of the
1465 // source and target vector lengths.
1466 unsigned Rep = VecTy.getVectorNumElements() / ResLen;
1467 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
1468 for (unsigned i = 0; i != HwLen/Rep; ++i) {
1469 for (unsigned j = 0; j != Rep; ++j)
1470 Mask.push_back(Elt: i + Offset);
1471 }
1472 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1473 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: ShuffV);
1474 }
1475
1476 // Converting between a vector predicate and a scalar predicate. In the
1477 // vector predicate, a group of BitBytes bits will correspond to a single
1478 // i1 element of the source vector type. Those bits will all have the same
1479 // value. The same will be true for ByteVec, where each byte corresponds
1480 // to a bit in the vector predicate.
1481 // The algorithm is to traverse the ByteVec, going over the i1 values from
1482 // the source vector, and generate the corresponding representation in an
1483 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
1484 // elements so that the interesting 8 bytes will be in the low end of the
1485 // vector.
1486 unsigned Rep = 8 / ResLen;
1487 // Make sure the output fill the entire vector register, so repeat the
1488 // 8-byte groups as many times as necessary.
1489 for (unsigned r = 0; r != HwLen / 8; ++r) {
1490 // This will generate the indexes of the 8 interesting bytes.
1491 for (unsigned i = 0; i != ResLen; ++i) {
1492 for (unsigned j = 0; j != Rep; ++j)
1493 Mask.push_back(Elt: Offset + i*BitBytes);
1494 }
1495 }
1496
1497 SDValue Zero = getZero(dl, Ty: MVT::i32, DAG);
1498 SDValue ShuffV = DAG.getVectorShuffle(VT: ByteTy, dl, N1: ByteVec, N2: Undef, Mask);
1499 // Combine the two low words from ShuffV into a v8i8, and byte-compare
1500 // them against 0.
1501 SDValue W0 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32, Ops: {ShuffV, Zero});
1502 SDValue W1 = DAG.getNode(Opcode: HexagonISD::VEXTRACTW, DL: dl, VT: MVT::i32,
1503 Ops: {ShuffV, DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32)});
1504 SDValue Vec64 = getCombine(Hi: W1, Lo: W0, dl, ResTy: MVT::v8i8, DAG);
1505 return getInstr(MachineOpc: Hexagon::A4_vcmpbgtui, dl, Ty: ResTy,
1506 Ops: {Vec64, DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32)}, DAG);
1507}
1508
1509SDValue
1510HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
1511 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1512 MVT VecTy = ty(Op: VecV);
1513 MVT SubTy = ty(Op: SubV);
1514 unsigned HwLen = Subtarget.getVectorLength();
1515 MVT ElemTy = VecTy.getVectorElementType();
1516 unsigned ElemWidth = ElemTy.getSizeInBits();
1517
1518 bool IsPair = isHvxPairTy(Ty: VecTy);
1519 MVT SingleTy = MVT::getVectorVT(VT: ElemTy, NumElements: (8*HwLen)/ElemWidth);
1520 // The two single vectors that VecV consists of, if it's a pair.
1521 SDValue V0, V1;
1522 SDValue SingleV = VecV;
1523 SDValue PickHi;
1524
1525 if (IsPair) {
1526 V0 = LoHalf(V: VecV, DAG);
1527 V1 = HiHalf(V: VecV, DAG);
1528
1529 SDValue HalfV = DAG.getConstant(Val: SingleTy.getVectorNumElements(),
1530 DL: dl, VT: MVT::i32);
1531 PickHi = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: IdxV, RHS: HalfV, Cond: ISD::SETUGT);
1532 if (isHvxSingleTy(Ty: SubTy)) {
1533 if (const auto *CN = dyn_cast<const ConstantSDNode>(Val: IdxV.getNode())) {
1534 unsigned Idx = CN->getZExtValue();
1535 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
1536 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
1537 return DAG.getTargetInsertSubreg(SRIdx: SubIdx, DL: dl, VT: VecTy, Operand: VecV, Subreg: SubV);
1538 }
1539 // If IdxV is not a constant, generate the two variants: with the
1540 // SubV as the high and as the low subregister, and select the right
1541 // pair based on the IdxV.
1542 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SubV, V1});
1543 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SubV});
1544 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1545 }
1546 // The subvector being inserted must be entirely contained in one of
1547 // the vectors V0 or V1. Set SingleV to the correct one, and update
1548 // IdxV to be the index relative to the beginning of that vector.
1549 SDValue S = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: IdxV, N2: HalfV);
1550 IdxV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: PickHi, N2: S, N3: IdxV);
1551 SingleV = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: SingleTy, N1: PickHi, N2: V1, N3: V0);
1552 }
1553
1554 // The only meaningful subvectors of a single HVX vector are those that
1555 // fit in a scalar register.
1556 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
1557 // Convert IdxV to be index in bytes.
1558 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1559 if (!IdxN || !IdxN->isZero()) {
1560 IdxV = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1561 N2: DAG.getConstant(Val: ElemWidth/8, DL: dl, VT: MVT::i32));
1562 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: IdxV);
1563 }
1564 // When inserting a single word, the rotation back to the original position
1565 // would be by HwLen-Idx, but if two words are inserted, it will need to be
1566 // by (HwLen-4)-Idx.
1567 unsigned RolBase = HwLen;
1568 if (SubTy.getSizeInBits() == 32) {
1569 SDValue V = DAG.getBitcast(VT: MVT::i32, V: SubV);
1570 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: V);
1571 } else {
1572 SDValue V = DAG.getBitcast(VT: MVT::i64, V: SubV);
1573 SDValue R0 = LoHalf(V, DAG);
1574 SDValue R1 = HiHalf(V, DAG);
1575 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R0);
1576 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV,
1577 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
1578 SingleV = DAG.getNode(Opcode: HexagonISD::VINSERTW0, DL: dl, VT: SingleTy, N1: SingleV, N2: R1);
1579 RolBase = HwLen-4;
1580 }
1581 // If the vector wasn't ror'ed, don't ror it back.
1582 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1583 SDValue RolV = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
1584 N1: DAG.getConstant(Val: RolBase, DL: dl, VT: MVT::i32), N2: IdxV);
1585 SingleV = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: SingleTy, N1: SingleV, N2: RolV);
1586 }
1587
1588 if (IsPair) {
1589 SDValue InLo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {SingleV, V1});
1590 SDValue InHi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, Ops: {V0, SingleV});
1591 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: PickHi, N2: InHi, N3: InLo);
1592 }
1593 return SingleV;
1594}
1595
1596SDValue
1597HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1598 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1599 MVT VecTy = ty(Op: VecV);
1600 MVT SubTy = ty(Op: SubV);
1601 assert(Subtarget.isHVXVectorType(VecTy, true));
1602 // VecV is an HVX vector predicate. SubV may be either an HVX vector
1603 // predicate as well, or it can be a scalar predicate.
1604
1605 unsigned VecLen = VecTy.getVectorNumElements();
1606 unsigned HwLen = Subtarget.getVectorLength();
1607 assert(HwLen % VecLen == 0 && "Unexpected vector type");
1608
1609 unsigned Scale = VecLen / SubTy.getVectorNumElements();
1610 unsigned BitBytes = HwLen / VecLen;
1611 unsigned BlockLen = HwLen / Scale;
1612
1613 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1614 SDValue ByteVec = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: VecV);
1615 SDValue ByteSub = createHvxPrefixPred(PredV: SubV, dl, BitBytes, ZeroFill: false, DAG);
1616 SDValue ByteIdx;
1617
1618 auto *IdxN = dyn_cast<ConstantSDNode>(Val: IdxV.getNode());
1619 if (!IdxN || !IdxN->isZero()) {
1620 ByteIdx = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: IdxV,
1621 N2: DAG.getConstant(Val: BitBytes, DL: dl, VT: MVT::i32));
1622 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteIdx);
1623 }
1624
1625 // ByteVec is the target vector VecV rotated in such a way that the
1626 // subvector should be inserted at index 0. Generate a predicate mask
1627 // and use vmux to do the insertion.
1628 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1629 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
1630 SDValue Q = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
1631 Ops: {DAG.getConstant(Val: BlockLen, DL: dl, VT: MVT::i32)}, DAG);
1632 ByteVec = getInstr(MachineOpc: Hexagon::V6_vmux, dl, Ty: ByteTy, Ops: {Q, ByteSub, ByteVec}, DAG);
1633 // Rotate ByteVec back, and convert to a vector predicate.
1634 if (!IdxN || !IdxN->isZero()) {
1635 SDValue HwLenV = DAG.getConstant(Val: HwLen, DL: dl, VT: MVT::i32);
1636 SDValue ByteXdi = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32, N1: HwLenV, N2: ByteIdx);
1637 ByteVec = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: ByteVec, N2: ByteXdi);
1638 }
1639 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: ByteVec);
1640}
1641
1642SDValue
1643HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1644 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1645 // Sign- and any-extending of a vector predicate to a vector register is
1646 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1647 // a vector of 1s (where the 1s are of type matching the vector type).
1648 assert(Subtarget.isHVXVectorType(ResTy));
1649 if (!ZeroExt)
1650 return DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ResTy, Operand: VecV);
1651
1652 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1653 SDValue True = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
1654 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1655 SDValue False = getZero(dl, Ty: ResTy, DAG);
1656 return DAG.getSelect(DL: dl, VT: ResTy, Cond: VecV, LHS: True, RHS: False);
1657}
1658
1659SDValue
1660HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1661 MVT ResTy, SelectionDAG &DAG) const {
1662 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1663 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1664 // vector register. The remaining bits of the vector register are
1665 // unspecified.
1666
1667 MachineFunction &MF = DAG.getMachineFunction();
1668 unsigned HwLen = Subtarget.getVectorLength();
1669 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1670 MVT PredTy = ty(Op: VecQ);
1671 unsigned PredLen = PredTy.getVectorNumElements();
1672 assert(HwLen % PredLen == 0);
1673 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8*HwLen/PredLen), NumElements: PredLen);
1674
1675 Type *Int8Ty = Type::getInt8Ty(C&: *DAG.getContext());
1676 SmallVector<Constant*, 128> Tmp;
1677 // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
1678 // These are bytes with the LSB rotated left with respect to their index.
1679 for (unsigned i = 0; i != HwLen/8; ++i) {
1680 for (unsigned j = 0; j != 8; ++j)
1681 Tmp.push_back(Elt: ConstantInt::get(Ty: Int8Ty, V: 1ull << j));
1682 }
1683 Constant *CV = ConstantVector::get(V: Tmp);
1684 Align Alignment(HwLen);
1685 SDValue CP = LowerConstantPool(
1686 Op: DAG.getConstantPool(C: CV, VT: getPointerTy(DL: DAG.getDataLayout()), Align: Alignment),
1687 DAG);
1688 SDValue Bytes =
1689 DAG.getLoad(VT: ByteTy, dl, Chain: DAG.getEntryNode(), Ptr: CP,
1690 PtrInfo: MachinePointerInfo::getConstantPool(MF), Alignment);
1691
1692 // Select the bytes that correspond to true bits in the vector predicate.
1693 SDValue Sel = DAG.getSelect(DL: dl, VT: VecTy, Cond: VecQ, LHS: DAG.getBitcast(VT: VecTy, V: Bytes),
1694 RHS: getZero(dl, Ty: VecTy, DAG));
1695 // Calculate the OR of all bytes in each group of 8. That will compress
1696 // all the individual bits into a single byte.
1697 // First, OR groups of 4, via vrmpy with 0x01010101.
1698 SDValue All1 =
1699 DAG.getSplatBuildVector(VT: MVT::v4i8, DL: dl, Op: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
1700 SDValue Vrmpy = getInstr(MachineOpc: Hexagon::V6_vrmpyub, dl, Ty: ByteTy, Ops: {Sel, All1}, DAG);
1701 // Then rotate the accumulated vector by 4 bytes, and do the final OR.
1702 SDValue Rot = getInstr(MachineOpc: Hexagon::V6_valignbi, dl, Ty: ByteTy,
1703 Ops: {Vrmpy, Vrmpy, DAG.getTargetConstant(Val: 4, DL: dl, VT: MVT::i32)}, DAG);
1704 SDValue Vor = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, Ops: {Vrmpy, Rot});
1705
1706 // Pick every 8th byte and coalesce them at the beginning of the output.
1707 // For symmetry, coalesce every 1+8th byte after that, then every 2+8th
1708 // byte and so on.
1709 SmallVector<int,128> Mask;
1710 for (unsigned i = 0; i != HwLen; ++i)
1711 Mask.push_back(Elt: (8*i) % HwLen + i/(HwLen/8));
1712 SDValue Collect =
1713 DAG.getVectorShuffle(VT: ByteTy, dl, N1: Vor, N2: DAG.getUNDEF(VT: ByteTy), Mask);
1714 return DAG.getBitcast(VT: ResTy, V: Collect);
1715}
1716
1717SDValue
1718HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed,
1719 const SDLoc &dl, SelectionDAG &DAG) const {
1720 // Take a vector and resize the element type to match the given type.
1721 MVT InpTy = ty(Op: VecV);
1722 if (InpTy == ResTy)
1723 return VecV;
1724
1725 unsigned InpWidth = InpTy.getSizeInBits();
1726 unsigned ResWidth = ResTy.getSizeInBits();
1727
1728 if (InpTy.isFloatingPoint()) {
1729 return InpWidth < ResWidth
1730 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: ResTy, Operand: VecV)
1731 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ResTy, N1: VecV,
1732 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
1733 }
1734
1735 assert(InpTy.isInteger());
1736
1737 if (InpWidth < ResWidth) {
1738 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1739 return DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ResTy, Operand: VecV);
1740 } else {
1741 unsigned NarOpc = Signed ? HexagonISD::SSAT : HexagonISD::USAT;
1742 return DAG.getNode(Opcode: NarOpc, DL: dl, VT: ResTy, N1: VecV, N2: DAG.getValueType(ResTy));
1743 }
1744}
1745
1746SDValue
1747HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx,
1748 SelectionDAG &DAG) const {
1749 assert(ty(Vec).getSizeInBits() % SubTy.getSizeInBits() == 0);
1750
1751 const SDLoc &dl(Vec);
1752 unsigned ElemIdx = SubIdx * SubTy.getVectorNumElements();
1753 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubTy,
1754 Ops: {Vec, DAG.getConstant(Val: ElemIdx, DL: dl, VT: MVT::i32)});
1755}
1756
1757SDValue
1758HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1759 const {
1760 const SDLoc &dl(Op);
1761 MVT VecTy = ty(Op);
1762
1763 unsigned Size = Op.getNumOperands();
1764 SmallVector<SDValue,128> Ops;
1765 for (unsigned i = 0; i != Size; ++i)
1766 Ops.push_back(Elt: Op.getOperand(i));
1767
1768 if (VecTy.getVectorElementType() == MVT::i1)
1769 return buildHvxVectorPred(Values: Ops, dl, VecTy, DAG);
1770
1771 // In case of MVT::f16 BUILD_VECTOR, since MVT::f16 is
1772 // not a legal type, just bitcast the node to use i16
1773 // types and bitcast the result back to f16
1774 if (VecTy.getVectorElementType() == MVT::f16 ||
1775 VecTy.getVectorElementType() == MVT::bf16) {
1776 SmallVector<SDValue, 64> NewOps;
1777 for (unsigned i = 0; i != Size; i++)
1778 NewOps.push_back(Elt: DAG.getBitcast(VT: MVT::i16, V: Ops[i]));
1779
1780 SDValue T0 =
1781 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: tyVector(Ty: VecTy, ElemTy: MVT::i16), Ops: NewOps);
1782 return DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: VecTy.getVectorElementType()), V: T0);
1783 }
1784
1785 // First, split the BUILD_VECTOR for vector pairs. We could generate
1786 // some pairs directly (via splat), but splats should be generated
1787 // by the combiner prior to getting here.
1788 if (VecTy.getSizeInBits() == 16 * Subtarget.getVectorLength()) {
1789 ArrayRef<SDValue> A(Ops);
1790 MVT SingleTy = typeSplit(VecTy).first;
1791 SDValue V0 = buildHvxVectorReg(Values: A.take_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1792 SDValue V1 = buildHvxVectorReg(Values: A.drop_front(N: Size / 2), dl, VecTy: SingleTy, DAG);
1793 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: VecTy, N1: V0, N2: V1);
1794 }
1795
1796 return buildHvxVectorReg(Values: Ops, dl, VecTy, DAG);
1797}
1798
1799SDValue
1800HexagonTargetLowering::LowerHvxSplatVector(SDValue Op, SelectionDAG &DAG)
1801 const {
1802 const SDLoc &dl(Op);
1803 MVT VecTy = ty(Op);
1804 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
1805
1806 if (ArgTy == MVT::f16 || ArgTy == MVT::bf16) {
1807 MVT SplatTy = MVT::getVectorVT(VT: MVT::i16, NumElements: VecTy.getVectorNumElements());
1808 SDValue ToInt16 = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: 0));
1809 SDValue ToInt32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: ToInt16);
1810 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: SplatTy, Operand: ToInt32);
1811 return DAG.getBitcast(VT: VecTy, V: Splat);
1812 }
1813
1814 return SDValue();
1815}
1816
1817SDValue
1818HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1819 const {
1820 // Vector concatenation of two integer (non-bool) vectors does not need
1821 // special lowering. Custom-lower concats of bool vectors and expand
1822 // concats of more than 2 vectors.
1823 MVT VecTy = ty(Op);
1824 const SDLoc &dl(Op);
1825 unsigned NumOp = Op.getNumOperands();
1826 if (VecTy.getVectorElementType() != MVT::i1) {
1827 if (NumOp == 2)
1828 return Op;
1829 // Expand the other cases into a build-vector.
1830 SmallVector<SDValue,8> Elems;
1831 for (SDValue V : Op.getNode()->ops())
1832 DAG.ExtractVectorElements(Op: V, Args&: Elems);
1833 // A vector of i16 will be broken up into a build_vector of i16's.
1834 // This is a problem, since at the time of operation legalization,
1835 // all operations are expected to be type-legalized, and i16 is not
1836 // a legal type. If any of the extracted elements is not of a valid
1837 // type, sign-extend it to a valid one.
1838 for (SDValue &V : Elems) {
1839 MVT Ty = ty(Op: V);
1840 if (!isTypeLegal(VT: Ty)) {
1841 MVT NTy = typeLegalize(Ty, DAG);
1842 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1843 V = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: NTy,
1844 N1: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: NTy,
1845 N1: V.getOperand(i: 0), N2: V.getOperand(i: 1)),
1846 N2: DAG.getValueType(Ty));
1847 continue;
1848 }
1849 // A few less complicated cases.
1850 switch (V.getOpcode()) {
1851 case ISD::Constant:
1852 V = DAG.getSExtOrTrunc(Op: V, DL: dl, VT: NTy);
1853 break;
1854 case ISD::UNDEF:
1855 V = DAG.getUNDEF(VT: NTy);
1856 break;
1857 case ISD::TRUNCATE:
1858 V = V.getOperand(i: 0);
1859 break;
1860 default:
1861 llvm_unreachable("Unexpected vector element");
1862 }
1863 }
1864 }
1865 return DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Elems);
1866 }
1867
1868 assert(VecTy.getVectorElementType() == MVT::i1);
1869 unsigned HwLen = Subtarget.getVectorLength();
1870 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1871
1872 SDValue Op0 = Op.getOperand(i: 0);
1873
1874 // If the operands are HVX types (i.e. not scalar predicates), then
1875 // defer the concatenation, and create QCAT instead.
1876 if (Subtarget.isHVXVectorType(VecTy: ty(Op: Op0), IncludeBool: true)) {
1877 if (NumOp == 2)
1878 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: Op0, N2: Op.getOperand(i: 1));
1879
1880 ArrayRef<SDUse> U(Op.getNode()->ops());
1881 SmallVector<SDValue, 4> SV(U);
1882 ArrayRef<SDValue> Ops(SV);
1883
1884 MVT HalfTy = typeSplit(VecTy).first;
1885 SDValue V0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1886 Ops: Ops.take_front(N: NumOp/2));
1887 SDValue V1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HalfTy,
1888 Ops: Ops.take_back(N: NumOp/2));
1889 return DAG.getNode(Opcode: HexagonISD::QCAT, DL: dl, VT: VecTy, N1: V0, N2: V1);
1890 }
1891
1892 // Count how many bytes (in a vector register) each bit in VecTy
1893 // corresponds to.
1894 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1895
1896 // Make sure that createHvxPrefixPred will only ever need to expand
1897 // the predicate, i.e. bytes-per-bit in the input is not greater than
1898 // the target bytes-per-bit in the result.
1899 SDValue Combined = combineConcatOfScalarPreds(Op, BitBytes, DAG);
1900 SmallVector<SDValue,8> Prefixes;
1901 for (SDValue V : Combined.getNode()->op_values()) {
1902 SDValue P = createHvxPrefixPred(PredV: V, dl, BitBytes, ZeroFill: true, DAG);
1903 Prefixes.push_back(Elt: P);
1904 }
1905
1906 unsigned InpLen = ty(Op: Combined.getOperand(i: 0)).getVectorNumElements();
1907 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
1908 SDValue S = DAG.getConstant(Val: HwLen - InpLen*BitBytes, DL: dl, VT: MVT::i32);
1909 SDValue Res = getZero(dl, Ty: ByteTy, DAG);
1910 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1911 Res = DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ByteTy, N1: Res, N2: S);
1912 Res = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ByteTy, N1: Res, N2: Prefixes[e-i-1]);
1913 }
1914 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: VecTy, Operand: Res);
1915}
1916
1917SDValue
1918HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1919 const {
1920 // Change the type of the extracted element to i32.
1921 SDValue VecV = Op.getOperand(i: 0);
1922 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1923 const SDLoc &dl(Op);
1924 SDValue IdxV = Op.getOperand(i: 1);
1925 if (ElemTy == MVT::i1)
1926 return extractHvxElementPred(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1927
1928 return extractHvxElementReg(VecV, IdxV, dl, ResTy: ty(Op), DAG);
1929}
1930
1931SDValue
1932HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1933 const {
1934 const SDLoc &dl(Op);
1935 MVT VecTy = ty(Op);
1936 SDValue VecV = Op.getOperand(i: 0);
1937 SDValue ValV = Op.getOperand(i: 1);
1938 SDValue IdxV = Op.getOperand(i: 2);
1939 MVT ElemTy = ty(Op: VecV).getVectorElementType();
1940 if (ElemTy == MVT::i1)
1941 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1942
1943 if (ElemTy == MVT::f16 || ElemTy == MVT::bf16) {
1944 SDValue T0 = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
1945 VT: tyVector(Ty: VecTy, ElemTy: MVT::i16),
1946 N1: DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy: MVT::i16), V: VecV),
1947 N2: DAG.getBitcast(VT: MVT::i16, V: ValV), N3: IdxV);
1948 return DAG.getBitcast(VT: tyVector(Ty: VecTy, ElemTy), V: T0);
1949 }
1950
1951 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1952}
1953
1954SDValue
1955HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1956 const {
1957 SDValue SrcV = Op.getOperand(i: 0);
1958 MVT SrcTy = ty(Op: SrcV);
1959 MVT DstTy = ty(Op);
1960 SDValue IdxV = Op.getOperand(i: 1);
1961 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1962 assert(Idx % DstTy.getVectorNumElements() == 0);
1963 (void)Idx;
1964 const SDLoc &dl(Op);
1965
1966 MVT ElemTy = SrcTy.getVectorElementType();
1967 if (ElemTy == MVT::i1)
1968 return extractHvxSubvectorPred(VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1969
1970 return extractHvxSubvectorReg(OrigOp: Op, VecV: SrcV, IdxV, dl, ResTy: DstTy, DAG);
1971}
1972
1973SDValue
1974HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1975 const {
1976 // Idx does not need to be a constant.
1977 SDValue VecV = Op.getOperand(i: 0);
1978 SDValue ValV = Op.getOperand(i: 1);
1979 SDValue IdxV = Op.getOperand(i: 2);
1980
1981 const SDLoc &dl(Op);
1982 MVT VecTy = ty(Op: VecV);
1983 MVT ElemTy = VecTy.getVectorElementType();
1984 if (ElemTy == MVT::i1)
1985 return insertHvxSubvectorPred(VecV, SubV: ValV, IdxV, dl, DAG);
1986
1987 return insertHvxSubvectorReg(VecV, SubV: ValV, IdxV, dl, DAG);
1988}
1989
1990SDValue
1991HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1992 // Lower any-extends of boolean vectors to sign-extends, since they
1993 // translate directly to Q2V. Zero-extending could also be done equally
1994 // fast, but Q2V is used/recognized in more places.
1995 // For all other vectors, use zero-extend.
1996 MVT ResTy = ty(Op);
1997 SDValue InpV = Op.getOperand(i: 0);
1998 MVT ElemTy = ty(Op: InpV).getVectorElementType();
1999 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2000 return LowerHvxSignExt(Op, DAG);
2001 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Op), VT: ResTy, Operand: InpV);
2002}
2003
2004SDValue
2005HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
2006 MVT ResTy = ty(Op);
2007 SDValue InpV = Op.getOperand(i: 0);
2008 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2009 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2010 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: false, DAG);
2011 return Op;
2012}
2013
2014SDValue
2015HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
2016 MVT ResTy = ty(Op);
2017 SDValue InpV = Op.getOperand(i: 0);
2018 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2019 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(VecTy: ResTy))
2020 return extendHvxVectorPred(VecV: InpV, dl: SDLoc(Op), ResTy: ty(Op), ZeroExt: true, DAG);
2021 return Op;
2022}
2023
2024SDValue
2025HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
2026 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
2027 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
2028 const SDLoc &dl(Op);
2029 MVT ResTy = ty(Op);
2030 SDValue InpV = Op.getOperand(i: 0);
2031 assert(ResTy == ty(InpV));
2032
2033 // Calculate the vectors of 1 and bitwidth(x).
2034 MVT ElemTy = ty(Op: InpV).getVectorElementType();
2035 unsigned ElemWidth = ElemTy.getSizeInBits();
2036
2037 SDValue Vec1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2038 Operand: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
2039 SDValue VecW = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2040 Operand: DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32));
2041 SDValue VecN1 = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ResTy,
2042 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
2043
2044 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
2045 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
2046 // it separately in custom combine or selection).
2047 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy,
2048 Ops: {DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {InpV, VecN1}),
2049 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {InpV, Vec1})});
2050 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy,
2051 Ops: {VecW, DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: ResTy, Operand: A)});
2052}
2053
2054SDValue
2055HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
2056 const SDLoc &dl(Op);
2057 MVT ResTy = ty(Op);
2058 assert(ResTy.getVectorElementType() == MVT::i32);
2059
2060 SDValue Vs = Op.getOperand(i: 0);
2061 SDValue Vt = Op.getOperand(i: 1);
2062
2063 SDVTList ResTys = DAG.getVTList(VT1: ResTy, VT2: ResTy);
2064 unsigned Opc = Op.getOpcode();
2065
2066 // On HVX v62+ producing the full product is cheap, so legalize MULH to LOHI.
2067 if (Opc == ISD::MULHU)
2068 return DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2069 if (Opc == ISD::MULHS)
2070 return DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: ResTys, Ops: {Vs, Vt}).getValue(R: 1);
2071
2072#ifndef NDEBUG
2073 Op.dump(&DAG);
2074#endif
2075 llvm_unreachable("Unexpected mulh operation");
2076}
2077
2078SDValue
2079HexagonTargetLowering::LowerHvxMulLoHi(SDValue Op, SelectionDAG &DAG) const {
2080 const SDLoc &dl(Op);
2081 unsigned Opc = Op.getOpcode();
2082 SDValue Vu = Op.getOperand(i: 0);
2083 SDValue Vv = Op.getOperand(i: 1);
2084
2085 // If the HI part is not used, convert it to a regular MUL.
2086 if (auto HiVal = Op.getValue(R: 1); HiVal.use_empty()) {
2087 // Need to preserve the types and the number of values.
2088 SDValue Hi = DAG.getUNDEF(VT: ty(Op: HiVal));
2089 SDValue Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: ty(Op), Ops: {Vu, Vv});
2090 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2091 }
2092
2093 bool SignedVu = Opc == HexagonISD::SMUL_LOHI;
2094 bool SignedVv = Opc == HexagonISD::SMUL_LOHI || Opc == HexagonISD::USMUL_LOHI;
2095
2096 // Legal on HVX v62+, but lower it here because patterns can't handle multi-
2097 // valued nodes.
2098 if (Subtarget.useHVXV62Ops())
2099 return emitHvxMulLoHiV62(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2100
2101 if (Opc == HexagonISD::SMUL_LOHI) {
2102 // Direct MULHS expansion is cheaper than doing the whole SMUL_LOHI,
2103 // for other signedness LOHI is cheaper.
2104 if (auto LoVal = Op.getValue(R: 0); LoVal.use_empty()) {
2105 SDValue Hi = emitHvxMulHsV60(A: Vu, B: Vv, dl, DAG);
2106 SDValue Lo = DAG.getUNDEF(VT: ty(Op: LoVal));
2107 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
2108 }
2109 }
2110
2111 return emitHvxMulLoHiV60(A: Vu, SignedA: SignedVu, B: Vv, SignedB: SignedVv, dl, DAG);
2112}
2113
2114SDValue
2115HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
2116 SDValue Val = Op.getOperand(i: 0);
2117 MVT ResTy = ty(Op);
2118 MVT ValTy = ty(Op: Val);
2119 const SDLoc &dl(Op);
2120
2121 if (isHvxBoolTy(Ty: ValTy) && ResTy.isScalarInteger()) {
2122 unsigned HwLen = Subtarget.getVectorLength();
2123 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen/4);
2124
2125 // When the predicate is shorter than the predicate register, each boolean
2126 // is represented by multiple consecutive bits in the input register.
2127 // Condense the bits so each boolean is represented by one bit. This only
2128 // handles 2x and 4x compaction ratios.
2129 unsigned PredLen = ValTy.getVectorNumElements();
2130 if (PredLen < HwLen) {
2131 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2132 Val = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Val);
2133 if (HwLen > PredLen * 2) {
2134 assert(HwLen == PredLen * 4);
2135 PredLen *= 2;
2136 Val = getInstr(MachineOpc: Hexagon::V6_vdealh, dl, Ty: ByteTy, Ops: Val, DAG);
2137 }
2138 if (HwLen > PredLen) {
2139 assert(HwLen == PredLen * 2);
2140 Val = getInstr(MachineOpc: Hexagon::V6_vdealb, dl, Ty: ByteTy, Ops: Val, DAG);
2141 }
2142 Val = DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ValTy, Operand: Val);
2143 }
2144
2145 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2146 unsigned BitWidth = ResTy.getSizeInBits();
2147
2148 if (BitWidth < 64) {
2149 SDValue W0 = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
2150 dl, ResTy: MVT::i32, DAG);
2151 if (BitWidth == 32)
2152 return W0;
2153 assert(BitWidth < 32u);
2154 return DAG.getZExtOrTrunc(Op: W0, DL: dl, VT: ResTy);
2155 }
2156
2157 // The result is >= 64 bits. The only options are 64 or 128.
2158 assert(BitWidth == 64 || BitWidth == 128);
2159 SmallVector<SDValue,4> Words;
2160 for (unsigned i = 0; i != BitWidth/32; ++i) {
2161 SDValue W = extractHvxElementReg(
2162 VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl, ResTy: MVT::i32, DAG);
2163 Words.push_back(Elt: W);
2164 }
2165 SmallVector<SDValue,2> Combines;
2166 assert(Words.size() % 2 == 0);
2167 for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
2168 SDValue C = getCombine(Hi: Words[i+1], Lo: Words[i], dl, ResTy: MVT::i64, DAG);
2169 Combines.push_back(Elt: C);
2170 }
2171
2172 if (BitWidth == 64)
2173 return Combines[0];
2174
2175 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: ResTy, Ops: Combines);
2176 }
2177
2178 // Handle bitcast from i32, v2i16, and v4i8 to v32i1.
2179 // Splat the input into a 32-element i32 vector, then AND each element
2180 // with a unique bitmask to isolate individual bits.
2181 auto bitcastI32ToV32I1 = [&](SDValue Val32) {
2182 assert(Val32.getValueType().getSizeInBits() == 32 &&
2183 "Input must be 32 bits");
2184 MVT VecTy = MVT::getVectorVT(VT: MVT::i32, NumElements: 32);
2185 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: VecTy, Operand: Val32);
2186 SmallVector<SDValue, 32> Mask;
2187 for (unsigned i = 0; i < 32; ++i)
2188 Mask.push_back(Elt: DAG.getConstant(Val: 1ull << i, DL: dl, VT: MVT::i32));
2189
2190 SDValue MaskVec = DAG.getBuildVector(VT: VecTy, DL: dl, Ops: Mask);
2191 SDValue Anded = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: VecTy, N1: Splat, N2: MaskVec);
2192 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: MVT::v32i1, Operand: Anded);
2193 };
2194 // === Case: v32i1 ===
2195 if (ResTy == MVT::v32i1 &&
2196 (ValTy == MVT::i32 || ValTy == MVT::v2i16 || ValTy == MVT::v4i8) &&
2197 Subtarget.useHVX128BOps()) {
2198 SDValue Val32 = Val;
2199 if (ValTy == MVT::v2i16 || ValTy == MVT::v4i8)
2200 Val32 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Val);
2201 return bitcastI32ToV32I1(Val32);
2202 }
2203 // === Case: v64i1 ===
2204 if (ResTy == MVT::v64i1 && ValTy == MVT::i64 && Subtarget.useHVX128BOps()) {
2205 // Split i64 into lo/hi 32-bit halves.
2206 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Val);
2207 SDValue HiShifted = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Val,
2208 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64));
2209 SDValue Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: HiShifted);
2210
2211 // Reuse the same 32-bit logic twice.
2212 SDValue LoRes = bitcastI32ToV32I1(Lo);
2213 SDValue HiRes = bitcastI32ToV32I1(Hi);
2214
2215 // Concatenate into a v64i1 predicate.
2216 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v64i1, N1: LoRes, N2: HiRes);
2217 }
2218
2219 if (isHvxBoolTy(Ty: ResTy) && ValTy.isScalarInteger()) {
2220 // Handle bitcast from i128 -> v128i1 and i64 -> v64i1.
2221 unsigned BitWidth = ValTy.getSizeInBits();
2222 unsigned HwLen = Subtarget.getVectorLength();
2223 assert(BitWidth == HwLen);
2224
2225 MVT ValAsVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: BitWidth / 8);
2226 SDValue ValAsVec = DAG.getBitcast(VT: ValAsVecTy, V: Val);
2227 // Splat each byte of Val 8 times.
2228 // Bytes = [(b0)x8, (b1)x8, ...., (b15)x8]
2229 // where b0, b1,..., b15 are least to most significant bytes of I.
2230 SmallVector<SDValue, 128> Bytes;
2231 // Tmp: 0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80, 0x01,0x02,0x04,0x08,...
2232 // These are bytes with the LSB rotated left with respect to their index.
2233 SmallVector<SDValue, 128> Tmp;
2234 for (unsigned I = 0; I != HwLen / 8; ++I) {
2235 SDValue Idx = DAG.getConstant(Val: I, DL: dl, VT: MVT::i32);
2236 SDValue Byte =
2237 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i8, N1: ValAsVec, N2: Idx);
2238 for (unsigned J = 0; J != 8; ++J) {
2239 Bytes.push_back(Elt: Byte);
2240 Tmp.push_back(Elt: DAG.getConstant(Val: 1ull << J, DL: dl, VT: MVT::i8));
2241 }
2242 }
2243
2244 MVT ConstantVecTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2245 SDValue ConstantVec = DAG.getBuildVector(VT: ConstantVecTy, DL: dl, Ops: Tmp);
2246 SDValue I2V = buildHvxVectorReg(Values: Bytes, dl, VecTy: ConstantVecTy, DAG);
2247
2248 // Each Byte in the I2V will be set iff corresponding bit is set in Val.
2249 I2V = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ConstantVecTy, Ops: {I2V, ConstantVec});
2250 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: I2V);
2251 }
2252
2253 return Op;
2254}
2255
2256SDValue HexagonTargetLowering::LowerHvxStore(SDValue Op,
2257 SelectionDAG &DAG) const {
2258 const SDLoc &dl(Op);
2259 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
2260 SDValue Val = SN->getValue();
2261 MVT ValTy = ty(Op: Val);
2262
2263 // Check if this is a store of an HVX bool vector (predicate)
2264 if (!isHvxBoolTy(Ty: ValTy))
2265 return SDValue();
2266
2267 unsigned NumBits = ValTy.getVectorNumElements();
2268 MachineMemOperand *MMO = SN->getMemOperand();
2269
2270 // Check alignment requirements based on predicate size
2271 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2272 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2273 return SDValue();
2274
2275 unsigned HwLen = Subtarget.getVectorLength();
2276 MVT WordTy = MVT::getVectorVT(VT: MVT::i32, NumElements: HwLen / 4);
2277
2278 // Compress the predicate into a vector register
2279 SDValue VQ = compressHvxPred(VecQ: Val, dl, ResTy: WordTy, DAG);
2280
2281 // Extract words from the compressed vector
2282 SmallVector<SDValue, 4> Words;
2283 for (unsigned i = 0; i != NumBits / 32; ++i) {
2284 SDValue W = extractHvxElementReg(VecV: VQ, IdxV: DAG.getConstant(Val: i, DL: dl, VT: MVT::i32), dl,
2285 ResTy: MVT::i32, DAG);
2286 Words.push_back(Elt: W);
2287 }
2288
2289 SDValue Chain = SN->getChain();
2290 SDValue BasePtr = SN->getBasePtr();
2291 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2292
2293 if (NumBits == 32)
2294 return DAG.getStore(Chain, dl, Val: Words[0], Ptr: BasePtr, PtrInfo,
2295 Alignment: MMO->getBaseAlign());
2296
2297 if (NumBits == 64) {
2298 SDValue W64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2299 return DAG.getStore(Chain, dl, Val: W64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2300 }
2301
2302 if (NumBits == 128) {
2303 SDValue Lo64 = getCombine(Hi: Words[1], Lo: Words[0], dl, ResTy: MVT::i64, DAG);
2304 SDValue Hi64 = getCombine(Hi: Words[3], Lo: Words[2], dl, ResTy: MVT::i64, DAG);
2305
2306 Chain =
2307 DAG.getStore(Chain, dl, Val: Lo64, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2308
2309 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2310 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2311 return DAG.getStore(Chain, dl, Val: Hi64, Ptr: Ptr8, PtrInfo: PtrInfo.getWithOffset(O: 8),
2312 Alignment: Align(8));
2313 }
2314
2315 return SDValue();
2316}
2317
2318SDValue HexagonTargetLowering::LowerHvxLoad(SDValue Op,
2319 SelectionDAG &DAG) const {
2320 const SDLoc &dl(Op);
2321 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
2322 MVT ResTy = ty(Op);
2323
2324 // Check if this is a load of an HVX bool vector (predicate)
2325 if (!isHvxBoolTy(Ty: ResTy))
2326 return SDValue();
2327
2328 unsigned NumBits = ResTy.getVectorNumElements();
2329 MachineMemOperand *MMO = LN->getMemOperand();
2330
2331 unsigned RequiredAlign = (NumBits == 32) ? 4 : 8;
2332 if (MMO->getBaseAlign().value() % RequiredAlign != 0)
2333 return SDValue();
2334
2335 SDValue Chain = LN->getChain();
2336 SDValue BasePtr = LN->getBasePtr();
2337 MachinePointerInfo PtrInfo = MMO->getPointerInfo();
2338
2339 if (NumBits == 32) {
2340 SDValue W32 =
2341 DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2342 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v32i1, Operand: W32);
2343 SDValue Ops[] = {Pred, W32.getValue(R: 1)};
2344 return DAG.getMergeValues(Ops, dl);
2345 }
2346
2347 if (NumBits == 64) {
2348 SDValue W64 =
2349 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2350 SDValue Pred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: W64);
2351 SDValue Ops[] = {Pred, W64.getValue(R: 1)};
2352 return DAG.getMergeValues(Ops, dl);
2353 }
2354
2355 if (NumBits == 128) {
2356 SDValue Lo64 =
2357 DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: BasePtr, PtrInfo, Alignment: MMO->getBaseAlign());
2358 Chain = Lo64.getValue(R: 1);
2359
2360 SDValue Offset8 = DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32);
2361 SDValue Ptr8 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: BasePtr, N2: Offset8);
2362 SDValue Hi64 = DAG.getLoad(VT: MVT::i64, dl, Chain, Ptr: Ptr8,
2363 PtrInfo: PtrInfo.getWithOffset(O: 8), Alignment: Align(8));
2364
2365 SDValue LoPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Lo64);
2366 SDValue HiPred = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v64i1, Operand: Hi64);
2367 SDValue Pred =
2368 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MVT::v128i1, N1: LoPred, N2: HiPred);
2369
2370 SDValue Ops[] = {Pred, Hi64.getValue(R: 1)};
2371 return DAG.getMergeValues(Ops, dl);
2372 }
2373
2374 return SDValue();
2375}
2376
2377SDValue
2378HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
2379 // Sign- and zero-extends are legal.
2380 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
2381 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL: SDLoc(Op), VT: ty(Op),
2382 Operand: Op.getOperand(i: 0));
2383}
2384
2385SDValue
2386HexagonTargetLowering::LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const {
2387 MVT ResTy = ty(Op);
2388 if (ResTy.getVectorElementType() != MVT::i1)
2389 return Op;
2390
2391 const SDLoc &dl(Op);
2392 unsigned HwLen = Subtarget.getVectorLength();
2393 unsigned VecLen = ResTy.getVectorNumElements();
2394 assert(HwLen % VecLen == 0);
2395 unsigned ElemSize = HwLen / VecLen;
2396
2397 MVT VecTy = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemSize * 8), NumElements: VecLen);
2398 SDValue S =
2399 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: VecTy, N1: Op.getOperand(i: 0),
2400 N2: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 1)),
2401 N3: DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: VecTy, Operand: Op.getOperand(i: 2)));
2402 return DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: ResTy, Operand: S);
2403}
2404
2405SDValue
2406HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
2407 if (SDValue S = getVectorShiftByInt(Op, DAG))
2408 return S;
2409 return Op;
2410}
2411
2412SDValue
2413HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op,
2414 SelectionDAG &DAG) const {
2415 unsigned Opc = Op.getOpcode();
2416 assert(Opc == ISD::FSHL || Opc == ISD::FSHR);
2417
2418 // Make sure the shift amount is within the range of the bitwidth
2419 // of the element type.
2420 SDValue A = Op.getOperand(i: 0);
2421 SDValue B = Op.getOperand(i: 1);
2422 SDValue S = Op.getOperand(i: 2);
2423
2424 MVT InpTy = ty(Op: A);
2425 MVT ElemTy = InpTy.getVectorElementType();
2426
2427 const SDLoc &dl(Op);
2428 unsigned ElemWidth = ElemTy.getSizeInBits();
2429 bool IsLeft = Opc == ISD::FSHL;
2430
2431 // The expansion into regular shifts produces worse code for i8 and for
2432 // right shift of i32 on v65+.
2433 bool UseShifts = ElemTy != MVT::i8;
2434 if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32)
2435 UseShifts = false;
2436
2437 if (SDValue SplatV = getSplatValue(Op: S, DAG); SplatV && UseShifts) {
2438 // If this is a funnel shift by a scalar, lower it into regular shifts.
2439 SDValue Mask = DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: MVT::i32);
2440 SDValue ModS =
2441 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
2442 Ops: {DAG.getZExtOrTrunc(Op: SplatV, DL: dl, VT: MVT::i32), Mask});
2443 SDValue NegS =
2444 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: MVT::i32,
2445 Ops: {DAG.getConstant(Val: ElemWidth, DL: dl, VT: MVT::i32), ModS});
2446 SDValue IsZero =
2447 DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: ModS, RHS: getZero(dl, Ty: MVT::i32, DAG), Cond: ISD::SETEQ);
2448 // FSHL A, B => A << | B >>n
2449 // FSHR A, B => A <<n | B >>
2450 SDValue Part1 =
2451 DAG.getNode(Opcode: HexagonISD::VASL, DL: dl, VT: InpTy, Ops: {A, IsLeft ? ModS : NegS});
2452 SDValue Part2 =
2453 DAG.getNode(Opcode: HexagonISD::VLSR, DL: dl, VT: InpTy, Ops: {B, IsLeft ? NegS : ModS});
2454 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Part1, Part2});
2455 // If the shift amount was 0, pick A or B, depending on the direction.
2456 // The opposite shift will also be by 0, so the "Or" will be incorrect.
2457 return DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InpTy, Ops: {IsZero, (IsLeft ? A : B), Or});
2458 }
2459
2460 SDValue Mask = DAG.getSplatBuildVector(
2461 VT: InpTy, DL: dl, Op: DAG.getConstant(Val: ElemWidth - 1, DL: dl, VT: ElemTy));
2462
2463 unsigned MOpc = Opc == ISD::FSHL ? HexagonISD::MFSHL : HexagonISD::MFSHR;
2464 return DAG.getNode(Opcode: MOpc, DL: dl, VT: ty(Op),
2465 Ops: {A, B, DAG.getNode(Opcode: ISD::AND, DL: dl, VT: InpTy, Ops: {S, Mask})});
2466}
2467
2468SDValue
2469HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
2470 const SDLoc &dl(Op);
2471 unsigned IntNo = Op.getConstantOperandVal(i: 0);
2472 SmallVector<SDValue> Ops(Op->ops());
2473
2474 auto Swap = [&](SDValue P) {
2475 return DAG.getMergeValues(Ops: {P.getValue(R: 1), P.getValue(R: 0)}, dl);
2476 };
2477
2478 switch (IntNo) {
2479 case Intrinsic::hexagon_V6_pred_typecast:
2480 case Intrinsic::hexagon_V6_pred_typecast_128B: {
2481 MVT ResTy = ty(Op), InpTy = ty(Op: Ops[1]);
2482 if (isHvxBoolTy(Ty: ResTy) && isHvxBoolTy(Ty: InpTy)) {
2483 if (ResTy == InpTy)
2484 return Ops[1];
2485 return DAG.getNode(Opcode: HexagonISD::TYPECAST, DL: dl, VT: ResTy, Operand: Ops[1]);
2486 }
2487 break;
2488 }
2489 case Intrinsic::hexagon_V6_vmpyss_parts:
2490 case Intrinsic::hexagon_V6_vmpyss_parts_128B:
2491 return Swap(DAG.getNode(Opcode: HexagonISD::SMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2492 Ops: {Ops[1], Ops[2]}));
2493 case Intrinsic::hexagon_V6_vmpyuu_parts:
2494 case Intrinsic::hexagon_V6_vmpyuu_parts_128B:
2495 return Swap(DAG.getNode(Opcode: HexagonISD::UMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2496 Ops: {Ops[1], Ops[2]}));
2497 case Intrinsic::hexagon_V6_vmpyus_parts:
2498 case Intrinsic::hexagon_V6_vmpyus_parts_128B: {
2499 return Swap(DAG.getNode(Opcode: HexagonISD::USMUL_LOHI, DL: dl, VTList: Op->getVTList(),
2500 Ops: {Ops[1], Ops[2]}));
2501 }
2502 } // switch
2503
2504 return Op;
2505}
2506
2507SDValue
2508HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const {
2509 const SDLoc &dl(Op);
2510 unsigned HwLen = Subtarget.getVectorLength();
2511 MachineFunction &MF = DAG.getMachineFunction();
2512 auto *MaskN = cast<MaskedLoadStoreSDNode>(Val: Op.getNode());
2513 SDValue Mask = MaskN->getMask();
2514 SDValue Chain = MaskN->getChain();
2515 SDValue Base = MaskN->getBasePtr();
2516 auto *MemOp = MF.getMachineMemOperand(MMO: MaskN->getMemOperand(), Offset: 0, Size: HwLen);
2517
2518 unsigned Opc = Op->getOpcode();
2519 assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE);
2520
2521 if (Opc == ISD::MLOAD) {
2522 MVT ValTy = ty(Op);
2523 SDValue Load = DAG.getLoad(VT: ValTy, dl, Chain, Ptr: Base, MMO: MemOp);
2524 SDValue Thru = cast<MaskedLoadSDNode>(Val: MaskN)->getPassThru();
2525 if (isUndef(Op: Thru))
2526 return Load;
2527 SDValue VSel = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ValTy, N1: Mask, N2: Load, N3: Thru);
2528 return DAG.getMergeValues(Ops: {VSel, Load.getValue(R: 1)}, dl);
2529 }
2530
2531 // MSTORE
2532 // HVX only has aligned masked stores.
2533
2534 // TODO: Fold negations of the mask into the store.
2535 unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
2536 SDValue Value = cast<MaskedStoreSDNode>(Val: MaskN)->getValue();
2537 SDValue Offset0 = DAG.getTargetConstant(Val: 0, DL: dl, VT: ty(Op: Base));
2538
2539 if (MaskN->getAlign().value() % HwLen == 0) {
2540 SDValue Store = getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2541 Ops: {Mask, Base, Offset0, Value, Chain}, DAG);
2542 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Store.getNode()), NewMemRefs: {MemOp});
2543 return Store;
2544 }
2545
2546 // Unaligned case.
2547 auto StoreAlign = [&](SDValue V, SDValue A) {
2548 SDValue Z = getZero(dl, Ty: ty(Op: V), DAG);
2549 // TODO: use funnel shifts?
2550 // vlalign(Vu,Vv,Rt) rotates the pair Vu:Vv left by Rt and takes the
2551 // upper half.
2552 SDValue LoV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {V, Z, A}, DAG);
2553 SDValue HiV = getInstr(MachineOpc: Hexagon::V6_vlalignb, dl, Ty: ty(Op: V), Ops: {Z, V, A}, DAG);
2554 return std::make_pair(x&: LoV, y&: HiV);
2555 };
2556
2557 MVT ByteTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
2558 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
2559 SDValue MaskV = DAG.getNode(Opcode: HexagonISD::Q2V, DL: dl, VT: ByteTy, Operand: Mask);
2560 VectorPair Tmp = StoreAlign(MaskV, Base);
2561 VectorPair MaskU = {DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.first),
2562 DAG.getNode(Opcode: HexagonISD::V2Q, DL: dl, VT: BoolTy, Operand: Tmp.second)};
2563 VectorPair ValueU = StoreAlign(Value, Base);
2564
2565 SDValue Offset1 = DAG.getTargetConstant(Val: HwLen, DL: dl, VT: MVT::i32);
2566 SDValue StoreLo =
2567 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2568 Ops: {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
2569 SDValue StoreHi =
2570 getInstr(MachineOpc: StoreOpc, dl, Ty: MVT::Other,
2571 Ops: {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
2572 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreLo.getNode()), NewMemRefs: {MemOp});
2573 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: StoreHi.getNode()), NewMemRefs: {MemOp});
2574 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: {StoreLo, StoreHi});
2575}
2576
2577SDValue HexagonTargetLowering::LowerHvxFpExtend(SDValue Op,
2578 SelectionDAG &DAG) const {
2579 // This conversion only applies to QFloat. IEEE extension from f16 to f32
2580 // is legal (done via a pattern).
2581 assert(Subtarget.useHVXQFloatOps());
2582
2583 assert(Op->getOpcode() == ISD::FP_EXTEND);
2584
2585 MVT VecTy = ty(Op);
2586 MVT ArgTy = ty(Op: Op.getOperand(i: 0));
2587 const SDLoc &dl(Op);
2588
2589 if (ArgTy == MVT::v64bf16) {
2590 MVT HalfTy = typeSplit(VecTy).first;
2591 SDValue BF16Vec = Op.getOperand(i: 0);
2592 SDValue Zeroes =
2593 getInstr(MachineOpc: Hexagon::V6_vxor, dl, Ty: HalfTy, Ops: {BF16Vec, BF16Vec}, DAG);
2594 // Interleave zero vector with the bf16 vector, with zeroes in the lower
2595 // half of each 32 bit lane, effectively extending the bf16 values to fp32
2596 // values.
2597 SDValue ShuffVec =
2598 getInstr(MachineOpc: Hexagon::V6_vshufoeh, dl, Ty: VecTy, Ops: {BF16Vec, Zeroes}, DAG);
2599 VectorPair VecPair = opSplit(Vec: ShuffVec, dl, DAG);
2600 SDValue Result = getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2601 Ops: {VecPair.second, VecPair.first,
2602 DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)},
2603 DAG);
2604 return Result;
2605 }
2606
2607 assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16);
2608
2609 SDValue F16Vec = Op.getOperand(i: 0);
2610
2611 APFloat FloatVal = APFloat(1.0f);
2612 bool Ignored;
2613 FloatVal.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
2614 SDValue Fp16Ones = DAG.getConstantFP(Val: FloatVal, DL: dl, VT: ArgTy);
2615 SDValue VmpyVec =
2616 getInstr(MachineOpc: Hexagon::V6_vmpy_qf32_hf, dl, Ty: VecTy, Ops: {F16Vec, Fp16Ones}, DAG);
2617
2618 MVT HalfTy = typeSplit(VecTy).first;
2619 VectorPair Pair = opSplit(Vec: VmpyVec, dl, DAG);
2620 SDValue LoVec =
2621 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.first}, DAG);
2622 SDValue HiVec =
2623 getInstr(MachineOpc: Hexagon::V6_vconv_sf_qf32, dl, Ty: HalfTy, Ops: {Pair.second}, DAG);
2624
2625 SDValue ShuffVec =
2626 getInstr(MachineOpc: Hexagon::V6_vshuffvdd, dl, Ty: VecTy,
2627 Ops: {HiVec, LoVec, DAG.getSignedConstant(Val: -4, DL: dl, VT: MVT::i32)}, DAG);
2628
2629 return ShuffVec;
2630}
2631
2632SDValue
2633HexagonTargetLowering::LowerHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
2634 // Catch invalid conversion ops (just in case).
2635 assert(Op.getOpcode() == ISD::FP_TO_SINT ||
2636 Op.getOpcode() == ISD::FP_TO_UINT);
2637
2638 MVT ResTy = ty(Op);
2639 MVT FpTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2640 MVT IntTy = ResTy.getVectorElementType();
2641
2642 if (Subtarget.useHVXIEEEFPOps()) {
2643 // There are only conversions from f16.
2644 if (FpTy == MVT::f16) {
2645 // Other int types aren't legal in HVX, so we shouldn't see them here.
2646 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2647 // Conversions to i8 and i16 are legal.
2648 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2649 return Op;
2650 }
2651 }
2652
2653 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2654 return EqualizeFpIntConversion(Op, DAG);
2655
2656 return ExpandHvxFpToInt(Op, DAG);
2657}
2658
2659// For vector type v32i1 uint_to_fp/sint_to_fp to v32f32:
2660// R1 = #1, R2 holds the v32i1 param
2661// V1 = vsplat(R1)
2662// V2 = vsplat(R2)
2663// Q0 = vand(V1,R1)
2664// V0.w=prefixsum(Q0)
2665// V0.w=vsub(V0.w,V1.w)
2666// V2.w = vlsr(V2.w,V0.w)
2667// V2 = vand(V2,V1)
2668// V2.sf = V2.w
2669SDValue HexagonTargetLowering::LowerHvxPred32ToFp(SDValue PredOp,
2670 SelectionDAG &DAG) const {
2671
2672 MVT ResTy = ty(Op: PredOp);
2673 const SDLoc &dl(PredOp);
2674
2675 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2676 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2677 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2678 Op1: SDValue(RegConst, 0));
2679 SDNode *PredTransfer =
2680 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2681 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2682 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2683 Op1: SDValue(PredTransfer, 0));
2684 SDNode *SplatParam = DAG.getMachineNode(
2685 Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2686 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: PredOp.getOperand(i: 0)));
2687 SDNode *Vsub =
2688 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2689 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2690 SDNode *IndexShift =
2691 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2692 Op1: SDValue(SplatParam, 0), Op2: SDValue(Vsub, 0));
2693 SDNode *MaskOff =
2694 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2695 Op1: SDValue(IndexShift, 0), Op2: SDValue(SplatConst, 0));
2696 SDNode *Convert = DAG.getMachineNode(Opcode: Hexagon::V6_vconv_sf_w, dl, VT: ResTy,
2697 Op1: SDValue(MaskOff, 0));
2698 return SDValue(Convert, 0);
2699}
2700
2701// For vector type v64i1 uint_to_fo to v64f16:
2702// i64 R32 = bitcast v64i1 R3:2 (R3:2 holds v64i1)
2703// R3 = subreg_high (R32)
2704// R2 = subreg_low (R32)
2705// R1 = #1
2706// V1 = vsplat(R1)
2707// V2 = vsplat(R2)
2708// V3 = vsplat(R3)
2709// Q0 = vand(V1,R1)
2710// V0.w=prefixsum(Q0)
2711// V0.w=vsub(V0.w,V1.w)
2712// V2.w = vlsr(V2.w,V0.w)
2713// V3.w = vlsr(V3.w,V0.w)
2714// V2 = vand(V2,V1)
2715// V3 = vand(V3,V1)
2716// V2.h = vpacke(V3.w,V2.w)
2717// V2.hf = V2.h
2718SDValue HexagonTargetLowering::LowerHvxPred64ToFp(SDValue PredOp,
2719 SelectionDAG &DAG) const {
2720
2721 MVT ResTy = ty(Op: PredOp);
2722 const SDLoc &dl(PredOp);
2723
2724 SDValue Inp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: PredOp.getOperand(i: 0));
2725 // Get the hi and lo regs
2726 SDValue HiReg =
2727 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_hi, DL: dl, VT: MVT::i32, Operand: Inp);
2728 SDValue LoReg =
2729 DAG.getTargetExtractSubreg(SRIdx: Hexagon::isub_lo, DL: dl, VT: MVT::i32, Operand: Inp);
2730 // Get constant #1 and splat into vector V1
2731 SDValue Const = DAG.getTargetConstant(Val: 0x1, DL: dl, VT: MVT::i32);
2732 SDNode *RegConst = DAG.getMachineNode(Opcode: Hexagon::A2_tfrsi, dl, VT: MVT::i32, Op1: Const);
2733 SDNode *SplatConst = DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2734 Op1: SDValue(RegConst, 0));
2735 // Splat the hi and lo args
2736 SDNode *SplatHi =
2737 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2738 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: HiReg));
2739 SDNode *SplatLo =
2740 DAG.getMachineNode(Opcode: Hexagon::V6_lvsplatw, dl, VT: MVT::v32i32,
2741 Op1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: LoReg));
2742 // vand between splatted const and const
2743 SDNode *PredTransfer =
2744 DAG.getMachineNode(Opcode: Hexagon::V6_vandvrt, dl, VT: MVT::v32i1,
2745 Op1: SDValue(SplatConst, 0), Op2: SDValue(RegConst, 0));
2746 // Get the prefixsum
2747 SDNode *PrefixSum = DAG.getMachineNode(Opcode: Hexagon::V6_vprefixqw, dl, VT: MVT::v32i32,
2748 Op1: SDValue(PredTransfer, 0));
2749 // Get the vsub
2750 SDNode *Vsub =
2751 DAG.getMachineNode(Opcode: Hexagon::V6_vsubw, dl, VT: MVT::v32i32,
2752 Op1: SDValue(PrefixSum, 0), Op2: SDValue(SplatConst, 0));
2753 // Get vlsr for hi and lo
2754 SDNode *IndexShift_hi =
2755 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2756 Op1: SDValue(SplatHi, 0), Op2: SDValue(Vsub, 0));
2757 SDNode *IndexShift_lo =
2758 DAG.getMachineNode(Opcode: Hexagon::V6_vlsrwv, dl, VT: MVT::v32i32,
2759 Op1: SDValue(SplatLo, 0), Op2: SDValue(Vsub, 0));
2760 // Get vand of hi and lo
2761 SDNode *MaskOff_hi =
2762 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2763 Op1: SDValue(IndexShift_hi, 0), Op2: SDValue(SplatConst, 0));
2764 SDNode *MaskOff_lo =
2765 DAG.getMachineNode(Opcode: Hexagon::V6_vand, dl, VT: MVT::v32i32,
2766 Op1: SDValue(IndexShift_lo, 0), Op2: SDValue(SplatConst, 0));
2767 // Pack them
2768 SDNode *Pack =
2769 DAG.getMachineNode(Opcode: Hexagon::V6_vpackeh, dl, VT: MVT::v64i16,
2770 Op1: SDValue(MaskOff_hi, 0), Op2: SDValue(MaskOff_lo, 0));
2771 SDNode *Convert =
2772 DAG.getMachineNode(Opcode: Hexagon::V6_vconv_hf_h, dl, VT: ResTy, Op1: SDValue(Pack, 0));
2773 return SDValue(Convert, 0);
2774}
2775
2776SDValue
2777HexagonTargetLowering::LowerHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
2778 // Catch invalid conversion ops (just in case).
2779 assert(Op.getOpcode() == ISD::SINT_TO_FP ||
2780 Op.getOpcode() == ISD::UINT_TO_FP);
2781
2782 MVT ResTy = ty(Op);
2783 MVT IntTy = ty(Op: Op.getOperand(i: 0)).getVectorElementType();
2784 MVT FpTy = ResTy.getVectorElementType();
2785
2786 if (Op.getOpcode() == ISD::UINT_TO_FP || Op.getOpcode() == ISD::SINT_TO_FP) {
2787 if (ResTy == MVT::v32f32 && ty(Op: Op.getOperand(i: 0)) == MVT::v32i1)
2788 return LowerHvxPred32ToFp(PredOp: Op, DAG);
2789 if (ResTy == MVT::v64f16 && ty(Op: Op.getOperand(i: 0)) == MVT::v64i1)
2790 return LowerHvxPred64ToFp(PredOp: Op, DAG);
2791 }
2792
2793 if (Subtarget.useHVXIEEEFPOps()) {
2794 // There are only conversions to f16.
2795 if (FpTy == MVT::f16) {
2796 // Other int types aren't legal in HVX, so we shouldn't see them here.
2797 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2798 // i8, i16 -> f16 is legal.
2799 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2800 return Op;
2801 }
2802 }
2803
2804 if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
2805 return EqualizeFpIntConversion(Op, DAG);
2806
2807 return ExpandHvxIntToFp(Op, DAG);
2808}
2809
2810HexagonTargetLowering::TypePair
2811HexagonTargetLowering::typeExtendToWider(MVT Ty0, MVT Ty1) const {
2812 // Compare the widths of elements of the two types, and extend the narrower
2813 // type to match the with of the wider type. For vector types, apply this
2814 // to the element type.
2815 assert(Ty0.isVector() == Ty1.isVector());
2816
2817 MVT ElemTy0 = Ty0.getScalarType();
2818 MVT ElemTy1 = Ty1.getScalarType();
2819
2820 unsigned Width0 = ElemTy0.getSizeInBits();
2821 unsigned Width1 = ElemTy1.getSizeInBits();
2822 unsigned MaxWidth = std::max(a: Width0, b: Width1);
2823
2824 auto getScalarWithWidth = [](MVT ScalarTy, unsigned Width) {
2825 if (ScalarTy.isInteger())
2826 return MVT::getIntegerVT(BitWidth: Width);
2827 assert(ScalarTy.isFloatingPoint());
2828 return MVT::getFloatingPointVT(BitWidth: Width);
2829 };
2830
2831 MVT WideETy0 = getScalarWithWidth(ElemTy0, MaxWidth);
2832 MVT WideETy1 = getScalarWithWidth(ElemTy1, MaxWidth);
2833
2834 if (!Ty0.isVector()) {
2835 // Both types are scalars.
2836 return {WideETy0, WideETy1};
2837 }
2838
2839 // Vector types.
2840 unsigned NumElem = Ty0.getVectorNumElements();
2841 assert(NumElem == Ty1.getVectorNumElements());
2842
2843 return {MVT::getVectorVT(VT: WideETy0, NumElements: NumElem),
2844 MVT::getVectorVT(VT: WideETy1, NumElements: NumElem)};
2845}
2846
2847HexagonTargetLowering::TypePair
2848HexagonTargetLowering::typeWidenToWider(MVT Ty0, MVT Ty1) const {
2849 // Compare the numbers of elements of two vector types, and widen the
2850 // narrower one to match the number of elements in the wider one.
2851 assert(Ty0.isVector() && Ty1.isVector());
2852
2853 unsigned Len0 = Ty0.getVectorNumElements();
2854 unsigned Len1 = Ty1.getVectorNumElements();
2855 if (Len0 == Len1)
2856 return {Ty0, Ty1};
2857
2858 unsigned MaxLen = std::max(a: Len0, b: Len1);
2859 return {MVT::getVectorVT(VT: Ty0.getVectorElementType(), NumElements: MaxLen),
2860 MVT::getVectorVT(VT: Ty1.getVectorElementType(), NumElements: MaxLen)};
2861}
2862
2863MVT
2864HexagonTargetLowering::typeLegalize(MVT Ty, SelectionDAG &DAG) const {
2865 EVT LegalTy = getTypeToTransformTo(Context&: *DAG.getContext(), VT: Ty);
2866 assert(LegalTy.isSimple());
2867 return LegalTy.getSimpleVT();
2868}
2869
2870MVT
2871HexagonTargetLowering::typeWidenToHvx(MVT Ty) const {
2872 unsigned HwWidth = 8 * Subtarget.getVectorLength();
2873 assert(Ty.getSizeInBits() <= HwWidth);
2874 if (Ty.getSizeInBits() == HwWidth)
2875 return Ty;
2876
2877 MVT ElemTy = Ty.getScalarType();
2878 return MVT::getVectorVT(VT: ElemTy, NumElements: HwWidth / ElemTy.getSizeInBits());
2879}
2880
2881HexagonTargetLowering::VectorPair
2882HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B,
2883 const SDLoc &dl, bool Signed, SelectionDAG &DAG) const {
2884 // Compute A+B, return {A+B, O}, where O = vector predicate indicating
2885 // whether an overflow has occurred.
2886 MVT ResTy = ty(Op: A);
2887 assert(ResTy == ty(B));
2888 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: ResTy.getVectorNumElements());
2889
2890 if (!Signed) {
2891 // V62+ has V6_vaddcarry, but it requires input predicate, so it doesn't
2892 // save any instructions.
2893 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2894 SDValue Ovf = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Add, RHS: A, Cond: ISD::SETULT);
2895 return {Add, Ovf};
2896 }
2897
2898 // Signed overflow has happened, if:
2899 // (A, B have the same sign) and (A+B has a different sign from either)
2900 // i.e. (~A xor B) & ((A+B) xor B), then check the sign bit
2901 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResTy, Ops: {A, B});
2902 SDValue NotA =
2903 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {A, DAG.getAllOnesConstant(DL: dl, VT: ResTy)});
2904 SDValue Xor0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {NotA, B});
2905 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: ResTy, Ops: {Add, B});
2906 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResTy, Ops: {Xor0, Xor1});
2907 SDValue MSB =
2908 DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: getZero(dl, Ty: ResTy, DAG), Cond: ISD::SETLT);
2909 return {Add, MSB};
2910}
2911
2912HexagonTargetLowering::VectorPair
2913HexagonTargetLowering::emitHvxShiftRightRnd(SDValue Val, unsigned Amt,
2914 bool Signed, SelectionDAG &DAG) const {
2915 // Shift Val right by Amt bits, round the result to the nearest integer,
2916 // tie-break by rounding halves to even integer.
2917
2918 const SDLoc &dl(Val);
2919 MVT ValTy = ty(Op: Val);
2920
2921 // This should also work for signed integers.
2922 //
2923 // uint tmp0 = inp + ((1 << (Amt-1)) - 1);
2924 // bool ovf = (inp > tmp0);
2925 // uint rup = inp & (1 << (Amt+1));
2926 //
2927 // uint tmp1 = inp >> (Amt-1); // tmp1 == tmp2 iff
2928 // uint tmp2 = tmp0 >> (Amt-1); // the Amt-1 lower bits were all 0
2929 // uint tmp3 = tmp2 + rup;
2930 // uint frac = (tmp1 != tmp2) ? tmp2 >> 1 : tmp3 >> 1;
2931 unsigned ElemWidth = ValTy.getVectorElementType().getSizeInBits();
2932 MVT ElemTy = MVT::getIntegerVT(BitWidth: ElemWidth);
2933 MVT IntTy = tyVector(Ty: ValTy, ElemTy);
2934 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: IntTy.getVectorNumElements());
2935 unsigned ShRight = Signed ? ISD::SRA : ISD::SRL;
2936
2937 SDValue Inp = DAG.getBitcast(VT: IntTy, V: Val);
2938 SDValue LowBits = DAG.getConstant(Val: (1ull << (Amt - 1)) - 1, DL: dl, VT: IntTy);
2939
2940 SDValue AmtP1 = DAG.getConstant(Val: 1ull << Amt, DL: dl, VT: IntTy);
2941 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntTy, Ops: {Inp, AmtP1});
2942 SDValue Zero = getZero(dl, Ty: IntTy, DAG);
2943 SDValue Bit = DAG.getSetCC(DL: dl, VT: PredTy, LHS: And, RHS: Zero, Cond: ISD::SETNE);
2944 SDValue Rup = DAG.getZExtOrTrunc(Op: Bit, DL: dl, VT: IntTy);
2945 auto [Tmp0, Ovf] = emitHvxAddWithOverflow(A: Inp, B: LowBits, dl, Signed, DAG);
2946
2947 SDValue AmtM1 = DAG.getConstant(Val: Amt - 1, DL: dl, VT: IntTy);
2948 SDValue Tmp1 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Inp, N2: AmtM1);
2949 SDValue Tmp2 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, N1: Tmp0, N2: AmtM1);
2950 SDValue Tmp3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntTy, N1: Tmp2, N2: Rup);
2951
2952 SDValue Eq = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Tmp1, RHS: Tmp2, Cond: ISD::SETEQ);
2953 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntTy);
2954 SDValue Tmp4 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp2, One});
2955 SDValue Tmp5 = DAG.getNode(Opcode: ShRight, DL: dl, VT: IntTy, Ops: {Tmp3, One});
2956 SDValue Mux = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: IntTy, Ops: {Eq, Tmp5, Tmp4});
2957 return {Mux, Ovf};
2958}
2959
2960SDValue
2961HexagonTargetLowering::emitHvxMulHsV60(SDValue A, SDValue B, const SDLoc &dl,
2962 SelectionDAG &DAG) const {
2963 MVT VecTy = ty(Op: A);
2964 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
2965 assert(VecTy.getVectorElementType() == MVT::i32);
2966
2967 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
2968
2969 // mulhs(A,B) =
2970 // = [(Hi(A)*2^16 + Lo(A)) *s (Hi(B)*2^16 + Lo(B))] >> 32
2971 // = [Hi(A)*2^16 *s Hi(B)*2^16 + Hi(A) *su Lo(B)*2^16
2972 // + Lo(A) *us (Hi(B)*2^16 + Lo(B))] >> 32
2973 // = [Hi(A) *s Hi(B)*2^32 + Hi(A) *su Lo(B)*2^16 + Lo(A) *us B] >> 32
2974 // The low half of Lo(A)*Lo(B) will be discarded (it's not added to
2975 // anything, so it cannot produce any carry over to higher bits),
2976 // so everything in [] can be shifted by 16 without loss of precision.
2977 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + Lo(A)*B >> 16] >> 16
2978 // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + V6_vmpyewuh(A,B)] >> 16
2979 // The final additions need to make sure to properly maintain any carry-
2980 // out bits.
2981 //
2982 // Hi(B) Lo(B)
2983 // Hi(A) Lo(A)
2984 // --------------
2985 // Lo(B)*Lo(A) | T0 = V6_vmpyewuh(B,A) does this,
2986 // Hi(B)*Lo(A) | + dropping the low 16 bits
2987 // Hi(A)*Lo(B) | T2
2988 // Hi(B)*Hi(A)
2989
2990 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh, dl, Ty: VecTy, Ops: {B, A}, DAG);
2991 // T1 = get Hi(A) into low halves.
2992 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {A, S16}, DAG);
2993 // P0 = interleaved T1.h*B.uh (full precision product)
2994 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyhus, dl, Ty: PairTy, Ops: {T1, B}, DAG);
2995 // T2 = T1.even(h) * B.even(uh), i.e. Hi(A)*Lo(B)
2996 SDValue T2 = LoHalf(V: P0, DAG);
2997 // We need to add T0+T2, recording the carry-out, which will be 1<<16
2998 // added to the final sum.
2999 // P1 = interleaved even/odd 32-bit (unsigned) sums of 16-bit halves
3000 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3001 // P2 = interleaved even/odd 32-bit (signed) sums of 16-bit halves
3002 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vaddhw, dl, Ty: PairTy, Ops: {T0, T2}, DAG);
3003 // T3 = full-precision(T0+T2) >> 16
3004 // The low halves are added-unsigned, the high ones are added-signed.
3005 SDValue T3 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3006 Ops: {HiHalf(V: P2, DAG), LoHalf(V: P1, DAG), S16}, DAG);
3007 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw, dl, Ty: VecTy, Ops: {B, S16}, DAG);
3008 // P3 = interleaved Hi(B)*Hi(A) (full precision),
3009 // which is now Lo(T1)*Lo(T4), so we want to keep the even product.
3010 SDValue P3 = getInstr(MachineOpc: Hexagon::V6_vmpyhv, dl, Ty: PairTy, Ops: {T1, T4}, DAG);
3011 SDValue T5 = LoHalf(V: P3, DAG);
3012 // Add:
3013 SDValue T6 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {T3, T5});
3014 return T6;
3015}
3016
3017SDValue
3018HexagonTargetLowering::emitHvxMulLoHiV60(SDValue A, bool SignedA, SDValue B,
3019 bool SignedB, const SDLoc &dl,
3020 SelectionDAG &DAG) const {
3021 MVT VecTy = ty(Op: A);
3022 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3023 assert(VecTy.getVectorElementType() == MVT::i32);
3024
3025 SDValue S16 = DAG.getConstant(Val: 16, DL: dl, VT: MVT::i32);
3026
3027 if (SignedA && !SignedB) {
3028 // Make A:unsigned, B:signed.
3029 std::swap(a&: A, b&: B);
3030 std::swap(a&: SignedA, b&: SignedB);
3031 }
3032
3033 // Do halfword-wise multiplications for unsigned*unsigned product, then
3034 // add corrections for signed and unsigned*signed.
3035
3036 SDValue Lo, Hi;
3037
3038 // P0:lo = (uu) products of low halves of A and B,
3039 // P0:hi = (uu) products of high halves.
3040 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, B}, DAG);
3041
3042 // Swap low/high halves in B
3043 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_lvsplatw, dl, Ty: VecTy,
3044 Ops: {DAG.getConstant(Val: 0x02020202, DL: dl, VT: MVT::i32)}, DAG);
3045 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vdelta, dl, Ty: VecTy, Ops: {B, T0}, DAG);
3046 // P1 = products of even/odd halfwords.
3047 // P1:lo = (uu) products of even(A.uh) * odd(B.uh)
3048 // P1:hi = (uu) products of odd(A.uh) * even(B.uh)
3049 SDValue P1 = getInstr(MachineOpc: Hexagon::V6_vmpyuhv, dl, Ty: PairTy, Ops: {A, T1}, DAG);
3050
3051 // P2:lo = low halves of P1:lo + P1:hi,
3052 // P2:hi = high halves of P1:lo + P1:hi.
3053 SDValue P2 = getInstr(MachineOpc: Hexagon::V6_vadduhw, dl, Ty: PairTy,
3054 Ops: {HiHalf(V: P1, DAG), LoHalf(V: P1, DAG)}, DAG);
3055 // Still need to add the high halves of P0:lo to P2:lo
3056 SDValue T2 =
3057 getInstr(MachineOpc: Hexagon::V6_vlsrw, dl, Ty: VecTy, Ops: {LoHalf(V: P0, DAG), S16}, DAG);
3058 SDValue T3 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {LoHalf(V: P2, DAG), T2});
3059
3060 // The high halves of T3 will contribute to the HI part of LOHI.
3061 SDValue T4 = getInstr(MachineOpc: Hexagon::V6_vasrw_acc, dl, Ty: VecTy,
3062 Ops: {HiHalf(V: P2, DAG), T3, S16}, DAG);
3063
3064 // The low halves of P2 need to be added to high halves of the LO part.
3065 Lo = getInstr(MachineOpc: Hexagon::V6_vaslw_acc, dl, Ty: VecTy,
3066 Ops: {LoHalf(V: P0, DAG), LoHalf(V: P2, DAG), S16}, DAG);
3067 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: VecTy, Ops: {HiHalf(V: P0, DAG), T4});
3068
3069 if (SignedA) {
3070 assert(SignedB && "Signed A and unsigned B should have been inverted");
3071
3072 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3073 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3074 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3075 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3076 SDValue X0 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: VecTy, Ops: {Q0, B, Zero});
3077 SDValue X1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, X0, A}, DAG);
3078 Hi = getInstr(MachineOpc: Hexagon::V6_vsubw, dl, Ty: VecTy, Ops: {Hi, X1}, DAG);
3079 } else if (SignedB) {
3080 // Same correction as for mulhus:
3081 // mulhus(A.uw,B.w) = mulhu(A.uw,B.uw) - (A.w if B < 0)
3082 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3083 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3084 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3085 Hi = getInstr(MachineOpc: Hexagon::V6_vsubwq, dl, Ty: VecTy, Ops: {Q1, Hi, A}, DAG);
3086 } else {
3087 assert(!SignedA && !SignedB);
3088 }
3089
3090 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3091}
3092
3093SDValue
3094HexagonTargetLowering::emitHvxMulLoHiV62(SDValue A, bool SignedA,
3095 SDValue B, bool SignedB,
3096 const SDLoc &dl,
3097 SelectionDAG &DAG) const {
3098 MVT VecTy = ty(Op: A);
3099 MVT PairTy = typeJoin(Tys: {VecTy, VecTy});
3100 assert(VecTy.getVectorElementType() == MVT::i32);
3101
3102 if (SignedA && !SignedB) {
3103 // Make A:unsigned, B:signed.
3104 std::swap(a&: A, b&: B);
3105 std::swap(a&: SignedA, b&: SignedB);
3106 }
3107
3108 // Do S*S first, then make corrections for U*S or U*U if needed.
3109 SDValue P0 = getInstr(MachineOpc: Hexagon::V6_vmpyewuh_64, dl, Ty: PairTy, Ops: {A, B}, DAG);
3110 SDValue P1 =
3111 getInstr(MachineOpc: Hexagon::V6_vmpyowh_64_acc, dl, Ty: PairTy, Ops: {P0, A, B}, DAG);
3112 SDValue Lo = LoHalf(V: P1, DAG);
3113 SDValue Hi = HiHalf(V: P1, DAG);
3114
3115 if (!SignedB) {
3116 assert(!SignedA && "Signed A and unsigned B should have been inverted");
3117 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3118 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3119
3120 // Mulhu(X, Y) = Mulhs(X, Y) + (X, if Y < 0) + (Y, if X < 0).
3121 // def: Pat<(VecI32 (mulhu HVI32:$A, HVI32:$B)),
3122 // (V6_vaddw (HiHalf (Muls64O $A, $B)),
3123 // (V6_vaddwq (V6_vgtw (V6_vd0), $B),
3124 // (V6_vandvqv (V6_vgtw (V6_vd0), $A), $B),
3125 // $A))>;
3126 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3127 SDValue Q1 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: B, RHS: Zero, Cond: ISD::SETLT);
3128 SDValue T0 = getInstr(MachineOpc: Hexagon::V6_vandvqv, dl, Ty: VecTy, Ops: {Q0, B}, DAG);
3129 SDValue T1 = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q1, T0, A}, DAG);
3130 Hi = getInstr(MachineOpc: Hexagon::V6_vaddw, dl, Ty: VecTy, Ops: {Hi, T1}, DAG);
3131 } else if (!SignedA) {
3132 SDValue Zero = getZero(dl, Ty: VecTy, DAG);
3133 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, NumElements: VecTy.getVectorNumElements());
3134
3135 // Mulhus(unsigned X, signed Y) = Mulhs(X, Y) + (Y, if X < 0).
3136 // def: Pat<(VecI32 (HexagonMULHUS HVI32:$A, HVI32:$B)),
3137 // (V6_vaddwq (V6_vgtw (V6_vd0), $A),
3138 // (HiHalf (Muls64O $A, $B)),
3139 // $B)>;
3140 SDValue Q0 = DAG.getSetCC(DL: dl, VT: PredTy, LHS: A, RHS: Zero, Cond: ISD::SETLT);
3141 Hi = getInstr(MachineOpc: Hexagon::V6_vaddwq, dl, Ty: VecTy, Ops: {Q0, Hi, B}, DAG);
3142 }
3143
3144 return DAG.getMergeValues(Ops: {Lo, Hi}, dl);
3145}
3146
3147SDValue
3148HexagonTargetLowering::EqualizeFpIntConversion(SDValue Op, SelectionDAG &DAG)
3149 const {
3150 // Rewrite conversion between integer and floating-point in such a way that
3151 // the integer type is extended/narrowed to match the bitwidth of the
3152 // floating-point type, combined with additional integer-integer extensions
3153 // or narrowings to match the original input/result types.
3154 // E.g. f32 -> i8 ==> f32 -> i32 -> i8
3155 //
3156 // The input/result types are not required to be legal, but if they are
3157 // legal, this function should not introduce illegal types.
3158
3159 unsigned Opc = Op.getOpcode();
3160 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT ||
3161 Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3162
3163 SDValue Inp = Op.getOperand(i: 0);
3164 MVT InpTy = ty(Op: Inp);
3165 MVT ResTy = ty(Op);
3166
3167 if (InpTy == ResTy)
3168 return Op;
3169
3170 const SDLoc &dl(Op);
3171 bool Signed = Opc == ISD::FP_TO_SINT || Opc == ISD::SINT_TO_FP;
3172
3173 auto [WInpTy, WResTy] = typeExtendToWider(Ty0: InpTy, Ty1: ResTy);
3174 SDValue WInp = resizeToWidth(VecV: Inp, ResTy: WInpTy, Signed, dl, DAG);
3175 SDValue Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: WResTy, Operand: WInp);
3176 SDValue Res = resizeToWidth(VecV: Conv, ResTy, Signed, dl, DAG);
3177 return Res;
3178}
3179
3180SDValue
3181HexagonTargetLowering::ExpandHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
3182 unsigned Opc = Op.getOpcode();
3183 assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT);
3184
3185 const SDLoc &dl(Op);
3186 SDValue Op0 = Op.getOperand(i: 0);
3187 MVT InpTy = ty(Op: Op0);
3188 MVT ResTy = ty(Op);
3189 assert(InpTy.changeTypeToInteger() == ResTy);
3190
3191 // At this point this is an experiment under a flag.
3192 // In arch before V81 the rounding mode is towards nearest value.
3193 // The C/C++ standard requires rounding towards zero:
3194 // C (C99 and later): ISO/IEC 9899:2018 (C18), section 6.3.1.4 — "When a
3195 // finite value of real floating type is converted to an integer type, the
3196 // fractional part is discarded (i.e., the value is truncated toward zero)."
3197 // C++: ISO/IEC 14882:2020 (C++20), section 7.3.7 — "A prvalue of a
3198 // floating-point type can be converted to a prvalue of an integer type. The
3199 // conversion truncates; that is, the fractional part is discarded."
3200 if (InpTy == MVT::v64f16) {
3201 if (Subtarget.useHVXV81Ops()) {
3202 // This is c/c++ compliant
3203 SDValue ConvVec =
3204 getInstr(MachineOpc: Hexagon::V6_vconv_h_hf_rnd, dl, Ty: ResTy, Ops: {Op0}, DAG);
3205 return ConvVec;
3206 } else if (EnableFpFastConvert) {
3207 // Vd32.h=Vu32.hf same as Q6_Vh_equals_Vhf
3208 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_h_hf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3209 return ConvVec;
3210 }
3211 } else if (EnableFpFastConvert && InpTy == MVT::v32f32) {
3212 // Vd32.w=Vu32.sf same as Q6_Vw_equals_Vsf
3213 SDValue ConvVec = getInstr(MachineOpc: Hexagon::V6_vconv_w_sf, dl, Ty: ResTy, Ops: {Op0}, DAG);
3214 return ConvVec;
3215 }
3216
3217 // int32_t conv_f32_to_i32(uint32_t inp) {
3218 // // s | exp8 | frac23
3219 //
3220 // int neg = (int32_t)inp < 0;
3221 //
3222 // // "expm1" is the actual exponent minus 1: instead of "bias", subtract
3223 // // "bias+1". When the encoded exp is "all-1" (i.e. inf/nan), this will
3224 // // produce a large positive "expm1", which will result in max u/int.
3225 // // In all IEEE formats, bias is the largest positive number that can be
3226 // // represented in bias-width bits (i.e. 011..1).
3227 // int32_t expm1 = (inp << 1) - 0x80000000;
3228 // expm1 >>= 24;
3229 //
3230 // // Always insert the "implicit 1". Subnormal numbers will become 0
3231 // // regardless.
3232 // uint32_t frac = (inp << 8) | 0x80000000;
3233 //
3234 // // "frac" is the fraction part represented as Q1.31. If it was
3235 // // interpreted as uint32_t, it would be the fraction part multiplied
3236 // // by 2^31.
3237 //
3238 // // Calculate the amount of right shift, since shifting further to the
3239 // // left would lose significant bits. Limit it to 32, because we want
3240 // // shifts by 32+ to produce 0, whereas V6_vlsrwv treats the shift
3241 // // amount as a 6-bit signed value (so 33 is same as -31, i.e. shift
3242 // // left by 31). "rsh" can be negative.
3243 // int32_t rsh = min(31 - (expm1 + 1), 32);
3244 //
3245 // frac >>= rsh; // rsh == 32 will produce 0
3246 //
3247 // // Everything up to this point is the same for conversion to signed
3248 // // unsigned integer.
3249 //
3250 // if (neg) // Only for signed int
3251 // frac = -frac; //
3252 // if (rsh <= 0 && neg) // bound = neg ? 0x80000000 : 0x7fffffff
3253 // frac = 0x80000000; // frac = rsh <= 0 ? bound : frac
3254 // if (rsh <= 0 && !neg) //
3255 // frac = 0x7fffffff; //
3256 //
3257 // if (neg) // Only for unsigned int
3258 // frac = 0; //
3259 // if (rsh < 0 && !neg) // frac = rsh < 0 ? 0x7fffffff : frac;
3260 // frac = 0x7fffffff; // frac = neg ? 0 : frac;
3261 //
3262 // return frac;
3263 // }
3264
3265 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: ResTy.getVectorElementCount());
3266
3267 // Zero = V6_vd0();
3268 // Neg = V6_vgtw(Zero, Inp);
3269 // One = V6_lvsplatw(1);
3270 // M80 = V6_lvsplatw(0x80000000);
3271 // Exp00 = V6_vaslwv(Inp, One);
3272 // Exp01 = V6_vsubw(Exp00, M80);
3273 // ExpM1 = V6_vasrw(Exp01, 24);
3274 // Frc00 = V6_vaslw(Inp, 8);
3275 // Frc01 = V6_vor(Frc00, M80);
3276 // Rsh00 = V6_vsubw(V6_lvsplatw(30), ExpM1);
3277 // Rsh01 = V6_vminw(Rsh00, V6_lvsplatw(32));
3278 // Frc02 = V6_vlsrwv(Frc01, Rsh01);
3279
3280 // if signed int:
3281 // Bnd = V6_vmux(Neg, M80, V6_lvsplatw(0x7fffffff))
3282 // Pos = V6_vgtw(Rsh01, Zero);
3283 // Frc13 = V6_vsubw(Zero, Frc02);
3284 // Frc14 = V6_vmux(Neg, Frc13, Frc02);
3285 // Int = V6_vmux(Pos, Frc14, Bnd);
3286 //
3287 // if unsigned int:
3288 // Rsn = V6_vgtw(Zero, Rsh01)
3289 // Frc23 = V6_vmux(Rsn, V6_lvsplatw(0x7fffffff), Frc02)
3290 // Int = V6_vmux(Neg, Zero, Frc23)
3291
3292 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: InpTy);
3293 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3294 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias));
3295
3296 SDValue Inp = DAG.getBitcast(VT: ResTy, V: Op0);
3297 SDValue Zero = getZero(dl, Ty: ResTy, DAG);
3298 SDValue Neg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Inp, RHS: Zero, Cond: ISD::SETLT);
3299 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: ResTy);
3300 SDValue M7F = DAG.getConstant(Val: (1ull << (ElemWidth - 1)) - 1, DL: dl, VT: ResTy);
3301 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResTy);
3302 SDValue Exp00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, One});
3303 SDValue Exp01 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Exp00, M80});
3304 SDValue MNE = DAG.getConstant(Val: ElemWidth - ExpWidth, DL: dl, VT: ResTy);
3305 SDValue ExpM1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: ResTy, Ops: {Exp01, MNE});
3306
3307 SDValue ExpW = DAG.getConstant(Val: ExpWidth, DL: dl, VT: ResTy);
3308 SDValue Frc00 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ResTy, Ops: {Inp, ExpW});
3309 SDValue Frc01 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: ResTy, Ops: {Frc00, M80});
3310
3311 SDValue MN2 = DAG.getConstant(Val: ElemWidth - 2, DL: dl, VT: ResTy);
3312 SDValue Rsh00 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {MN2, ExpM1});
3313 SDValue MW = DAG.getConstant(Val: ElemWidth, DL: dl, VT: ResTy);
3314 SDValue Rsh01 = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ResTy, Ops: {Rsh00, MW});
3315 SDValue Frc02 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ResTy, Ops: {Frc01, Rsh01});
3316
3317 SDValue Int;
3318
3319 if (Opc == ISD::FP_TO_SINT) {
3320 SDValue Bnd = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, M80, M7F});
3321 SDValue Pos = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETGT);
3322 SDValue Frc13 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ResTy, Ops: {Zero, Frc02});
3323 SDValue Frc14 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Neg, Frc13, Frc02});
3324 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, Ops: {Pos, Frc14, Bnd});
3325 } else {
3326 assert(Opc == ISD::FP_TO_UINT);
3327 SDValue Rsn = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Rsh01, RHS: Zero, Cond: ISD::SETLT);
3328 SDValue Frc23 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Rsn, N2: M7F, N3: Frc02);
3329 Int = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: ResTy, N1: Neg, N2: Zero, N3: Frc23);
3330 }
3331
3332 return Int;
3333}
3334
3335SDValue
3336HexagonTargetLowering::ExpandHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
3337 unsigned Opc = Op.getOpcode();
3338 assert(Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP);
3339
3340 const SDLoc &dl(Op);
3341 SDValue Op0 = Op.getOperand(i: 0);
3342 MVT InpTy = ty(Op: Op0);
3343 MVT ResTy = ty(Op);
3344 assert(ResTy.changeTypeToInteger() == InpTy);
3345
3346 // uint32_t vnoc1_rnd(int32_t w) {
3347 // int32_t iszero = w == 0;
3348 // int32_t isneg = w < 0;
3349 // uint32_t u = __builtin_HEXAGON_A2_abs(w);
3350 //
3351 // uint32_t norm_left = __builtin_HEXAGON_S2_cl0(u) + 1;
3352 // uint32_t frac0 = (uint64_t)u << norm_left;
3353 //
3354 // // Rounding:
3355 // uint32_t frac1 = frac0 + ((1 << 8) - 1);
3356 // uint32_t renorm = (frac0 > frac1);
3357 // uint32_t rup = (int)(frac0 << 22) < 0;
3358 //
3359 // uint32_t frac2 = frac0 >> 8;
3360 // uint32_t frac3 = frac1 >> 8;
3361 // uint32_t frac = (frac2 != frac3) ? frac3 >> 1 : (frac3 + rup) >> 1;
3362 //
3363 // int32_t exp = 32 - norm_left + renorm + 127;
3364 // exp <<= 23;
3365 //
3366 // uint32_t sign = 0x80000000 * isneg;
3367 // uint32_t f = sign | exp | frac;
3368 // return iszero ? 0 : f;
3369 // }
3370
3371 MVT PredTy = MVT::getVectorVT(VT: MVT::i1, EC: InpTy.getVectorElementCount());
3372 bool Signed = Opc == ISD::SINT_TO_FP;
3373
3374 auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(Ty: ResTy);
3375 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3376
3377 SDValue Zero = getZero(dl, Ty: InpTy, DAG);
3378 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: InpTy);
3379 SDValue IsZero = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ);
3380 SDValue Abs = Signed ? DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: InpTy, Operand: Op0) : Op0;
3381 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT: InpTy, Operand: Abs);
3382 SDValue NLeft = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Clz, One});
3383 SDValue Frac0 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy, Ops: {Abs, NLeft});
3384
3385 auto [Frac, Ovf] = emitHvxShiftRightRnd(Val: Frac0, Amt: ExpWidth + 1, Signed: false, DAG);
3386 if (Signed) {
3387 SDValue IsNeg = DAG.getSetCC(DL: dl, VT: PredTy, LHS: Op0, RHS: Zero, Cond: ISD::SETLT);
3388 SDValue M80 = DAG.getConstant(Val: 1ull << (ElemWidth - 1), DL: dl, VT: InpTy);
3389 SDValue Sign = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsNeg, M80, Zero});
3390 Frac = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Sign, Frac});
3391 }
3392
3393 SDValue Rnrm = DAG.getZExtOrTrunc(Op: Ovf, DL: dl, VT: InpTy);
3394 SDValue Exp0 = DAG.getConstant(Val: ElemWidth + ExpBias, DL: dl, VT: InpTy);
3395 SDValue Exp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: InpTy, Ops: {Rnrm, Exp0});
3396 SDValue Exp2 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: InpTy, Ops: {Exp1, NLeft});
3397 SDValue Exp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: InpTy,
3398 Ops: {Exp2, DAG.getConstant(Val: FracWidth, DL: dl, VT: InpTy)});
3399 SDValue Flt0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: InpTy, Ops: {Frac, Exp3});
3400 SDValue Flt1 = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: InpTy, Ops: {IsZero, Zero, Flt0});
3401 SDValue Flt = DAG.getBitcast(VT: ResTy, V: Flt1);
3402
3403 return Flt;
3404}
3405
3406SDValue
3407HexagonTargetLowering::CreateTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3408 unsigned Opc = Op.getOpcode();
3409 unsigned TLOpc;
3410 switch (Opc) {
3411 case ISD::ANY_EXTEND:
3412 case ISD::SIGN_EXTEND:
3413 case ISD::ZERO_EXTEND:
3414 TLOpc = HexagonISD::TL_EXTEND;
3415 break;
3416 case ISD::TRUNCATE:
3417 TLOpc = HexagonISD::TL_TRUNCATE;
3418 break;
3419#ifndef NDEBUG
3420 Op.dump(&DAG);
3421#endif
3422 llvm_unreachable("Unexpected operator");
3423 }
3424
3425 const SDLoc &dl(Op);
3426 return DAG.getNode(Opcode: TLOpc, DL: dl, VT: ty(Op), N1: Op.getOperand(i: 0),
3427 N2: DAG.getUNDEF(VT: MVT::i128), // illegal type
3428 N3: DAG.getConstant(Val: Opc, DL: dl, VT: MVT::i32));
3429}
3430
3431SDValue
3432HexagonTargetLowering::RemoveTLWrapper(SDValue Op, SelectionDAG &DAG) const {
3433 assert(Op.getOpcode() == HexagonISD::TL_EXTEND ||
3434 Op.getOpcode() == HexagonISD::TL_TRUNCATE);
3435 unsigned Opc = Op.getConstantOperandVal(i: 2);
3436 return DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: ty(Op), Operand: Op.getOperand(i: 0));
3437}
3438
3439HexagonTargetLowering::VectorPair
3440HexagonTargetLowering::SplitVectorOp(SDValue Op, SelectionDAG &DAG) const {
3441 assert(!Op.isMachineOpcode());
3442 SmallVector<SDValue, 2> OpsL, OpsH;
3443 const SDLoc &dl(Op);
3444
3445 auto SplitVTNode = [&DAG, this](const VTSDNode *N) {
3446 MVT Ty = typeSplit(VecTy: N->getVT().getSimpleVT()).first;
3447 SDValue TV = DAG.getValueType(Ty);
3448 return std::make_pair(x&: TV, y&: TV);
3449 };
3450
3451 for (SDValue A : Op.getNode()->ops()) {
3452 auto [Lo, Hi] =
3453 ty(Op: A).isVector() ? opSplit(Vec: A, dl, DAG) : std::make_pair(x&: A, y&: A);
3454 // Special case for type operand.
3455 switch (Op.getOpcode()) {
3456 case ISD::SIGN_EXTEND_INREG:
3457 case HexagonISD::SSAT:
3458 case HexagonISD::USAT:
3459 if (const auto *N = dyn_cast<const VTSDNode>(Val: A.getNode()))
3460 std::tie(args&: Lo, args&: Hi) = SplitVTNode(N);
3461 break;
3462 }
3463 OpsL.push_back(Elt: Lo);
3464 OpsH.push_back(Elt: Hi);
3465 }
3466
3467 MVT ResTy = ty(Op);
3468 MVT HalfTy = typeSplit(VecTy: ResTy).first;
3469 SDValue L = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsL);
3470 SDValue H = DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT: HalfTy, Ops: OpsH);
3471 return {L, H};
3472}
3473
3474SDValue
3475HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
3476 auto *MemN = cast<MemSDNode>(Val: Op.getNode());
3477 unsigned MemOpc = MemN->getOpcode();
3478 EVT MemTy = MemN->getMemoryVT();
3479
3480 if ((MemOpc == ISD::STORE || MemOpc == ISD::LOAD) &&
3481 (!MemTy.isSimple() || !isHvxPairTy(Ty: MemTy.getSimpleVT())))
3482 return Op;
3483
3484 EVT ValueType;
3485 if (MemOpc == ISD::STORE)
3486 ValueType = ty(Op: cast<StoreSDNode>(Val&: Op)->getValue());
3487 else if (MemOpc == ISD::MSTORE)
3488 ValueType = ty(Op: cast<MaskedStoreSDNode>(Val&: Op)->getValue());
3489 else // ISD::LOAD, ISD::MLOAD.
3490 ValueType = MemN->getValueType(ResNo: 0);
3491
3492 EVT LoVT, HiVT;
3493 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: ValueType);
3494
3495 EVT LoMemVT, HiMemVT;
3496 bool HiIsEmpty = false;
3497 std::tie(args&: LoMemVT, args&: HiMemVT) =
3498 DAG.GetDependentSplitDestVTs(VT: MemTy, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
3499
3500 uint64_t LoSize = LoMemVT.getSizeInBits().getFixedValue() / 8;
3501 uint64_t HiSize = HiMemVT.getSizeInBits().getFixedValue() / 8;
3502
3503 const SDLoc &dl(Op);
3504 SDValue Chain = MemN->getChain();
3505 SDValue Base0 = MemN->getBasePtr();
3506 SDValue Base1 =
3507 DAG.getMemBasePlusOffset(Base: Base0, Offset: TypeSize::getFixed(ExactSize: LoSize), DL: dl);
3508
3509 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
3510 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
3511 MachineFunction &MF = DAG.getMachineFunction();
3512 auto MemSize = [=](uint64_t Size) {
3513 return (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
3514 ? (uint64_t)MemoryLocation::UnknownSize
3515 : Size;
3516 };
3517 // MOp1 will not be used if HiIsEmpty for masked loads and stores (MLOAD and
3518 // MSTORE). Non-masked loads and store are always of double-vector size (see
3519 // isHvxPairTy() check above).
3520 MOp0 = MF.getMachineMemOperand(MMO, Offset: 0, Size: MemSize(LoSize));
3521 MOp1 = MF.getMachineMemOperand(MMO, Offset: LoSize, Size: MemSize(HiSize));
3522 }
3523
3524 if (MemOpc == ISD::LOAD) {
3525 assert(cast<LoadSDNode>(Op)->isUnindexed());
3526 SDValue Load0 = DAG.getLoad(VT: LoVT, dl, Chain, Ptr: Base0, MMO: MOp0);
3527 SDValue Load1 = DAG.getLoad(VT: HiVT, dl, Chain, Ptr: Base1, MMO: MOp1);
3528 return DAG.getMergeValues(
3529 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: Load0,
3530 N2: Load1),
3531 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Load0.getValue(R: 1),
3532 N2: Load1.getValue(R: 1))},
3533 dl);
3534 }
3535 if (MemOpc == ISD::STORE) {
3536 assert(cast<StoreSDNode>(Op)->isUnindexed());
3537 VectorPair Vals = opSplit(Vec: cast<StoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3538 SDValue Store0 = DAG.getStore(Chain, dl, Val: Vals.first, Ptr: Base0, MMO: MOp0);
3539 SDValue Store1 = DAG.getStore(Chain, dl, Val: Vals.second, Ptr: Base1, MMO: MOp1);
3540 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store0, N2: Store1);
3541 }
3542
3543 assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE);
3544
3545 auto MaskN = cast<MaskedLoadStoreSDNode>(Val&: Op);
3546 assert(MaskN->isUnindexed());
3547 VectorPair Masks = opSplit(Vec: MaskN->getMask(), dl, DAG);
3548 SDValue Offset = DAG.getUNDEF(VT: MVT::i32);
3549
3550 if (MemOpc == ISD::MLOAD) {
3551 VectorPair Thru =
3552 opSplit(Vec: cast<MaskedLoadSDNode>(Val&: Op)->getPassThru(), dl, DAG);
3553 SDValue MLoad0 = DAG.getMaskedLoad(VT: LoVT, dl, Chain, Base: Base0, Offset,
3554 Mask: Masks.first, Src0: Thru.first, MemVT: LoMemVT, MMO: MOp0,
3555 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3556
3557 // The hi masked load has zero storage size. We therefore simply set it to
3558 // the low masked load and rely on subsequent removal from the chain as it
3559 // is unused. See DAGTypeLegalizer::SplitVecRes_MLOAD() for the same logic.
3560 SDValue MLoad1 =
3561 HiIsEmpty ? MLoad0
3562 : DAG.getMaskedLoad(VT: HiVT, dl, Chain, Base: Base1, Offset,
3563 Mask: Masks.second, Src0: Thru.second, MemVT: HiMemVT, MMO: MOp1,
3564 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3565 return DAG.getMergeValues(
3566 Ops: {DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: MemN->getValueType(ResNo: 0), N1: MLoad0,
3567 N2: MLoad1),
3568 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MLoad0.getValue(R: 1),
3569 N2: MLoad1.getValue(R: 1))},
3570 dl);
3571 }
3572 if (MemOpc == ISD::MSTORE) {
3573 VectorPair Vals = opSplit(Vec: cast<MaskedStoreSDNode>(Val&: Op)->getValue(), dl, DAG);
3574 SDValue MStore0 =
3575 DAG.getMaskedStore(Chain, dl, Val: Vals.first, Base: Base0, Offset, Mask: Masks.first,
3576 MemVT: LoMemVT, MMO: MOp0, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3577 if (HiIsEmpty)
3578 return MStore0;
3579 SDValue MStore1 =
3580 DAG.getMaskedStore(Chain, dl, Val: Vals.second, Base: Base1, Offset, Mask: Masks.second,
3581 MemVT: HiMemVT, MMO: MOp1, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3582 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: MStore0, N2: MStore1);
3583 }
3584
3585 std::string Name = "Unexpected operation: " + Op->getOperationName(G: &DAG);
3586 llvm_unreachable(Name.c_str());
3587}
3588
3589SDValue
3590HexagonTargetLowering::WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const {
3591 const SDLoc &dl(Op);
3592 auto *LoadN = cast<LoadSDNode>(Val: Op.getNode());
3593 assert(LoadN->isUnindexed() && "Not widening indexed loads yet");
3594 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3595 "Not widening loads of i1 yet");
3596
3597 SDValue Chain = LoadN->getChain();
3598 SDValue Base = LoadN->getBasePtr();
3599 SDValue Offset = DAG.getUNDEF(VT: MVT::i32);
3600
3601 MVT ResTy = ty(Op);
3602 unsigned HwLen = Subtarget.getVectorLength();
3603 unsigned ResLen = ResTy.getStoreSize();
3604 assert(ResLen < HwLen && "vsetq(v1) prerequisite");
3605
3606 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3607 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3608 Ops: {DAG.getConstant(Val: ResLen, DL: dl, VT: MVT::i32)}, DAG);
3609
3610 MVT LoadTy = MVT::getVectorVT(VT: MVT::i8, NumElements: HwLen);
3611 MachineFunction &MF = DAG.getMachineFunction();
3612 auto *MemOp = MF.getMachineMemOperand(MMO: LoadN->getMemOperand(), Offset: 0, Size: HwLen);
3613
3614 SDValue Load = DAG.getMaskedLoad(VT: LoadTy, dl, Chain, Base, Offset, Mask,
3615 Src0: DAG.getUNDEF(VT: LoadTy), MemVT: LoadTy, MMO: MemOp,
3616 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding: false);
3617 SDValue Value = opCastElem(Vec: Load, ElemTy: ResTy.getVectorElementType(), DAG);
3618 return DAG.getMergeValues(Ops: {Value, Load.getValue(R: 1)}, dl);
3619}
3620
3621SDValue
3622HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const {
3623 const SDLoc &dl(Op);
3624 auto *StoreN = cast<StoreSDNode>(Val: Op.getNode());
3625 assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
3626 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3627 "Not widening stores of i1 yet");
3628
3629 SDValue Chain = StoreN->getChain();
3630 SDValue Base = StoreN->getBasePtr();
3631 SDValue Offset = DAG.getUNDEF(VT: MVT::i32);
3632
3633 SDValue Value = opCastElem(Vec: StoreN->getValue(), ElemTy: MVT::i8, DAG);
3634 MVT ValueTy = ty(Op: Value);
3635 unsigned ValueLen = ValueTy.getVectorNumElements();
3636 unsigned HwLen = Subtarget.getVectorLength();
3637 assert(isPowerOf2_32(ValueLen));
3638
3639 for (unsigned Len = ValueLen; Len < HwLen; ) {
3640 Value = opJoin(Ops: {Value, DAG.getUNDEF(VT: ty(Op: Value))}, dl, DAG);
3641 Len = ty(Op: Value).getVectorNumElements(); // This is Len *= 2
3642 }
3643 assert(ty(Value).getVectorNumElements() == HwLen); // Paranoia
3644
3645 assert(ValueLen < HwLen && "vsetq(v1) prerequisite");
3646 MVT BoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: HwLen);
3647 SDValue Mask = getInstr(MachineOpc: Hexagon::V6_pred_scalar2, dl, Ty: BoolTy,
3648 Ops: {DAG.getConstant(Val: ValueLen, DL: dl, VT: MVT::i32)}, DAG);
3649 MachineFunction &MF = DAG.getMachineFunction();
3650 auto *MemOp = MF.getMachineMemOperand(MMO: StoreN->getMemOperand(), Offset: 0, Size: HwLen);
3651 return DAG.getMaskedStore(Chain, dl, Val: Value, Base, Offset, Mask, MemVT: ty(Op: Value),
3652 MMO: MemOp, AM: ISD::UNINDEXED, IsTruncating: false, IsCompressing: false);
3653}
3654
3655SDValue
3656HexagonTargetLowering::WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const {
3657 const SDLoc &dl(Op);
3658 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
3659 MVT ElemTy = ty(Op: Op0).getVectorElementType();
3660 unsigned HwLen = Subtarget.getVectorLength();
3661
3662 unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits();
3663 assert(WideOpLen * ElemTy.getSizeInBits() == 8 * HwLen);
3664 MVT WideOpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideOpLen);
3665 if (!Subtarget.isHVXVectorType(VecTy: WideOpTy, IncludeBool: true))
3666 return SDValue();
3667
3668 SDValue WideOp0 = appendUndef(Val: Op0, ResTy: WideOpTy, DAG);
3669 SDValue WideOp1 = appendUndef(Val: Op1, ResTy: WideOpTy, DAG);
3670 EVT ResTy =
3671 getSetCCResultType(DAG.getDataLayout(), C&: *DAG.getContext(), VT: WideOpTy);
3672 SDValue SetCC = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: ResTy,
3673 Ops: {WideOp0, WideOp1, Op.getOperand(i: 2)});
3674
3675 EVT RetTy = typeLegalize(Ty: ty(Op), DAG);
3676 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3677 Ops: {SetCC, getZero(dl, Ty: MVT::i32, DAG)});
3678}
3679
3680SDValue HexagonTargetLowering::WidenHvxTruncateToBool(SDValue Op,
3681 SelectionDAG &DAG) const {
3682 // Handle truncation to boolean vector where the result boolean type
3683 // needs widening (e.g., v16i32 -> v16i1 where v16i1 is not a standard
3684 // HVX predicate type, or v16i8 -> v16i1 in 128-byte mode).
3685 // Widen the input to HVX width, perform the truncate to the widened
3686 // boolean type, then extract the result.
3687 const SDLoc &dl(Op);
3688 SDValue Inp = Op.getOperand(i: 0);
3689 MVT InpTy = ty(Op: Inp);
3690 MVT ResTy = ty(Op);
3691
3692 assert(ResTy.getVectorElementType() == MVT::i1 &&
3693 "Expected boolean result type");
3694
3695 MVT ElemTy = InpTy.getVectorElementType();
3696 unsigned HwLen = Subtarget.getVectorLength();
3697
3698 // Calculate the widened input type that fills the HVX register.
3699 unsigned WideLen = (8 * HwLen) / ElemTy.getSizeInBits();
3700 MVT WideInpTy = MVT::getVectorVT(VT: ElemTy, NumElements: WideLen);
3701 if (!Subtarget.isHVXVectorType(VecTy: WideInpTy, IncludeBool: false))
3702 return SDValue();
3703
3704 // Widen the input to HVX width.
3705 SDValue WideInp = appendUndef(Val: Inp, ResTy: WideInpTy, DAG);
3706
3707 // Perform the truncate to widened boolean type.
3708 MVT WideBoolTy = MVT::getVectorVT(VT: MVT::i1, NumElements: WideLen);
3709 SDValue WideTrunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: WideBoolTy, Operand: WideInp);
3710
3711 // Extract the result.
3712 EVT RetTy = typeLegalize(Ty: ResTy, DAG);
3713 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: RetTy,
3714 Ops: {WideTrunc, getZero(dl, Ty: MVT::i32, DAG)});
3715}
3716
3717SDValue
3718HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
3719 unsigned Opc = Op.getOpcode();
3720 bool IsPairOp = isHvxPairTy(Ty: ty(Op)) ||
3721 llvm::any_of(Range: Op.getNode()->ops(), P: [this] (SDValue V) {
3722 return isHvxPairTy(Ty: ty(Op: V));
3723 });
3724
3725 if (IsPairOp) {
3726 switch (Opc) {
3727 default:
3728 break;
3729 case ISD::LOAD:
3730 case ISD::STORE:
3731 case ISD::MLOAD:
3732 case ISD::MSTORE:
3733 return SplitHvxMemOp(Op, DAG);
3734 case ISD::SINT_TO_FP:
3735 case ISD::UINT_TO_FP:
3736 case ISD::FP_TO_SINT:
3737 case ISD::FP_TO_UINT:
3738 if (ty(Op).getSizeInBits() == ty(Op: Op.getOperand(i: 0)).getSizeInBits())
3739 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3740 break;
3741 case ISD::ABS:
3742 case ISD::CTPOP:
3743 case ISD::CTLZ:
3744 case ISD::CTTZ:
3745 case ISD::MUL:
3746 case ISD::FADD:
3747 case ISD::FSUB:
3748 case ISD::FMUL:
3749 case ISD::FMINIMUMNUM:
3750 case ISD::FMAXIMUMNUM:
3751 case ISD::FMINNUM:
3752 case ISD::FMAXNUM:
3753 case ISD::MULHS:
3754 case ISD::MULHU:
3755 case ISD::AND:
3756 case ISD::OR:
3757 case ISD::XOR:
3758 case ISD::SRA:
3759 case ISD::SHL:
3760 case ISD::SRL:
3761 case ISD::FSHL:
3762 case ISD::FSHR:
3763 case ISD::SMIN:
3764 case ISD::SMAX:
3765 case ISD::UMIN:
3766 case ISD::UMAX:
3767 case ISD::SETCC:
3768 case ISD::VSELECT:
3769 case ISD::SIGN_EXTEND_INREG:
3770 case ISD::SPLAT_VECTOR:
3771 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3772 case ISD::SIGN_EXTEND:
3773 case ISD::ZERO_EXTEND:
3774 // In general, sign- and zero-extends can't be split and still
3775 // be legal. The only exception is extending bool vectors.
3776 if (ty(Op: Op.getOperand(i: 0)).getVectorElementType() == MVT::i1)
3777 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3778 break;
3779 }
3780 }
3781
3782 switch (Opc) {
3783 default:
3784 break;
3785 // clang-format off
3786 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
3787 case ISD::SPLAT_VECTOR: return LowerHvxSplatVector(Op, DAG);
3788 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
3789 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
3790 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
3791 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
3792 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
3793 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
3794 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
3795 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
3796 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
3797 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
3798 case ISD::SELECT: return LowerHvxSelect(Op, DAG);
3799 case ISD::SRA:
3800 case ISD::SHL:
3801 case ISD::SRL: return LowerHvxShift(Op, DAG);
3802 case ISD::FSHL:
3803 case ISD::FSHR: return LowerHvxFunnelShift(Op, DAG);
3804 case ISD::MULHS:
3805 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
3806 case ISD::SMUL_LOHI:
3807 case ISD::UMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3808 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
3809 case ISD::SETCC:
3810 case ISD::INTRINSIC_VOID: return Op;
3811 case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG);
3812 case ISD::MLOAD:
3813 case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG);
3814 // Unaligned loads will be handled by the default lowering.
3815 case ISD::LOAD: return LowerHvxLoad(Op, DAG);
3816 case ISD::STORE: return LowerHvxStore(Op, DAG);
3817 case ISD::FP_EXTEND: return LowerHvxFpExtend(Op, DAG);
3818 case ISD::FP_TO_SINT:
3819 case ISD::FP_TO_UINT: return LowerHvxFpToInt(Op, DAG);
3820 case ISD::SINT_TO_FP:
3821 case ISD::UINT_TO_FP: return LowerHvxIntToFp(Op, DAG);
3822
3823 // Special nodes:
3824 case HexagonISD::SMUL_LOHI:
3825 case HexagonISD::UMUL_LOHI:
3826 case HexagonISD::USMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
3827
3828 case ISD::PARTIAL_REDUCE_SMLA:
3829 case ISD::PARTIAL_REDUCE_UMLA:
3830 case ISD::PARTIAL_REDUCE_SUMLA:
3831 return LowerHvxPartialReduceMLA(Op, DAG);
3832 // clang-format on
3833 }
3834#ifndef NDEBUG
3835 Op.dumpr(&DAG);
3836#endif
3837 llvm_unreachable("Unhandled HVX operation");
3838}
3839
3840SDValue
3841HexagonTargetLowering::ExpandHvxResizeIntoSteps(SDValue Op, SelectionDAG &DAG)
3842 const {
3843 // Rewrite the extension/truncation/saturation op into steps where each
3844 // step changes the type widths by a factor of 2.
3845 // E.g. i8 -> i16 remains unchanged, but i8 -> i32 ==> i8 -> i16 -> i32.
3846 //
3847 // Some of the vector types in Op may not be legal.
3848
3849 unsigned Opc = Op.getOpcode();
3850 switch (Opc) {
3851 case HexagonISD::SSAT:
3852 case HexagonISD::USAT:
3853 case HexagonISD::TL_EXTEND:
3854 case HexagonISD::TL_TRUNCATE:
3855 break;
3856 case ISD::ANY_EXTEND:
3857 case ISD::ZERO_EXTEND:
3858 case ISD::SIGN_EXTEND:
3859 case ISD::TRUNCATE:
3860 llvm_unreachable("ISD:: ops will be auto-folded");
3861 break;
3862#ifndef NDEBUG
3863 Op.dump(&DAG);
3864#endif
3865 llvm_unreachable("Unexpected operation");
3866 }
3867
3868 SDValue Inp = Op.getOperand(i: 0);
3869 MVT InpTy = ty(Op: Inp);
3870 MVT ResTy = ty(Op);
3871
3872 unsigned InpWidth = InpTy.getVectorElementType().getSizeInBits();
3873 unsigned ResWidth = ResTy.getVectorElementType().getSizeInBits();
3874 assert(InpWidth != ResWidth);
3875
3876 if (InpWidth == 2 * ResWidth || ResWidth == 2 * InpWidth)
3877 return Op;
3878
3879 const SDLoc &dl(Op);
3880 unsigned NumElems = InpTy.getVectorNumElements();
3881 assert(NumElems == ResTy.getVectorNumElements());
3882
3883 auto repeatOp = [&](unsigned NewWidth, SDValue Arg) {
3884 MVT Ty = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: NewWidth), NumElements: NumElems);
3885 switch (Opc) {
3886 case HexagonISD::SSAT:
3887 case HexagonISD::USAT:
3888 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, DAG.getValueType(Ty)});
3889 case HexagonISD::TL_EXTEND:
3890 case HexagonISD::TL_TRUNCATE:
3891 return DAG.getNode(Opcode: Opc, DL: dl, VT: Ty, Ops: {Arg, Op.getOperand(i: 1), Op.getOperand(i: 2)});
3892 default:
3893 llvm_unreachable("Unexpected opcode");
3894 }
3895 };
3896
3897 SDValue S = Inp;
3898 if (InpWidth < ResWidth) {
3899 assert(ResWidth % InpWidth == 0 && isPowerOf2_32(ResWidth / InpWidth));
3900 while (InpWidth * 2 <= ResWidth)
3901 S = repeatOp(InpWidth *= 2, S);
3902 } else {
3903 // InpWidth > ResWidth
3904 assert(InpWidth % ResWidth == 0 && isPowerOf2_32(InpWidth / ResWidth));
3905 while (InpWidth / 2 >= ResWidth)
3906 S = repeatOp(InpWidth /= 2, S);
3907 }
3908 return S;
3909}
3910
3911SDValue
3912HexagonTargetLowering::LegalizeHvxResize(SDValue Op, SelectionDAG &DAG) const {
3913 SDValue Inp0 = Op.getOperand(i: 0);
3914 MVT InpTy = ty(Op: Inp0);
3915 MVT ResTy = ty(Op);
3916 unsigned InpWidth = InpTy.getSizeInBits();
3917 unsigned ResWidth = ResTy.getSizeInBits();
3918 unsigned Opc = Op.getOpcode();
3919
3920 if (shouldWidenToHvx(Ty: InpTy, DAG) || shouldWidenToHvx(Ty: ResTy, DAG)) {
3921 // First, make sure that the narrower type is widened to HVX.
3922 // This may cause the result to be wider than what the legalizer
3923 // expects, so insert EXTRACT_SUBVECTOR to bring it back to the
3924 // desired type.
3925 auto [WInpTy, WResTy] =
3926 InpWidth < ResWidth ? typeWidenToWider(Ty0: typeWidenToHvx(Ty: InpTy), Ty1: ResTy)
3927 : typeWidenToWider(Ty0: InpTy, Ty1: typeWidenToHvx(Ty: ResTy));
3928 SDValue W = appendUndef(Val: Inp0, ResTy: WInpTy, DAG);
3929 SDValue S;
3930 if (Opc == HexagonISD::TL_EXTEND || Opc == HexagonISD::TL_TRUNCATE) {
3931 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: Op.getOperand(i: 1),
3932 N3: Op.getOperand(i: 2));
3933 } else {
3934 S = DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: WResTy, N1: W, N2: DAG.getValueType(WResTy));
3935 }
3936 SDValue T = ExpandHvxResizeIntoSteps(Op: S, DAG);
3937 return extractSubvector(Vec: T, SubTy: typeLegalize(Ty: ResTy, DAG), SubIdx: 0, DAG);
3938 } else if (shouldSplitToHvx(Ty: InpWidth < ResWidth ? ResTy : InpTy, DAG)) {
3939 // For multi-step extends/truncates (e.g., i8->i32), expand into
3940 // single-step operations first. Splitting a multi-step TL_EXTEND
3941 // would halve the operand type to a sub-HVX size (e.g., v128i8 ->
3942 // v64i8), creating illegal types that cause issues in the type
3943 // legalizer's map tracking. Single-step operations (e.g., i16->i32)
3944 // are safe to split because their halved operand types remain legal.
3945 SDValue T = ExpandHvxResizeIntoSteps(Op, DAG);
3946 if (T != Op)
3947 return T;
3948 return opJoin(Ops: SplitVectorOp(Op, DAG), dl: SDLoc(Op), DAG);
3949 } else {
3950 assert(isTypeLegal(InpTy) && isTypeLegal(ResTy));
3951 return RemoveTLWrapper(Op, DAG);
3952 }
3953 llvm_unreachable("Unexpected situation");
3954}
3955
3956void
3957HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
3958 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
3959 unsigned Opc = N->getOpcode();
3960 SDValue Op(N, 0);
3961 SDValue Inp0; // Optional first argument.
3962 if (N->getNumOperands() > 0)
3963 Inp0 = Op.getOperand(i: 0);
3964
3965 switch (Opc) {
3966 case ISD::ANY_EXTEND:
3967 case ISD::SIGN_EXTEND:
3968 case ISD::ZERO_EXTEND:
3969 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
3970 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
3971 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
3972 }
3973 break;
3974 case ISD::TRUNCATE:
3975 // Handle truncate to boolean vector when the input is not a
3976 // standard HVX vector type (single or pair). This covers cases
3977 // where the input needs widening (e.g., v64i8 -> v64i1 in
3978 // 128-byte mode) and cases where the result boolean type itself
3979 // needs widening (e.g., v16i32 -> v16i1). When the input is
3980 // already an HVX type, tablegen patterns handle the truncation
3981 // directly (e.g., v64i16 -> v64i1 via V6_vandvrt).
3982 if (ty(Op).getVectorElementType() == MVT::i1 &&
3983 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
3984 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
3985 Results.push_back(Elt: T);
3986 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
3987 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
3988 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
3989 }
3990 break;
3991 case ISD::SETCC:
3992 if (shouldWidenToHvx(Ty: ty(Op: Inp0), DAG)) {
3993 if (SDValue T = WidenHvxSetCC(Op, DAG))
3994 Results.push_back(Elt: T);
3995 }
3996 break;
3997 case ISD::STORE: {
3998 if (shouldWidenToHvx(Ty: ty(Op: cast<StoreSDNode>(Val: N)->getValue()), DAG)) {
3999 SDValue Store = WidenHvxStore(Op, DAG);
4000 Results.push_back(Elt: Store);
4001 }
4002 break;
4003 }
4004 case ISD::MLOAD:
4005 if (isHvxPairTy(Ty: ty(Op))) {
4006 SDValue S = SplitHvxMemOp(Op, DAG);
4007 assert(S->getOpcode() == ISD::MERGE_VALUES);
4008 Results.push_back(Elt: S.getOperand(i: 0));
4009 Results.push_back(Elt: S.getOperand(i: 1));
4010 }
4011 break;
4012 case ISD::MSTORE:
4013 if (isHvxPairTy(Ty: ty(Op: Op->getOperand(Num: 1)))) { // Stored value
4014 SDValue S = SplitHvxMemOp(Op, DAG);
4015 Results.push_back(Elt: S);
4016 }
4017 break;
4018 case ISD::SINT_TO_FP:
4019 case ISD::UINT_TO_FP:
4020 case ISD::FP_TO_SINT:
4021 case ISD::FP_TO_UINT:
4022 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4023 SDValue T = EqualizeFpIntConversion(Op, DAG);
4024 Results.push_back(Elt: T);
4025 }
4026 break;
4027 case HexagonISD::SSAT:
4028 case HexagonISD::USAT:
4029 case HexagonISD::TL_EXTEND:
4030 case HexagonISD::TL_TRUNCATE:
4031 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4032 break;
4033 default:
4034 break;
4035 }
4036}
4037
4038void
4039HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
4040 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
4041 unsigned Opc = N->getOpcode();
4042 SDValue Op(N, 0);
4043 SDValue Inp0; // Optional first argument.
4044 if (N->getNumOperands() > 0)
4045 Inp0 = Op.getOperand(i: 0);
4046
4047 switch (Opc) {
4048 case ISD::ANY_EXTEND:
4049 case ISD::SIGN_EXTEND:
4050 case ISD::ZERO_EXTEND:
4051 if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4052 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4053 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4054 }
4055 break;
4056 case ISD::TRUNCATE:
4057 // Handle truncate to boolean vector when the input is not a
4058 // standard HVX vector type. See comment in LowerHvxOperationWrapper.
4059 if (ty(Op).getVectorElementType() == MVT::i1 &&
4060 !Subtarget.isHVXVectorType(VecTy: ty(Op: Inp0), IncludeBool: false)) {
4061 if (SDValue T = WidenHvxTruncateToBool(Op, DAG))
4062 Results.push_back(Elt: T);
4063 } else if (Subtarget.isHVXElementType(Ty: ty(Op)) &&
4064 Subtarget.isHVXElementType(Ty: ty(Op: Inp0))) {
4065 Results.push_back(Elt: CreateTLWrapper(Op, DAG));
4066 }
4067 break;
4068 case ISD::SETCC:
4069 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4070 if (SDValue T = WidenHvxSetCC(Op, DAG))
4071 Results.push_back(Elt: T);
4072 }
4073 break;
4074 case ISD::LOAD: {
4075 if (shouldWidenToHvx(Ty: ty(Op), DAG)) {
4076 SDValue Load = WidenHvxLoad(Op, DAG);
4077 assert(Load->getOpcode() == ISD::MERGE_VALUES);
4078 Results.push_back(Elt: Load.getOperand(i: 0));
4079 Results.push_back(Elt: Load.getOperand(i: 1));
4080 }
4081 break;
4082 }
4083 case ISD::BITCAST:
4084 if (isHvxBoolTy(Ty: ty(Op: Inp0))) {
4085 SDValue C = LowerHvxBitcast(Op, DAG);
4086 Results.push_back(Elt: C);
4087 }
4088 break;
4089 case ISD::FP_TO_SINT:
4090 case ISD::FP_TO_UINT:
4091 if (ty(Op).getSizeInBits() != ty(Op: Inp0).getSizeInBits()) {
4092 SDValue T = EqualizeFpIntConversion(Op, DAG);
4093 Results.push_back(Elt: T);
4094 }
4095 break;
4096 case HexagonISD::SSAT:
4097 case HexagonISD::USAT:
4098 case HexagonISD::TL_EXTEND:
4099 case HexagonISD::TL_TRUNCATE:
4100 Results.push_back(Elt: LegalizeHvxResize(Op, DAG));
4101 break;
4102 default:
4103 break;
4104 }
4105}
4106
4107SDValue
4108HexagonTargetLowering::combineTruncateBeforeLegal(SDValue Op,
4109 DAGCombinerInfo &DCI) const {
4110 // Simplify V:v2NiB --(bitcast)--> vNi2B --(truncate)--> vNiB
4111 // to extract-subvector (shuffle V, pick even, pick odd)
4112
4113 assert(Op.getOpcode() == ISD::TRUNCATE);
4114 SelectionDAG &DAG = DCI.DAG;
4115 const SDLoc &dl(Op);
4116
4117 if (Op.getOperand(i: 0).getOpcode() == ISD::BITCAST)
4118 return SDValue();
4119 SDValue Cast = Op.getOperand(i: 0);
4120 SDValue Src = Cast.getOperand(i: 0);
4121
4122 EVT TruncTy = Op.getValueType();
4123 EVT CastTy = Cast.getValueType();
4124 EVT SrcTy = Src.getValueType();
4125 if (SrcTy.isSimple())
4126 return SDValue();
4127 if (SrcTy.getVectorElementType() != TruncTy.getVectorElementType())
4128 return SDValue();
4129 unsigned SrcLen = SrcTy.getVectorNumElements();
4130 unsigned CastLen = CastTy.getVectorNumElements();
4131 if (2 * CastLen != SrcLen)
4132 return SDValue();
4133
4134 SmallVector<int, 128> Mask(SrcLen);
4135 for (int i = 0; i != static_cast<int>(CastLen); ++i) {
4136 Mask[i] = 2 * i;
4137 Mask[i + CastLen] = 2 * i + 1;
4138 }
4139 SDValue Deal =
4140 DAG.getVectorShuffle(VT: SrcTy, dl, N1: Src, N2: DAG.getUNDEF(VT: SrcTy), Mask);
4141 return opSplit(Vec: Deal, dl, DAG).first;
4142}
4143
4144SDValue
4145HexagonTargetLowering::combineConcatOfShuffles(SDValue Op,
4146 SelectionDAG &DAG) const {
4147 // Fold
4148 // concat (shuffle x, y, m1), (shuffle x, y, m2)
4149 // into
4150 // shuffle (concat x, y), undef, m3
4151 if (Op.getNumOperands() != 2)
4152 return SDValue();
4153
4154 const SDLoc &dl(Op);
4155 SDValue V0 = Op.getOperand(i: 0);
4156 SDValue V1 = Op.getOperand(i: 1);
4157
4158 if (V0.getOpcode() != ISD::VECTOR_SHUFFLE)
4159 return SDValue();
4160 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE)
4161 return SDValue();
4162
4163 SetVector<SDValue> Order;
4164 Order.insert(X: V0.getOperand(i: 0));
4165 Order.insert(X: V0.getOperand(i: 1));
4166 Order.insert(X: V1.getOperand(i: 0));
4167 Order.insert(X: V1.getOperand(i: 1));
4168
4169 if (Order.size() > 2)
4170 return SDValue();
4171
4172 // In ISD::VECTOR_SHUFFLE, the types of each input and the type of the
4173 // result must be the same.
4174 EVT InpTy = V0.getValueType();
4175 assert(InpTy.isVector());
4176 unsigned InpLen = InpTy.getVectorNumElements();
4177
4178 SmallVector<int, 128> LongMask;
4179 auto AppendToMask = [&](SDValue Shuffle) {
4180 auto *SV = cast<ShuffleVectorSDNode>(Val: Shuffle.getNode());
4181 ArrayRef<int> Mask = SV->getMask();
4182 SDValue X = Shuffle.getOperand(i: 0);
4183 SDValue Y = Shuffle.getOperand(i: 1);
4184 for (int M : Mask) {
4185 if (M == -1) {
4186 LongMask.push_back(Elt: M);
4187 continue;
4188 }
4189 SDValue Src = static_cast<unsigned>(M) < InpLen ? X : Y;
4190 if (static_cast<unsigned>(M) >= InpLen)
4191 M -= InpLen;
4192
4193 int OutOffset = Order[0] == Src ? 0 : InpLen;
4194 LongMask.push_back(Elt: M + OutOffset);
4195 }
4196 };
4197
4198 AppendToMask(V0);
4199 AppendToMask(V1);
4200
4201 SDValue C0 = Order.front();
4202 SDValue C1 = Order.back(); // Can be same as front
4203 EVT LongTy = InpTy.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4204
4205 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LongTy, Ops: {C0, C1});
4206 return DAG.getVectorShuffle(VT: LongTy, dl, N1: Cat, N2: DAG.getUNDEF(VT: LongTy), Mask: LongMask);
4207}
4208
4209// Reassociate concat(p1, p2, ...) into
4210// concat(concat(p1, ...), concat(pi, ...), ...)
4211// where each inner concat produces a predicate where each bit corresponds
4212// to at most BitBytes bytes.
4213// Concatenating predicates decreases the number of bytes per each predicate
4214// bit.
4215SDValue
4216HexagonTargetLowering::combineConcatOfScalarPreds(SDValue Op, unsigned BitBytes,
4217 SelectionDAG &DAG) const {
4218 const SDLoc &dl(Op);
4219 SmallVector<SDValue> Ops(Op->ops());
4220 MVT ResTy = ty(Op);
4221 MVT InpTy = ty(Op: Ops[0]);
4222 unsigned InpLen = InpTy.getVectorNumElements(); // Scalar predicate
4223 unsigned ResLen = ResTy.getVectorNumElements(); // HVX vector predicate
4224 assert(InpLen <= 8 && "Too long for scalar predicate");
4225 assert(ResLen > 8 && "Too short for HVX vector predicate");
4226
4227 unsigned Bytes = 8 / InpLen; // Bytes-per-bit in input
4228
4229 // Already in the right form?
4230 if (Bytes <= BitBytes)
4231 return Op;
4232
4233 ArrayRef<SDValue> Inputs(Ops);
4234 unsigned SliceLen = Bytes / BitBytes;
4235
4236 SmallVector<SDValue> Cats;
4237 // (8 / BitBytes) is the desired length of the result of the inner concat.
4238 MVT InnerTy = MVT::getVectorVT(VT: MVT::i1, NumElements: 8 / BitBytes);
4239 for (unsigned i = 0; i != ResLen / (8 / BitBytes); ++i) {
4240 SDValue Cat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: InnerTy,
4241 Ops: Inputs.slice(N: SliceLen * i, M: SliceLen));
4242 Cats.push_back(Elt: Cat);
4243 }
4244
4245 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResTy, Ops: Cats);
4246}
4247
4248SDValue HexagonTargetLowering::combineConcatVectorsBeforeLegal(
4249 SDValue Op, DAGCombinerInfo &DCI) const {
4250 MVT ResTy = ty(Op);
4251 MVT ElemTy = ResTy.getVectorElementType();
4252
4253 if (ElemTy != MVT::i1) {
4254 return combineConcatOfShuffles(Op, DAG&: DCI.DAG);
4255 }
4256 return SDValue();
4257}
4258
4259// Create the inner partial reduction MLA that can be efficiently lowered. This
4260// function is used by partial and full reductions.
4261SDValue HexagonTargetLowering::createExtendingPartialReduceMLA(
4262 unsigned Opcode, EVT AccEltType, unsigned AccNumElements, EVT InputType,
4263 const SDValue &A, const SDValue &B, unsigned &RemainingReductionRatio,
4264 const SDLoc &DL, SelectionDAG &DAG) const {
4265 const auto &Subtarget = DAG.getSubtarget<HexagonSubtarget>();
4266 if (!Subtarget.useHVXOps())
4267 return SDValue();
4268
4269 EVT InputEltType = InputType.getVectorElementType();
4270
4271 // Find if an optimized instruction for the sub-reduction is available.
4272 unsigned NativeRatio;
4273 if (AccEltType == MVT::i32 && InputEltType == MVT::i8)
4274 NativeRatio = 4;
4275 else
4276 return SDValue();
4277
4278 // We only handle the case when additional reduction will be needed, i.e.
4279 // input is longer by a larger factor than the result.
4280 ElementCount InputEC = InputType.getVectorElementCount();
4281 if (!InputEC.isKnownMultipleOf(RHS: AccNumElements * NativeRatio))
4282 return SDValue();
4283
4284 unsigned InputNumElements = InputEC.getFixedValue();
4285 RemainingReductionRatio = InputNumElements / (AccNumElements * NativeRatio);
4286 if (RemainingReductionRatio == 1)
4287 return SDValue();
4288
4289 // Create a reduction by the natively supported factor.
4290 EVT IntermediateType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType,
4291 NumElements: InputNumElements / NativeRatio);
4292
4293 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: IntermediateType);
4294 return DAG.getNode(Opcode, DL, VT: IntermediateType, N1: Zero, N2: A, N3: B);
4295}
4296
4297static bool DetectExtendingMultiply(const SDValue &N, EVT ScalarType,
4298 unsigned &Opcode, SDValue &A, SDValue &B) {
4299 SDValue Mul = N;
4300 EVT AccType = Mul.getValueType(); // Vector input type after extension.
4301 if (ScalarType != AccType.getVectorElementType())
4302 return false;
4303 bool swap = false;
4304 if (Mul->getOpcode() != ISD::MUL)
4305 return false;
4306 A = Mul->getOperand(Num: 0);
4307 B = Mul->getOperand(Num: 1);
4308 if (A.getOpcode() == ISD::ZERO_EXTEND) {
4309 if (B.getOpcode() == ISD::ZERO_EXTEND)
4310 Opcode = ISD::PARTIAL_REDUCE_UMLA;
4311 else if (B.getOpcode() == ISD::SIGN_EXTEND) {
4312 swap = true;
4313 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4314 } else
4315 return false;
4316 } else if (A.getOpcode() == ISD::SIGN_EXTEND) {
4317 if (B.getOpcode() == ISD::ZERO_EXTEND)
4318 Opcode = ISD::PARTIAL_REDUCE_SUMLA;
4319 else if (B.getOpcode() == ISD::SIGN_EXTEND)
4320 Opcode = ISD::PARTIAL_REDUCE_SMLA;
4321 else
4322 return false;
4323 } else
4324 return false;
4325
4326 // Get multiplication arguments before extension.
4327 A = A->getOperand(Num: 0);
4328 B = B->getOperand(Num: 0);
4329 if (A.getValueType() != B.getValueType())
4330 return false;
4331
4332 if (swap)
4333 std::swap(a&: A, b&: B);
4334
4335 return true;
4336}
4337
4338SDValue HexagonTargetLowering::splitVecReduceAdd(SDNode *N,
4339 SelectionDAG &DAG) const {
4340 if (!Subtarget.useHVXOps())
4341 return SDValue();
4342
4343 EVT ScalarType = N->getValueType(ResNo: 0);
4344 unsigned Opcode;
4345 SDValue A, B;
4346 if (!DetectExtendingMultiply(N: N->getOperand(Num: 0), ScalarType, Opcode, A, B))
4347 return SDValue();
4348
4349 SDLoc DL(N);
4350 unsigned RemainingReductionRatio;
4351 SDValue Partial =
4352 createExtendingPartialReduceMLA(Opcode, AccEltType: ScalarType, AccNumElements: 1, InputType: A.getValueType(),
4353 A, B, RemainingReductionRatio, DL, DAG);
4354 if (!Partial)
4355 return SDValue();
4356
4357 // We could have inserted a trivial MLA and rely on the folding action,
4358 // similar to how vector_partial_reduce_add is lowered to an MLA in
4359 // SelectionDAGBuilder. However, we just replace the final result since we
4360 // have analyzed the input completely.
4361 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: ScalarType, Operand: Partial);
4362}
4363
4364// When possible, separate an MLA reduction with extended operands but
4365// unsupported reduction factor into an extending partial reduction that
4366// can be efficiently lowered, and a follow-up partial reduction.
4367// partial_reduce_mla(a, x, y) ->
4368// partial_reduce_mla(a, partial_reduce_mla(0, x, y), 1)
4369SDValue
4370HexagonTargetLowering::splitExtendingPartialReduceMLA(SDNode *N,
4371 SelectionDAG &DAG) const {
4372 if (!Subtarget.useHVXOps())
4373 return SDValue();
4374
4375 SDValue Acc = N->getOperand(Num: 0);
4376 SDValue A = N->getOperand(Num: 1);
4377 SDValue B = N->getOperand(Num: 2);
4378 if (A.getValueType() != B.getValueType())
4379 return SDValue();
4380
4381 // The types should be declared as custom, but do not split already legal
4382 // operation.
4383 EVT AccType = Acc.getValueType();
4384 EVT InputType = A.getValueType();
4385 if (getPartialReduceMLAAction(Opc: N->getOpcode(), AccVT: AccType, InputVT: InputType) != Custom)
4386 return SDValue();
4387
4388 SDLoc DL(N);
4389 unsigned RemainingReductionRatio;
4390 SDValue Partial = createExtendingPartialReduceMLA(
4391 Opcode: N->getOpcode(), AccEltType: AccType.getVectorElementType(),
4392 AccNumElements: AccType.getVectorNumElements(), InputType, A, B, RemainingReductionRatio,
4393 DL, DAG);
4394 if (!Partial)
4395 return SDValue();
4396 assert(RemainingReductionRatio <= MaxExpandMLA);
4397
4398 // Create the reduction for the remaining ratio.
4399 EVT IntermediateType = Partial->getOperand(Num: 0).getValueType();
4400 SDValue One = DAG.getConstant(Val: 1, DL, VT: IntermediateType);
4401 return DAG.getNode(Opcode: N->getOpcode() == ISD::PARTIAL_REDUCE_UMLA
4402 ? ISD::PARTIAL_REDUCE_UMLA
4403 : ISD::PARTIAL_REDUCE_SUMLA,
4404 DL, VT: AccType, N1: Acc, N2: Partial, N3: One);
4405}
4406
4407SDValue
4408HexagonTargetLowering::LowerHvxPartialReduceMLA(SDValue Op,
4409 SelectionDAG &DAG) const {
4410 const SDLoc &DL(Op);
4411 SDValue Acc = Op.getOperand(i: 0);
4412 SDValue A = Op.getOperand(i: 1);
4413 SDValue B = Op.getOperand(i: 2);
4414
4415 // Split the input vectors into units of one HVX vector length.
4416 unsigned HwVectorSizeInBits = Subtarget.getVectorLength() * 8;
4417
4418 EVT AccType = Acc.getValueType();
4419 EVT AccEltType = AccType.getVectorElementType();
4420 unsigned AccSubvectorNumElements =
4421 HwVectorSizeInBits / AccEltType.getSizeInBits();
4422 EVT AccSubvectorType =
4423 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccEltType, NumElements: AccSubvectorNumElements);
4424
4425 EVT InputType = A.getValueType();
4426 assert(InputType.getSizeInBits() % HwVectorSizeInBits == 0);
4427 EVT InputEltType = InputType.getVectorElementType();
4428 unsigned InputSubvectorNumElements =
4429 HwVectorSizeInBits / InputEltType.getSizeInBits();
4430 EVT InputSubvectorType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InputEltType,
4431 NumElements: InputSubvectorNumElements);
4432
4433 unsigned SubvectorNum = InputType.getFixedSizeInBits() / HwVectorSizeInBits;
4434 SmallVector<SDValue, MaxExpandMLA> Subvectors;
4435
4436 for (unsigned I = 0; I != SubvectorNum; ++I) {
4437 SDValue SubvectorAcc = DAG.getExtractSubvector(DL, VT: AccSubvectorType, Vec: Acc,
4438 Idx: I * AccSubvectorNumElements);
4439 SDValue SubvectorA = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: A,
4440 Idx: I * InputSubvectorNumElements);
4441 SDValue SubvectorB = DAG.getExtractSubvector(DL, VT: InputSubvectorType, Vec: B,
4442 Idx: I * InputSubvectorNumElements);
4443 SDValue SubvectorMLA = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: AccSubvectorType,
4444 N1: SubvectorAcc, N2: SubvectorA, N3: SubvectorB);
4445 Subvectors.push_back(Elt: SubvectorMLA);
4446 }
4447
4448 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: AccType, Ops: Subvectors);
4449}
4450
4451SDValue
4452HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
4453 const {
4454 const SDLoc &dl(N);
4455 SelectionDAG &DAG = DCI.DAG;
4456 SDValue Op(N, 0);
4457 unsigned Opc = Op.getOpcode();
4458
4459 SmallVector<SDValue, 4> Ops(N->ops());
4460
4461 if (Opc == ISD::TRUNCATE)
4462 return combineTruncateBeforeLegal(Op, DCI);
4463 if (Opc == ISD::CONCAT_VECTORS)
4464 return combineConcatVectorsBeforeLegal(Op, DCI);
4465
4466 if (DCI.isBeforeLegalizeOps())
4467 return SDValue();
4468
4469 switch (Opc) {
4470 case HexagonISD::V2Q:
4471 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) {
4472 if (const auto *C = dyn_cast<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
4473 return C->isZero() ? DAG.getNode(Opcode: HexagonISD::QFALSE, DL: dl, VT: ty(Op))
4474 : DAG.getNode(Opcode: HexagonISD::QTRUE, DL: dl, VT: ty(Op));
4475 }
4476 break;
4477 case HexagonISD::Q2V:
4478 if (Ops[0].getOpcode() == HexagonISD::QTRUE)
4479 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: ty(Op),
4480 Operand: DAG.getAllOnesConstant(DL: dl, VT: MVT::i32));
4481 if (Ops[0].getOpcode() == HexagonISD::QFALSE)
4482 return getZero(dl, Ty: ty(Op), DAG);
4483 break;
4484 case HexagonISD::VINSERTW0:
4485 if (isUndef(Op: Ops[1]))
4486 return Ops[0];
4487 break;
4488 case HexagonISD::VROR: {
4489 if (Ops[0].getOpcode() == HexagonISD::VROR) {
4490 SDValue Vec = Ops[0].getOperand(i: 0);
4491 SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(i: 1);
4492 SDValue Rot = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ty(Op: Rot0), Ops: {Rot0, Rot1});
4493 return DAG.getNode(Opcode: HexagonISD::VROR, DL: dl, VT: ty(Op), Ops: {Vec, Rot});
4494 }
4495 break;
4496 }
4497 }
4498
4499 return SDValue();
4500}
4501
4502bool
4503HexagonTargetLowering::shouldSplitToHvx(MVT Ty, SelectionDAG &DAG) const {
4504 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4505 return false;
4506 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4507 if (Action == TargetLoweringBase::TypeSplitVector)
4508 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4509 return false;
4510}
4511
4512bool
4513HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const {
4514 if (Subtarget.isHVXVectorType(VecTy: Ty, IncludeBool: true))
4515 return false;
4516 auto Action = getPreferredHvxVectorAction(VecTy: Ty);
4517 if (Action == TargetLoweringBase::TypeWidenVector)
4518 return Subtarget.isHVXVectorType(VecTy: typeLegalize(Ty, DAG), IncludeBool: true);
4519 return false;
4520}
4521
4522bool
4523HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const {
4524 if (!Subtarget.useHVXOps())
4525 return false;
4526 // If the type of any result, or any operand type are HVX vector types,
4527 // this is an HVX operation.
4528 auto IsHvxTy = [this](EVT Ty) {
4529 return Ty.isSimple() && Subtarget.isHVXVectorType(VecTy: Ty.getSimpleVT(), IncludeBool: true);
4530 };
4531 auto IsHvxOp = [this](SDValue Op) {
4532 return Op.getValueType().isSimple() &&
4533 Subtarget.isHVXVectorType(VecTy: ty(Op), IncludeBool: true);
4534 };
4535 if (llvm::any_of(Range: N->values(), P: IsHvxTy) || llvm::any_of(Range: N->ops(), P: IsHvxOp))
4536 return true;
4537
4538 // Check if this could be an HVX operation after type widening.
4539 auto IsWidenedToHvx = [this, &DAG](SDValue Op) {
4540 if (!Op.getValueType().isSimple())
4541 return false;
4542 MVT ValTy = ty(Op);
4543 return ValTy.isVector() && shouldWidenToHvx(Ty: ValTy, DAG);
4544 };
4545
4546 for (int i = 0, e = N->getNumValues(); i != e; ++i) {
4547 if (IsWidenedToHvx(SDValue(N, i)))
4548 return true;
4549 }
4550 return llvm::any_of(Range: N->ops(), P: IsWidenedToHvx);
4551}
4552