1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
48 return;
49
50 switch (N->getOpcode()) {
51 default:
52#ifndef NDEBUG
53 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
54 N->dump(&DAG);
55 dbgs() << "\n";
56#endif
57 report_fatal_error(reason: "Do not know how to scalarize the result of this "
58 "operator!\n");
59
60 case ISD::LOOP_DEPENDENCE_WAR_MASK:
61 case ISD::LOOP_DEPENDENCE_RAW_MASK:
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
63 break;
64 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
65 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
66 case ISD::SPLAT_VECTOR:
67 case ISD::BUILD_VECTOR:
68 R = ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(N);
69 break;
70 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
71 case ISD::VECTOR_SHUFFLE_VAR:
72 R = ScalarizeVecRes_VECTOR_SHUFFLE_VAR(N);
73 break;
74 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
75 case ISD::CONVERT_FROM_ARBITRARY_FP:
76 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
77 break;
78 case ISD::CONVERT_TO_ARBITRARY_FP:
79 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
80 break;
81 case ISD::AssertZext:
82 case ISD::AssertSext:
83 case ISD::FPOWI:
84 case ISD::AssertNoFPClass:
85 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
86 break;
87 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
88 case ISD::ATOMIC_LOAD:
89 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
90 break;
91 case ISD::ATOMIC_SWAP:
92 case ISD::ATOMIC_LOAD_ADD:
93 case ISD::ATOMIC_LOAD_SUB:
94 case ISD::ATOMIC_LOAD_AND:
95 case ISD::ATOMIC_LOAD_OR:
96 case ISD::ATOMIC_LOAD_XOR:
97 case ISD::ATOMIC_LOAD_NAND:
98 case ISD::ATOMIC_LOAD_MIN:
99 case ISD::ATOMIC_LOAD_MAX:
100 case ISD::ATOMIC_LOAD_UMIN:
101 case ISD::ATOMIC_LOAD_UMAX:
102 case ISD::ATOMIC_LOAD_FADD:
103 case ISD::ATOMIC_LOAD_FSUB:
104 case ISD::ATOMIC_LOAD_FMIN:
105 case ISD::ATOMIC_LOAD_FMAX:
106 case ISD::ATOMIC_LOAD_FMINIMUM:
107 case ISD::ATOMIC_LOAD_FMAXIMUM:
108 case ISD::ATOMIC_LOAD_FMINIMUMNUM:
109 case ISD::ATOMIC_LOAD_FMAXIMUMNUM:
110 case ISD::ATOMIC_LOAD_UINC_WRAP:
111 case ISD::ATOMIC_LOAD_UDEC_WRAP:
112 case ISD::ATOMIC_LOAD_USUB_COND:
113 case ISD::ATOMIC_LOAD_USUB_SAT:
114 R = ScalarizeVecRes_ATOMIC_RMW(N: cast<AtomicSDNode>(Val: N));
115 break;
116 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
117 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
118 case ISD::VECTOR_DEINTERLEAVE:
119 case ISD::VECTOR_INTERLEAVE:
120 R = ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
121 break;
122 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
123 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
124 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
125 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
126 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
127 case ISD::VECTOR_MATCH:
128 R = ScalarizeVecRes_VECTOR_MATCH(N);
129 break;
130 case ISD::POISON:
131 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
132 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
133 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
134 case ISD::ANY_EXTEND_VECTOR_INREG:
135 case ISD::SIGN_EXTEND_VECTOR_INREG:
136 case ISD::ZERO_EXTEND_VECTOR_INREG:
137 R = ScalarizeVecRes_VecInregOp(N);
138 break;
139 case ISD::ABS:
140 case ISD::ABS_MIN_POISON:
141 case ISD::ANY_EXTEND:
142 case ISD::BITREVERSE:
143 case ISD::BSWAP:
144 case ISD::CTLZ:
145 case ISD::CTLZ_ZERO_POISON:
146 case ISD::CTPOP:
147 case ISD::CTTZ:
148 case ISD::CTTZ_ZERO_POISON:
149 case ISD::FABS:
150 case ISD::FACOS:
151 case ISD::FASIN:
152 case ISD::FATAN:
153 case ISD::FCEIL:
154 case ISD::FCOS:
155 case ISD::FCOSH:
156 case ISD::FEXP:
157 case ISD::FEXP2:
158 case ISD::FEXP10:
159 case ISD::FFLOOR:
160 case ISD::FLOG:
161 case ISD::FLOG10:
162 case ISD::FLOG2:
163 case ISD::FNEARBYINT:
164 case ISD::FNEG:
165 case ISD::FREEZE:
166 case ISD::ARITH_FENCE:
167 case ISD::FP_EXTEND:
168 case ISD::FP_TO_SINT:
169 case ISD::FP_TO_UINT:
170 case ISD::FRINT:
171 case ISD::LRINT:
172 case ISD::LLRINT:
173 case ISD::FROUND:
174 case ISD::FROUNDEVEN:
175 case ISD::LROUND:
176 case ISD::LLROUND:
177 case ISD::FSIN:
178 case ISD::FSINH:
179 case ISD::FSQRT:
180 case ISD::FTAN:
181 case ISD::FTANH:
182 case ISD::FTRUNC:
183 case ISD::SIGN_EXTEND:
184 case ISD::SINT_TO_FP:
185 case ISD::TRUNCATE:
186 case ISD::UINT_TO_FP:
187 case ISD::ZERO_EXTEND:
188 case ISD::FCANONICALIZE:
189 R = ScalarizeVecRes_UnaryOp(N);
190 break;
191 case ISD::ADDRSPACECAST:
192 R = ScalarizeVecRes_ADDRSPACECAST(N);
193 break;
194 case ISD::FMODF:
195 case ISD::FFREXP:
196 case ISD::FSINCOS:
197 case ISD::FSINCOSPI:
198 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
199 break;
200 case ISD::ADD:
201 case ISD::AND:
202 case ISD::AVGCEILS:
203 case ISD::AVGCEILU:
204 case ISD::AVGFLOORS:
205 case ISD::AVGFLOORU:
206 case ISD::FADD:
207 case ISD::FCOPYSIGN:
208 case ISD::FDIV:
209 case ISD::FMUL:
210 case ISD::FMINNUM:
211 case ISD::FMAXNUM:
212 case ISD::FMINNUM_IEEE:
213 case ISD::FMAXNUM_IEEE:
214 case ISD::FMINIMUM:
215 case ISD::FMAXIMUM:
216 case ISD::FMINIMUMNUM:
217 case ISD::FMAXIMUMNUM:
218 case ISD::ABDS:
219 case ISD::ABDU:
220 case ISD::SMIN:
221 case ISD::SMAX:
222 case ISD::UMIN:
223 case ISD::UMAX:
224
225 case ISD::SADDSAT:
226 case ISD::UADDSAT:
227 case ISD::SSUBSAT:
228 case ISD::USUBSAT:
229 case ISD::SSHLSAT:
230 case ISD::USHLSAT:
231
232 case ISD::FPOW:
233 case ISD::FATAN2:
234 case ISD::FREM:
235 case ISD::FSUB:
236 case ISD::MUL:
237 case ISD::MULHS:
238 case ISD::MULHU:
239 case ISD::OR:
240 case ISD::SDIV:
241 case ISD::SREM:
242 case ISD::SUB:
243 case ISD::UDIV:
244 case ISD::UREM:
245 case ISD::XOR:
246 case ISD::SHL:
247 case ISD::SRA:
248 case ISD::SRL:
249 case ISD::ROTL:
250 case ISD::ROTR:
251 case ISD::CLMUL:
252 case ISD::CLMULR:
253 case ISD::CLMULH:
254 case ISD::PEXT:
255 case ISD::PDEP:
256 R = ScalarizeVecRes_BinOp(N);
257 break;
258
259 case ISD::MASKED_UDIV:
260 case ISD::MASKED_SDIV:
261 case ISD::MASKED_UREM:
262 case ISD::MASKED_SREM:
263 R = ScalarizeVecRes_MaskedBinOp(N);
264 break;
265
266 case ISD::FLDEXP:
267 R = ScalarizeVecRes_FPOp_MultiType(N);
268 break;
269
270 case ISD::SCMP:
271 case ISD::UCMP:
272 R = ScalarizeVecRes_CMP(N);
273 break;
274
275 case ISD::FMA:
276 case ISD::FSHL:
277 case ISD::FSHR:
278 R = ScalarizeVecRes_TernaryOp(N);
279 break;
280
281#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
282 case ISD::STRICT_##DAGN:
283#include "llvm/IR/ConstrainedOps.def"
284 R = ScalarizeVecRes_StrictFPOp(N);
285 break;
286
287 case ISD::FP_TO_UINT_SAT:
288 case ISD::FP_TO_SINT_SAT:
289 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
290 break;
291
292 case ISD::UADDO:
293 case ISD::SADDO:
294 case ISD::USUBO:
295 case ISD::SSUBO:
296 case ISD::UMULO:
297 case ISD::SMULO:
298 R = ScalarizeVecRes_OverflowOp(N, ResNo);
299 break;
300 case ISD::SMULFIX:
301 case ISD::SMULFIXSAT:
302 case ISD::UMULFIX:
303 case ISD::UMULFIXSAT:
304 case ISD::SDIVFIX:
305 case ISD::SDIVFIXSAT:
306 case ISD::UDIVFIX:
307 case ISD::UDIVFIXSAT:
308 R = ScalarizeVecRes_FIX(N);
309 break;
310 case ISD::PARTIAL_REDUCE_FMLA:
311 case ISD::PARTIAL_REDUCE_SMLA:
312 case ISD::PARTIAL_REDUCE_SUMLA:
313 case ISD::PARTIAL_REDUCE_UMLA:
314 R = ScalarizeVecRes_PartialReduction(N);
315 break;
316 }
317
318 // If R is null, the sub-method took care of registering the result.
319 if (R.getNode())
320 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
321}
322
323SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
324 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
325 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
326 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
327 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
328}
329
330SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
331 SDLoc DL(N);
332 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
333 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
334 SDValue Mask = N->getOperand(Num: 2);
335 EVT MaskVT = Mask.getValueType();
336 // The vselect result and input vectors need scalarizing, but it's
337 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
338 // See the similar logic in ScalarizeVecRes_SETCC.
339 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeScalarizeVector)
340 Mask = GetScalarizedVector(Op: Mask);
341 else
342 Mask = DAG.getExtractVectorElt(DL, VT: MaskVT.getVectorElementType(), Vec: Mask, Idx: 0);
343 // Vectors may have a different boolean contents to scalars, so truncate to i1
344 // and let type legalization promote appropriately.
345 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
346 // Masked binary ops don't have UB on disabled lanes but produce poison, so
347 // use 1 as the divisor to avoid division by zero and overflow.
348 SDValue Divisor = DAG.getSelect(DL, VT: LHS.getValueType(), Cond: Mask, LHS: RHS,
349 RHS: DAG.getConstant(Val: 1, DL, VT: LHS.getValueType()));
350 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL,
351 VT: LHS.getValueType(), N1: LHS, N2: Divisor);
352}
353
354SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
355 SDLoc DL(N);
356
357 SDValue LHS = N->getOperand(Num: 0);
358 SDValue RHS = N->getOperand(Num: 1);
359 if (getTypeAction(VT: LHS.getValueType()) ==
360 TargetLowering::TypeScalarizeVector) {
361 LHS = GetScalarizedVector(Op: LHS);
362 RHS = GetScalarizedVector(Op: RHS);
363 } else {
364 EVT VT = LHS.getValueType().getVectorElementType();
365 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
366 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
367 }
368
369 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
370 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
371}
372
373SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
374 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
375 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
376 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
377 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
378 N3: Op2, Flags: N->getFlags());
379}
380
381SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
382 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
383 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
384 SDValue Op2 = N->getOperand(Num: 2);
385 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
386 N3: Op2, Flags: N->getFlags());
387}
388
389SDValue DAGTypeLegalizer::ScalarizeVecRes_PartialReduction(SDNode *N) {
390 SDValue Acc = GetScalarizedVector(Op: N->getOperand(Num: 0));
391 SDValue L = N->getOperand(Num: 1);
392 SDValue R = N->getOperand(Num: 2);
393 EVT ResVT = Acc.getValueType();
394 EVT InVT = L.getValueType();
395 bool IsFP = ResVT.isFloatingPoint();
396 SDLoc DL(N);
397
398 // Make sure we're not trying an extending partial reduction in this path.
399 assert(InVT.getScalarType() == ResVT && "Extending operands unsupported");
400
401 SDValue Mul = DAG.getNode(Opcode: IsFP ? ISD::FMUL : ISD::MUL, DL, VT: InVT, N1: L, N2: R);
402 SDValue In = DAG.getNode(Opcode: IsFP ? ISD::VECREDUCE_FADD : ISD::VECREDUCE_ADD, DL,
403 VT: ResVT, Operand: Mul);
404
405 return DAG.getNode(Opcode: IsFP ? ISD::FADD : ISD::ADD, DL, VT: ResVT, N1: Acc, N2: In);
406}
407
408SDValue
409DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
410 unsigned ResNo) {
411 assert(N->getValueType(0).getVectorNumElements() == 1 &&
412 "Unexpected vector type!");
413 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
414
415 EVT VT0 = N->getValueType(ResNo: 0);
416 EVT VT1 = N->getValueType(ResNo: 1);
417 SDLoc dl(N);
418
419 SDNode *ScalarNode =
420 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
421 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
422 .getNode();
423
424 // Replace the other vector result not being explicitly scalarized here.
425 unsigned OtherNo = 1 - ResNo;
426 EVT OtherVT = N->getValueType(ResNo: OtherNo);
427 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
428 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
429 } else {
430 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
431 Operand: SDValue(ScalarNode, OtherNo));
432 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
433 }
434
435 return SDValue(ScalarNode, ResNo);
436}
437
438SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
439 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
440 unsigned NumOpers = N->getNumOperands();
441 SDValue Chain = N->getOperand(Num: 0);
442 EVT ValueVTs[] = {VT, MVT::Other};
443 SDLoc dl(N);
444
445 SmallVector<SDValue, 4> Opers(NumOpers);
446
447 // The Chain is the first operand.
448 Opers[0] = Chain;
449
450 // Now process the remaining operands.
451 for (unsigned i = 1; i < NumOpers; ++i) {
452 SDValue Oper = N->getOperand(Num: i);
453 EVT OperVT = Oper.getValueType();
454
455 if (OperVT.isVector()) {
456 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
457 Oper = GetScalarizedVector(Op: Oper);
458 else
459 Oper =
460 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
461 }
462
463 Opers[i] = Oper;
464 }
465
466 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
467 Ops: Opers, Flags: N->getFlags());
468
469 // Legalize the chain result - switch anything that used the old chain to
470 // use the new one.
471 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
472 return Result;
473}
474
475SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
476 unsigned ResNo) {
477 SDLoc DL(N);
478 EVT ResVT = N->getValueType(ResNo: 0);
479 EVT OvVT = N->getValueType(ResNo: 1);
480
481 SDValue ScalarLHS, ScalarRHS;
482 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
483 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
484 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
485 } else {
486 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
487 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
488 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
489 ScalarLHS = ElemsLHS[0];
490 ScalarRHS = ElemsRHS[0];
491 }
492
493 SDVTList ScalarVTs = DAG.getVTList(
494 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
495 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
496 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
497 .getNode();
498
499 // Replace the other vector result not being explicitly scalarized here.
500 unsigned OtherNo = 1 - ResNo;
501 EVT OtherVT = N->getValueType(ResNo: OtherNo);
502 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
503 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
504 } else {
505 SDValue OtherVal = DAG.getNode(
506 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
507 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
508 }
509
510 return SDValue(ScalarNode, ResNo);
511}
512
513SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
514 unsigned ResNo) {
515 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
516 return GetScalarizedVector(Op);
517}
518
519SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
520 SDLoc DL(N);
521 // Reuse the expansion (which should scalarize).
522 SDValue Mask = TLI.expandLoopDependenceMask(N, DAG);
523 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
524 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
525 N2: DAG.getVectorIdxConstant(Val: 0, DL));
526}
527
528SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
529 SDValue Op = N->getOperand(Num: 0);
530 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
531 Op = GetScalarizedVector(Op);
532 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
533 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
534 VT: NewVT, Operand: Op);
535}
536
537SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(SDNode *N) {
538 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
539 SDValue InOp = N->getOperand(Num: 0);
540 // The BUILD_VECTOR / SPLAT operands may be of wider element types and
541 // we may need to truncate them back to the requested return type.
542 if (EltVT.isInteger())
543 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
544 return InOp;
545}
546
547SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
548 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
549 VT: N->getValueType(ResNo: 0).getVectorElementType(),
550 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
551}
552
553SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
554 // Index 0 is the only in-range index.
555 return GetScalarizedVector(Op: N->getOperand(Num: 0));
556}
557
558SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
559 SDLoc DL(N);
560 SDValue Op = N->getOperand(Num: 0);
561 EVT OpVT = Op.getValueType();
562 // The result needs scalarizing, but it's not a given that the source does.
563 // See similar logic in ScalarizeVecRes_UnaryOp.
564 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
565 Op = GetScalarizedVector(Op);
566 } else {
567 EVT VT = OpVT.getVectorElementType();
568 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
569 }
570 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
571 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
572 N2: N->getOperand(Num: 1));
573}
574
575SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
576 SDLoc DL(N);
577 SDValue Op = N->getOperand(Num: 0);
578 EVT OpVT = Op.getValueType();
579 // The result needs scalarizing, but it's not a given that the source does.
580 // See similar logic in ScalarizeVecRes_UnaryOp.
581 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
582 Op = GetScalarizedVector(Op);
583 } else {
584 EVT VT = OpVT.getVectorElementType();
585 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
586 }
587 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
588 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
589 N2: N->getOperand(Num: 1));
590}
591
592SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
593 SDLoc DL(N);
594 SDValue Op = N->getOperand(Num: 0);
595 EVT OpVT = Op.getValueType();
596 // The result needs scalarizing, but it's not a given that the source does.
597 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
598 Op = GetScalarizedVector(Op);
599 } else {
600 EVT VT = OpVT.getVectorElementType();
601 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
602 }
603 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL,
604 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
605 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
606}
607
608SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
609 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
610 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
611 N2: N->getOperand(Num: 1));
612}
613
614SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOp_MultiType(SDNode *N) {
615 SDLoc DL(N);
616 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
617 SDValue RHS = N->getOperand(Num: 1);
618 EVT RHSVT = RHS.getValueType();
619 // The exponent has its own type action and may not have been scalarized:
620 // v1i1 is legal on AVX-512, v1i32 is widened on AArch64.
621 if (RHSVT.isVector()) {
622 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeScalarizeVector)
623 RHS = GetScalarizedVector(Op: RHS);
624 else
625 RHS = DAG.getExtractVectorElt(DL, VT: RHSVT.getVectorElementType(), Vec: RHS, Idx: 0);
626 }
627 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHS.getValueType(), N1: LHS, N2: RHS,
628 Flags: N->getFlags());
629}
630
631SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
632 // The value to insert may have a wider type than the vector element type,
633 // so be sure to truncate it to the element type if necessary.
634 SDValue Op = N->getOperand(Num: 1);
635 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
636 if (Op.getValueType() != EltVT)
637 // FIXME: Can this happen for floating point types?
638 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
639 return Op;
640}
641
642SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
643 SDValue Result = DAG.getAtomicLoad(
644 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
645 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
646 MMO: N->getMemOperand());
647
648 // Legalize the chain result - switch anything that used the old chain to
649 // use the new one.
650 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
651 return Result;
652}
653
654SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_RMW(AtomicSDNode *N) {
655 SDValue Result = DAG.getAtomic(
656 Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
657 Chain: N->getChain(), Ptr: N->getBasePtr(), Val: GetScalarizedVector(Op: N->getVal()),
658 MMO: N->getMemOperand());
659
660 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
661 return Result;
662}
663
664SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
665 assert(N->isUnindexed() && "Indexed vector load?");
666
667 SDValue Result = DAG.getLoad(
668 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
669 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
670 Ptr: N->getBasePtr(), Offset: DAG.getPOISON(VT: N->getBasePtr().getValueType()),
671 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
672 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(),
673 Metadata: N->getMMOMetadataForSubAccess());
674
675 // Legalize the chain result - switch anything that used the old chain to
676 // use the new one.
677 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
678 return Result;
679}
680
681SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
682 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
683 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
684 SDValue Op = N->getOperand(Num: 0);
685 EVT OpVT = Op.getValueType();
686 SDLoc DL(N);
687 // The result needs scalarizing, but it's not a given that the source does.
688 // This is a workaround for targets where it's impossible to scalarize the
689 // result of a conversion, because the source type is legal.
690 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
691 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
692 // legal and was not scalarized.
693 // See the similar logic in ScalarizeVecRes_SETCC
694 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
695 Op = GetScalarizedVector(Op);
696 } else {
697 EVT VT = OpVT.getVectorElementType();
698 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
699 }
700 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
701}
702
703SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
704 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
705 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
706 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
707 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
708 N1: LHS, N2: DAG.getValueType(ExtVT));
709}
710
711SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
712 SDLoc DL(N);
713 SDValue Op = N->getOperand(Num: 0);
714
715 EVT OpVT = Op.getValueType();
716 EVT OpEltVT = OpVT.getVectorElementType();
717 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
718
719 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
720 Op = GetScalarizedVector(Op);
721 } else {
722 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
723 }
724
725 switch (N->getOpcode()) {
726 case ISD::ANY_EXTEND_VECTOR_INREG:
727 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
728 case ISD::SIGN_EXTEND_VECTOR_INREG:
729 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
730 case ISD::ZERO_EXTEND_VECTOR_INREG:
731 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
732 }
733
734 llvm_unreachable("Illegal extend_vector_inreg opcode");
735}
736
737SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
738 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
739 SDValue Op = N->getOperand(Num: 0);
740 EVT OpVT = Op.getValueType();
741 SDLoc DL(N);
742 // The result needs scalarizing, but it's not a given that the source does.
743 // This is a workaround for targets where it's impossible to scalarize the
744 // result of a conversion, because the source type is legal.
745 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
746 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
747 // legal and was not scalarized.
748 // See the similar logic in ScalarizeVecRes_SETCC
749 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
750 Op = GetScalarizedVector(Op);
751 } else {
752 EVT VT = OpVT.getVectorElementType();
753 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
754 }
755 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
756 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
757 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
758 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS,
759 Flags: AddrSpaceCastN->getFlags());
760}
761
762SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
763 // If the operand is wider than the vector element type then it is implicitly
764 // truncated. Make that explicit here.
765 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
766 SDValue InOp = N->getOperand(Num: 0);
767 if (InOp.getValueType() != EltVT)
768 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
769 return InOp;
770}
771
772SDValue
773DAGTypeLegalizer::ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
774 assert(N->getNumValues() == N->getNumOperands() &&
775 "Expected one result per operand");
776
777 // Interleaving or deinterleaving one-element vectors leaves each result
778 // equal to the corresponding operand.
779 for (unsigned I = 0; I != N->getNumValues(); ++I)
780 SetScalarizedVector(Op: SDValue(N, I), Result: GetScalarizedVector(Op: N->getOperand(Num: I)));
781 return SDValue();
782}
783
784SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
785 SDValue Cond = N->getOperand(Num: 0);
786 EVT OpVT = Cond.getValueType();
787 SDLoc DL(N);
788 // The vselect result and true/value operands needs scalarizing, but it's
789 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
790 // See the similar logic in ScalarizeVecRes_SETCC
791 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
792 Cond = GetScalarizedVector(Op: Cond);
793 } else {
794 EVT VT = OpVT.getVectorElementType();
795 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
796 }
797
798 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
799 TargetLowering::BooleanContent ScalarBool =
800 TLI.getBooleanContents(isVec: false, isFloat: false);
801 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
802
803 // If integer and float booleans have different contents then we can't
804 // reliably optimize in all cases. There is a full explanation for this in
805 // DAGCombiner::visitSELECT() where the same issue affects folding
806 // (select C, 0, 1) to (xor C, 1).
807 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
808 TLI.getBooleanContents(isVec: false, isFloat: true)) {
809 // At least try the common case where the boolean is generated by a
810 // comparison.
811 if (Cond->getOpcode() == ISD::SETCC) {
812 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
813 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
814 VecBool = TLI.getBooleanContents(Type: OpVT);
815 } else
816 ScalarBool = TargetLowering::UndefinedBooleanContent;
817 }
818
819 EVT CondVT = Cond.getValueType();
820 if (ScalarBool != VecBool) {
821 switch (ScalarBool) {
822 case TargetLowering::UndefinedBooleanContent:
823 break;
824 case TargetLowering::ZeroOrOneBooleanContent:
825 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
826 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
827 // Vector read from all ones, scalar expects a single 1 so mask.
828 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
829 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
830 break;
831 case TargetLowering::ZeroOrNegativeOneBooleanContent:
832 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
833 VecBool == TargetLowering::ZeroOrOneBooleanContent);
834 // Vector reads from a one, scalar from all ones so sign extend.
835 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
836 N1: Cond, N2: DAG.getValueType(MVT::i1));
837 break;
838 }
839 }
840
841 // Truncate the condition if needed
842 auto BoolVT = getSetCCResultType(VT: CondVT);
843 if (BoolVT.bitsLT(VT: CondVT))
844 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
845
846 return DAG.getSelect(DL: SDLoc(N), VT: LHS.getValueType(), Cond, LHS,
847 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)), Flags: N->getFlags());
848}
849
850SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
851 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
852 return DAG.getSelect(DL: SDLoc(N),
853 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
854 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
855}
856
857SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
858 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
859 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
860 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
861 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
862 N5: N->getOperand(Num: 4));
863}
864
865SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
866 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
867}
868
869SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
870 // Figure out if the scalar is the LHS or RHS and return it.
871 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
872 if (Arg.isUndef())
873 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
874 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
875 return GetScalarizedVector(Op: N->getOperand(Num: Op));
876}
877
878SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
879 SDValue Src = N->getOperand(Num: 0);
880 EVT SrcVT = Src.getValueType();
881 SDLoc dl(N);
882
883 // Handle case where result is scalarized but operand is not
884 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
885 Src = GetScalarizedVector(Op: Src);
886 else
887 Src = DAG.getNode(
888 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
889 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
890
891 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
892 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
893}
894
895SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
896 assert(N->getValueType(0).isVector() &&
897 N->getOperand(0).getValueType().isVector() &&
898 "Operand types must be vectors");
899 SDValue LHS = N->getOperand(Num: 0);
900 SDValue RHS = N->getOperand(Num: 1);
901 EVT OpVT = LHS.getValueType();
902 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
903 SDLoc DL(N);
904
905 // The result needs scalarizing, but it's not a given that the source does.
906 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
907 LHS = GetScalarizedVector(Op: LHS);
908 RHS = GetScalarizedVector(Op: RHS);
909 } else {
910 EVT VT = OpVT.getVectorElementType();
911 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
912 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
913 }
914
915 // Turn it into a scalar SETCC.
916 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
917 N3: N->getOperand(Num: 2));
918 // Vectors may have a different boolean contents to scalars. Promote the
919 // value appropriately.
920 ISD::NodeType ExtendCode =
921 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
922 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
923}
924
925SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
926 SDLoc DL(N);
927 SDValue Arg = N->getOperand(Num: 0);
928 SDValue Test = N->getOperand(Num: 1);
929 EVT ArgVT = Arg.getValueType();
930 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
931
932 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
933 Arg = GetScalarizedVector(Op: Arg);
934 } else {
935 EVT VT = ArgVT.getVectorElementType();
936 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
937 }
938
939 SDValue Res =
940 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
941 // Vectors may have a different boolean contents to scalars. Promote the
942 // value appropriately.
943 ISD::NodeType ExtendCode =
944 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
945 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
946}
947
948//===----------------------------------------------------------------------===//
949// Operand Vector Scalarization <1 x ty> -> ty.
950//===----------------------------------------------------------------------===//
951
952bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
953 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
954 N->dump(&DAG));
955 SDValue Res = SDValue();
956
957 // See if the target wants to custom scalarize this node.
958 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
959 return false;
960
961 switch (N->getOpcode()) {
962 default:
963#ifndef NDEBUG
964 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
965 N->dump(&DAG);
966 dbgs() << "\n";
967#endif
968 report_fatal_error(reason: "Do not know how to scalarize this operator's "
969 "operand!\n");
970 case ISD::BITCAST:
971 Res = ScalarizeVecOp_BITCAST(N);
972 break;
973 case ISD::FAKE_USE:
974 Res = ScalarizeVecOp_FAKE_USE(N);
975 break;
976 case ISD::VECTOR_SHUFFLE_VAR:
977 // Index 0 is the only in-range index.
978 Res = N->getOperand(Num: 0);
979 break;
980 case ISD::ANY_EXTEND:
981 case ISD::ZERO_EXTEND:
982 case ISD::SIGN_EXTEND:
983 case ISD::TRUNCATE:
984 case ISD::FP_TO_SINT:
985 case ISD::FP_TO_UINT:
986 case ISD::SINT_TO_FP:
987 case ISD::UINT_TO_FP:
988 case ISD::LROUND:
989 case ISD::LLROUND:
990 case ISD::LRINT:
991 case ISD::LLRINT:
992 Res = ScalarizeVecOp_UnaryOp(N);
993 break;
994 case ISD::FP_TO_SINT_SAT:
995 case ISD::FP_TO_UINT_SAT:
996 case ISD::CONVERT_FROM_ARBITRARY_FP:
997 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
998 break;
999 case ISD::CONVERT_TO_ARBITRARY_FP: {
1000 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1001 "Unexpected vector type!");
1002 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1003 SDValue Op = DAG.getNode(
1004 Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(), N1: Elt,
1005 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
1006 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1007 break;
1008 }
1009 case ISD::STRICT_SINT_TO_FP:
1010 case ISD::STRICT_UINT_TO_FP:
1011 case ISD::STRICT_FP_TO_SINT:
1012 case ISD::STRICT_FP_TO_UINT:
1013 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
1014 break;
1015 case ISD::CONCAT_VECTORS:
1016 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
1017 break;
1018 case ISD::INSERT_SUBVECTOR:
1019 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
1020 break;
1021 case ISD::EXTRACT_VECTOR_ELT:
1022 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
1023 break;
1024 case ISD::VSELECT:
1025 Res = ScalarizeVecOp_VSELECT(N);
1026 break;
1027 case ISD::SETCC:
1028 Res = ScalarizeVecOp_VSETCC(N);
1029 break;
1030 case ISD::STRICT_FSETCC:
1031 case ISD::STRICT_FSETCCS:
1032 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
1033 break;
1034 case ISD::STORE:
1035 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
1036 break;
1037 case ISD::ATOMIC_STORE:
1038 Res = ScalarizeVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
1039 break;
1040 case ISD::MLOAD:
1041 Res = ScalarizeVecOp_MLOAD(N: cast<MaskedLoadSDNode>(Val: N), OpNo);
1042 break;
1043 case ISD::MSTORE:
1044 Res = ScalarizeVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
1045 break;
1046 case ISD::STRICT_FP_ROUND:
1047 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
1048 break;
1049 case ISD::FP_ROUND:
1050 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
1051 break;
1052 case ISD::STRICT_FP_EXTEND:
1053 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
1054 break;
1055 case ISD::FP_EXTEND:
1056 Res = ScalarizeVecOp_FP_EXTEND(N);
1057 break;
1058 case ISD::VECREDUCE_FADD:
1059 case ISD::VECREDUCE_FMUL:
1060 case ISD::VECREDUCE_ADD:
1061 case ISD::VECREDUCE_MUL:
1062 case ISD::VECREDUCE_AND:
1063 case ISD::VECREDUCE_OR:
1064 case ISD::VECREDUCE_XOR:
1065 case ISD::VECREDUCE_SMAX:
1066 case ISD::VECREDUCE_SMIN:
1067 case ISD::VECREDUCE_UMAX:
1068 case ISD::VECREDUCE_UMIN:
1069 case ISD::VECREDUCE_FMAX:
1070 case ISD::VECREDUCE_FMIN:
1071 case ISD::VECREDUCE_FMAXIMUM:
1072 case ISD::VECREDUCE_FMINIMUM:
1073 case ISD::VECREDUCE_FMAXIMUMNUM:
1074 case ISD::VECREDUCE_FMINIMUMNUM:
1075 Res = ScalarizeVecOp_VECREDUCE(N);
1076 break;
1077 case ISD::VECREDUCE_SEQ_FADD:
1078 case ISD::VECREDUCE_SEQ_FMUL:
1079 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
1080 break;
1081 case ISD::SCMP:
1082 case ISD::UCMP:
1083 Res = ScalarizeVecOp_CMP(N);
1084 break;
1085 case ISD::VECTOR_FIND_LAST_ACTIVE:
1086 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
1087 break;
1088 case ISD::CTTZ_ELTS:
1089 case ISD::CTTZ_ELTS_ZERO_POISON:
1090 Res = ScalarizeVecOp_CTTZ_ELTS(N);
1091 break;
1092 case ISD::VECTOR_MATCH:
1093 Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
1094 break;
1095 case ISD::MASKED_UDIV:
1096 case ISD::MASKED_SDIV:
1097 case ISD::MASKED_UREM:
1098 case ISD::MASKED_SREM:
1099 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
1100 break;
1101 }
1102
1103 // If the result is null, the sub-method took care of registering results etc.
1104 if (!Res.getNode()) return false;
1105
1106 // If the result is N, the sub-method updated N in place. Tell the legalizer
1107 // core about this.
1108 if (Res.getNode() == N)
1109 return true;
1110
1111 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1112 "Invalid operand expansion");
1113
1114 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1115 return false;
1116}
1117
1118/// If the value to convert is a vector that needs to be scalarized, it must be
1119/// <1 x ty>. Convert the element instead.
1120SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
1121 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1122 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
1123 VT: N->getValueType(ResNo: 0), Operand: Elt);
1124}
1125
1126// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1127SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1128 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1129 "Fake Use: Unexpected vector type!");
1130 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1131 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
1132}
1133
1134/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1135/// Do the operation on the element instead.
1136SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1137 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1138 "Unexpected vector type!");
1139 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1140 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1141 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
1142 // Revectorize the result so the types line up with what the uses of this
1143 // expression expect.
1144 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1145}
1146
1147/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1148/// typesize).
1149SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1150 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1151 "Unexpected vector type!");
1152 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1153 SDValue Op =
1154 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
1155 N1: Elt, N2: N->getOperand(Num: 1));
1156 // Revectorize the result so the types line up with what the uses of this
1157 // expression expect.
1158 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1159}
1160
1161/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1162/// Do the strict FP operation on the element instead.
1163SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1164 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1165 "Unexpected vector type!");
1166 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1167 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1168 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
1169 Ops: { N->getOperand(Num: 0), Elt });
1170 // Legalize the chain result - switch anything that used the old chain to
1171 // use the new one.
1172 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1173 // Revectorize the result so the types line up with what the uses of this
1174 // expression expect.
1175 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1176
1177 // Do our own replacement and return SDValue() to tell the caller that we
1178 // handled all replacements since caller can only handle a single result.
1179 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1180 return SDValue();
1181}
1182
1183/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1184SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1185 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1186 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1187 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1188 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1189}
1190
1191/// The inserted subvector is to be scalarized - use insert vector element
1192/// instead.
1193SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1194 unsigned OpNo) {
1195 // We should not be attempting to scalarize the containing vector
1196 assert(OpNo == 1);
1197 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1198 SDValue ContainingVec = N->getOperand(Num: 0);
1199 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1200 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1201 N3: N->getOperand(Num: 2));
1202}
1203
1204/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1205/// so just return the element, ignoring the index.
1206SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1207 EVT VT = N->getValueType(ResNo: 0);
1208 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1209 if (Res.getValueType() != VT)
1210 Res = VT.isFloatingPoint()
1211 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1212 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1213 return Res;
1214}
1215
1216/// If the input condition is a vector that needs to be scalarized, it must be
1217/// <1 x i1>, so just convert to a normal ISD::SELECT
1218/// (still with vector output type since that was acceptable if we got here).
1219SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1220 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1221 EVT VT = N->getValueType(ResNo: 0);
1222
1223 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1224 N3: N->getOperand(Num: 2));
1225}
1226
1227/// If the operand is a vector that needs to be scalarized then the
1228/// result must be a single-element vector, so just convert to a scalar
1229/// SETCC and wrap with a scalar_to_vector since the res type is legal
1230/// if we got here
1231SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1232 assert(N->getValueType(0).isVector() &&
1233 N->getOperand(0).getValueType().isVector() &&
1234 "Operand types must be vectors");
1235 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1236 "Expected single-element vector type");
1237
1238 EVT VT = N->getValueType(ResNo: 0);
1239 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1240 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1241
1242 EVT OpVT = N->getOperand(Num: 0).getValueType();
1243 EVT NVT = VT.getVectorElementType();
1244 SDLoc DL(N);
1245 // Turn it into a scalar SETCC.
1246 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1247 N3: N->getOperand(Num: 2));
1248
1249 // Vectors may have a different boolean contents to scalars. Promote the
1250 // value appropriately.
1251 ISD::NodeType ExtendCode =
1252 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1253
1254 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1255
1256 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1257}
1258
1259// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1260SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1261 unsigned OpNo) {
1262 assert(OpNo == 1 && "Wrong operand for scalarization!");
1263 assert(N->getValueType(0).isVector() &&
1264 N->getOperand(1).getValueType().isVector() &&
1265 "Operand types must be vectors");
1266 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1267 "Expected single-element vector type");
1268
1269 EVT VT = N->getValueType(ResNo: 0);
1270 SDValue Ch = N->getOperand(Num: 0);
1271 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1272 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1273 SDValue CC = N->getOperand(Num: 3);
1274
1275 EVT OpVT = N->getOperand(Num: 1).getValueType();
1276 EVT NVT = VT.getVectorElementType();
1277 SDLoc DL(N);
1278 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1279 Ops: {Ch, LHS, RHS, CC});
1280
1281 // Legalize the chain result - switch anything that used the old chain to
1282 // use the new one.
1283 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1284
1285 ISD::NodeType ExtendCode =
1286 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1287
1288 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1289 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1290
1291 // Do our own replacement and return SDValue() to tell the caller that we
1292 // handled all replacements since caller can only handle a single result.
1293 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1294 return SDValue();
1295}
1296
1297/// If the value to store is a vector that needs to be scalarized, it must be
1298/// <1 x ty>. Just store the element.
1299SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1300 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1301 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1302 SDLoc dl(N);
1303
1304 if (N->isTruncatingStore())
1305 return DAG.getTruncStore(
1306 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1307 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1308 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1309 MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getMMOMetadataForSubAccess());
1310
1311 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1312 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1313 MMOFlags: N->getMemOperand()->getFlags(),
1314 Metadata: N->getMMOMetadataForSubAccess());
1315}
1316
1317/// If the value to store is a vector that needs to be scalarized, it must be
1318/// <1 x ty>. Just store the element.
1319SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1320 SDValue ScalarVal = GetScalarizedVector(Op: N->getVal());
1321 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SDLoc(N),
1322 MemVT: N->getMemoryVT().getVectorElementType(), Chain: N->getChain(),
1323 Ptr: ScalarVal, Val: N->getBasePtr(), MMO: N->getMemOperand());
1324}
1325
1326/// If the mask is a vector that needs to be scalarized, it must be <1 x i1>.
1327/// Promote it to a target boolean instead.
1328SDValue DAGTypeLegalizer::ScalarizeVecOp_MLOAD(MaskedLoadSDNode *N,
1329 unsigned OpNo) {
1330 assert(OpNo == 3 && "Can only scalarize the mask operand!");
1331 EVT DataVT = N->getValueType(ResNo: 0);
1332 SDValue Mask = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT);
1333 SmallVector<SDValue, 4> NewOps(N->ops());
1334 NewOps[OpNo] = Mask;
1335 SDNode *Res = DAG.UpdateNodeOperands(N, Ops: NewOps);
1336 if (Res == N)
1337 return SDValue(Res, 0);
1338
1339 // Update triggered CSE, do our own replacement since caller can't.
1340 ReplaceValueWith(From: SDValue(N, 0), To: SDValue(Res, 0));
1341 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res, 1));
1342 return SDValue();
1343}
1344
1345/// If the mask is a vector that needs to be scalarized, it must be <1 x i1>.
1346/// Promote it to a target boolean instead.
1347SDValue DAGTypeLegalizer::ScalarizeVecOp_MSTORE(MaskedStoreSDNode *N,
1348 unsigned OpNo) {
1349 assert(!N->isCompressingStore() && "CompressingStore not expected");
1350 if (OpNo != 4)
1351 reportFatalInternalError(reason: "Scalarization of MSTORE data is not supported");
1352 EVT DataVT = N->getValue().getValueType();
1353 SmallVector<SDValue, 4> NewOps(N->ops());
1354 NewOps[OpNo] = PromoteTargetBoolean(Bool: N->getOperand(Num: OpNo), ValVT: DataVT);
1355 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1356}
1357
1358/// If the value to round is a vector that needs to be scalarized, it must be
1359/// <1 x ty>. Convert the element instead.
1360SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1361 assert(OpNo == 0 && "Wrong operand for scalarization!");
1362 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1363 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1364 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1365 N2: N->getOperand(Num: 1));
1366 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1367}
1368
1369SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1370 unsigned OpNo) {
1371 assert(OpNo == 1 && "Wrong operand for scalarization!");
1372 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1373 SDValue Res =
1374 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1375 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1376 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1377 // Legalize the chain result - switch anything that used the old chain to
1378 // use the new one.
1379 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1380
1381 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1382
1383 // Do our own replacement and return SDValue() to tell the caller that we
1384 // handled all replacements since caller can only handle a single result.
1385 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1386 return SDValue();
1387}
1388
1389/// If the value to extend is a vector that needs to be scalarized, it must be
1390/// <1 x ty>. Convert the element instead.
1391SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1392 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1393 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1394 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1395 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1396}
1397
1398/// If the value to extend is a vector that needs to be scalarized, it must be
1399/// <1 x ty>. Convert the element instead.
1400SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1401 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1402 SDValue Res =
1403 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1404 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1405 Ops: {N->getOperand(Num: 0), Elt});
1406 // Legalize the chain result - switch anything that used the old chain to
1407 // use the new one.
1408 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1409
1410 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1411
1412 // Do our own replacement and return SDValue() to tell the caller that we
1413 // handled all replacements since caller can only handle a single result.
1414 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1415 return SDValue();
1416}
1417
1418SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1419 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1420 // Result type may be wider than element type.
1421 if (Res.getValueType() != N->getValueType(ResNo: 0))
1422 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1423 return Res;
1424}
1425
1426SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1427 SDValue AccOp = N->getOperand(Num: 0);
1428 SDValue VecOp = N->getOperand(Num: 1);
1429
1430 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1431
1432 SDValue Op = GetScalarizedVector(Op: VecOp);
1433 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1434 N1: AccOp, N2: Op, Flags: N->getFlags());
1435}
1436
1437SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1438 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1439 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1440
1441 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1442 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1443 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1444}
1445
1446SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1447 // Since there is no "none-active" result, the only valid return for <1 x ty>
1448 // is 0. Note: Since we check the high mask during splitting this is safe.
1449 // As e.g., a <2 x ty> operation would split to:
1450 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1451 // : `last_active(%lo_mask)`
1452 // Which then scalarizes to:
1453 // %mask[1] ? 1 : 0
1454 EVT VT = N->getValueType(ResNo: 0);
1455 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1456}
1457
1458SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1459 // The number of trailing zero elements is 1 if the element is 0, and 0
1460 // otherwise.
1461 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1462 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1463 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1464 SDValue SetCC =
1465 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1466 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1467 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1468}
1469
1470SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
1471 SDLoc DL(N);
1472 // Reuse the expansion (which should scalarize).
1473 SDValue Mask = TLI.expandVectorMatch(N, DAG);
1474 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL,
1475 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
1476 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1477}
1478
1479SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
1480 unsigned OpNo) {
1481 return TLI.expandVectorMatch(N, DAG);
1482}
1483
1484SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1485 assert(OpNo == 2 && "Can only scalarize mask operand");
1486 SDLoc DL(N);
1487 EVT VT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1488 SDValue LHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 0), Idx: 0);
1489 SDValue RHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 1), Idx: 0);
1490 SDValue Mask = GetScalarizedVector(Op: N->getOperand(Num: 2));
1491 // Vectors may have a different boolean contents to scalars, so truncate to i1
1492 // and let type legalization promote appropriately.
1493 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
1494 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1495 // use 1 as the divisor to avoid division by zero and overflow.
1496 SDValue BinOp =
1497 DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL, VT, N1: LHS,
1498 N2: DAG.getSelect(DL, VT, Cond: Mask, LHS: RHS, RHS: DAG.getConstant(Val: 1, DL, VT)));
1499 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: N->getValueType(ResNo: 0), Operand: BinOp);
1500}
1501
1502//===----------------------------------------------------------------------===//
1503// Result Vector Splitting
1504//===----------------------------------------------------------------------===//
1505
1506/// This method is called when the specified result of the specified node is
1507/// found to need vector splitting. At this point, the node may also have
1508/// invalid operands or may have other results that need legalization, we just
1509/// know that (at least) one result needs vector splitting.
1510void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1511 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1512 SDValue Lo, Hi;
1513
1514 // See if the target wants to custom expand this node.
1515 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1516 return;
1517
1518 switch (N->getOpcode()) {
1519 default:
1520#ifndef NDEBUG
1521 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1522 N->dump(&DAG);
1523 dbgs() << "\n";
1524#endif
1525 report_fatal_error(reason: "Do not know how to split the result of this "
1526 "operator!\n");
1527
1528 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1529 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1530 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1531 break;
1532 case ISD::MASK_BEFOREFIRST:
1533 SplitVecRes_MASK_BEFOREFIRST(N, Lo, Hi);
1534 break;
1535 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1536 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1537 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1538 case ISD::VSELECT:
1539 case ISD::SELECT:
1540 case ISD::VP_MERGE: SplitRes_Select(N, Lo, Hi); break;
1541 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1542 case ISD::POISON:
1543 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1544 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1545 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1546 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1547 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1548 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1549 case ISD::FPOWI:
1550 case ISD::FLDEXP:
1551 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1552 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1553 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1554 case ISD::SPLAT_VECTOR:
1555 case ISD::SCALAR_TO_VECTOR:
1556 SplitVecRes_ScalarOp(N, Lo, Hi);
1557 break;
1558 case ISD::STEP_VECTOR:
1559 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1560 break;
1561 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1562 case ISD::ATOMIC_LOAD:
1563 SplitVecRes_ATOMIC_LOAD(LD: cast<AtomicSDNode>(Val: N), Lo, Hi);
1564 break;
1565 case ISD::LOAD:
1566 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1567 break;
1568 case ISD::VP_LOAD:
1569 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1570 break;
1571 case ISD::VP_LOAD_FF:
1572 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1573 break;
1574 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1575 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1576 break;
1577 case ISD::MLOAD:
1578 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1579 break;
1580 case ISD::MGATHER:
1581 case ISD::VP_GATHER:
1582 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1583 break;
1584 case ISD::VECTOR_COMPRESS:
1585 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1586 break;
1587 case ISD::VECTOR_SHUFFLE_VAR:
1588 SplitVecRes_VECTOR_SHUFFLE_VAR(N, Lo, Hi);
1589 break;
1590 case ISD::SETCC:
1591 SplitVecRes_SETCC(N, Lo, Hi);
1592 break;
1593 case ISD::VECTOR_REPEAT:
1594 SplitVecRes_VECTOR_REPEAT(N, Lo, Hi);
1595 break;
1596 case ISD::VECTOR_REVERSE:
1597 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1598 break;
1599 case ISD::VECTOR_SHUFFLE:
1600 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1601 break;
1602 case ISD::VECTOR_SPLICE_LEFT:
1603 case ISD::VECTOR_SPLICE_RIGHT:
1604 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1605 break;
1606 case ISD::VECTOR_DEINTERLEAVE:
1607 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1608 return;
1609 case ISD::VECTOR_INTERLEAVE:
1610 SplitVecRes_VECTOR_INTERLEAVE(N);
1611 return;
1612 case ISD::VAARG:
1613 SplitVecRes_VAARG(N, Lo, Hi);
1614 break;
1615
1616 case ISD::ANY_EXTEND_VECTOR_INREG:
1617 case ISD::SIGN_EXTEND_VECTOR_INREG:
1618 case ISD::ZERO_EXTEND_VECTOR_INREG:
1619 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1620 break;
1621
1622 case ISD::ABS:
1623 case ISD::ABS_MIN_POISON:
1624 case ISD::BITREVERSE:
1625 case ISD::BSWAP:
1626 case ISD::CTLZ:
1627 case ISD::CTTZ:
1628 case ISD::CTLZ_ZERO_POISON:
1629 case ISD::CTTZ_ZERO_POISON:
1630 case ISD::CTPOP:
1631 case ISD::FABS:
1632 case ISD::FACOS:
1633 case ISD::FASIN:
1634 case ISD::FATAN:
1635 case ISD::FCEIL:
1636 case ISD::FCOS:
1637 case ISD::FCOSH:
1638 case ISD::FEXP:
1639 case ISD::FEXP2:
1640 case ISD::FEXP10:
1641 case ISD::FFLOOR:
1642 case ISD::FLOG:
1643 case ISD::FLOG10:
1644 case ISD::FLOG2:
1645 case ISD::FNEARBYINT:
1646 case ISD::FNEG:
1647 case ISD::FREEZE:
1648 case ISD::ARITH_FENCE:
1649 case ISD::FP_EXTEND:
1650 case ISD::FP_ROUND:
1651 case ISD::FP_TO_SINT:
1652 case ISD::FP_TO_UINT:
1653 case ISD::FRINT:
1654 case ISD::LRINT:
1655 case ISD::LLRINT:
1656 case ISD::FROUND:
1657 case ISD::FROUNDEVEN:
1658 case ISD::LROUND:
1659 case ISD::LLROUND:
1660 case ISD::FSIN:
1661 case ISD::FSINH:
1662 case ISD::FSQRT:
1663 case ISD::FTAN:
1664 case ISD::FTANH:
1665 case ISD::FTRUNC:
1666 case ISD::SINT_TO_FP:
1667 case ISD::TRUNCATE:
1668 case ISD::UINT_TO_FP:
1669 case ISD::FCANONICALIZE:
1670 case ISD::AssertNoFPClass:
1671 case ISD::CONVERT_FROM_ARBITRARY_FP:
1672 case ISD::CONVERT_TO_ARBITRARY_FP:
1673 SplitVecRes_UnaryOp(N, Lo, Hi);
1674 break;
1675 case ISD::ADDRSPACECAST:
1676 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1677 break;
1678 case ISD::FMODF:
1679 case ISD::FFREXP:
1680 case ISD::FSINCOS:
1681 case ISD::FSINCOSPI:
1682 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1683 break;
1684
1685 case ISD::ANY_EXTEND:
1686 case ISD::SIGN_EXTEND:
1687 case ISD::ZERO_EXTEND:
1688 SplitVecRes_ExtendOp(N, Lo, Hi);
1689 break;
1690
1691 case ISD::ADD:
1692 case ISD::SUB:
1693 case ISD::MUL:
1694 case ISD::CLMUL:
1695 case ISD::CLMULR:
1696 case ISD::CLMULH:
1697 case ISD::PEXT:
1698 case ISD::PDEP:
1699 case ISD::MULHS:
1700 case ISD::MULHU:
1701 case ISD::ABDS:
1702 case ISD::ABDU:
1703 case ISD::AVGCEILS:
1704 case ISD::AVGCEILU:
1705 case ISD::AVGFLOORS:
1706 case ISD::AVGFLOORU:
1707 case ISD::FADD:
1708 case ISD::FSUB:
1709 case ISD::FMUL:
1710 case ISD::FMINNUM:
1711 case ISD::FMINNUM_IEEE:
1712 case ISD::FMAXNUM:
1713 case ISD::FMAXNUM_IEEE:
1714 case ISD::FMINIMUM:
1715 case ISD::FMAXIMUM:
1716 case ISD::FMINIMUMNUM:
1717 case ISD::FMAXIMUMNUM:
1718 case ISD::SDIV: case ISD::VP_SDIV:
1719 case ISD::UDIV: case ISD::VP_UDIV:
1720 case ISD::FDIV:
1721 case ISD::FPOW:
1722 case ISD::FATAN2:
1723 case ISD::AND:
1724 case ISD::OR:
1725 case ISD::XOR:
1726 case ISD::SHL:
1727 case ISD::SRA:
1728 case ISD::SRL:
1729 case ISD::UREM: case ISD::VP_UREM:
1730 case ISD::SREM: case ISD::VP_SREM:
1731 case ISD::FREM:
1732 case ISD::SMIN:
1733 case ISD::SMAX:
1734 case ISD::UMIN:
1735 case ISD::UMAX:
1736 case ISD::SADDSAT:
1737 case ISD::UADDSAT:
1738 case ISD::SSUBSAT:
1739 case ISD::USUBSAT:
1740 case ISD::SSHLSAT:
1741 case ISD::USHLSAT:
1742 case ISD::ROTL:
1743 case ISD::ROTR:
1744 SplitVecRes_BinOp(N, Lo, Hi);
1745 break;
1746 case ISD::MASKED_UDIV:
1747 case ISD::MASKED_SDIV:
1748 case ISD::MASKED_UREM:
1749 case ISD::MASKED_SREM:
1750 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1751 break;
1752 case ISD::FMA:
1753 case ISD::FSHL:
1754 case ISD::FSHR:
1755 SplitVecRes_TernaryOp(N, Lo, Hi);
1756 break;
1757
1758 case ISD::SCMP: case ISD::UCMP:
1759 SplitVecRes_CMP(N, Lo, Hi);
1760 break;
1761
1762#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1763 case ISD::STRICT_##DAGN:
1764#include "llvm/IR/ConstrainedOps.def"
1765 SplitVecRes_StrictFPOp(N, Lo, Hi);
1766 break;
1767
1768 case ISD::FP_TO_UINT_SAT:
1769 case ISD::FP_TO_SINT_SAT:
1770 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1771 break;
1772
1773 case ISD::UADDO:
1774 case ISD::SADDO:
1775 case ISD::USUBO:
1776 case ISD::SSUBO:
1777 case ISD::UMULO:
1778 case ISD::SMULO:
1779 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1780 break;
1781 case ISD::SMULFIX:
1782 case ISD::SMULFIXSAT:
1783 case ISD::UMULFIX:
1784 case ISD::UMULFIXSAT:
1785 case ISD::SDIVFIX:
1786 case ISD::SDIVFIXSAT:
1787 case ISD::UDIVFIX:
1788 case ISD::UDIVFIXSAT:
1789 SplitVecRes_FIX(N, Lo, Hi);
1790 break;
1791 case ISD::EXPERIMENTAL_VP_SPLICE:
1792 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1793 break;
1794 case ISD::EXPERIMENTAL_VP_REVERSE:
1795 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1796 break;
1797 case ISD::PARTIAL_REDUCE_UMLA:
1798 case ISD::PARTIAL_REDUCE_SMLA:
1799 case ISD::PARTIAL_REDUCE_SUMLA:
1800 case ISD::PARTIAL_REDUCE_FMLA:
1801 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1802 break;
1803 case ISD::GET_ACTIVE_LANE_MASK:
1804 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1805 break;
1806 case ISD::VECTOR_MATCH:
1807 SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
1808 break;
1809 }
1810
1811 // If Lo/Hi is null, the sub-method took care of registering results etc.
1812 if (Lo.getNode())
1813 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1814}
1815
1816void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1817 MachinePointerInfo &MPI, SDValue &Ptr,
1818 uint64_t *ScaledOffset) {
1819 SDLoc DL(N);
1820 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1821
1822 if (MemVT.isScalableVector()) {
1823 SDValue BytesIncrement = DAG.getVScale(
1824 DL, VT: Ptr.getValueType(),
1825 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1826 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1827 if (ScaledOffset)
1828 *ScaledOffset += IncrementSize;
1829 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1830 Flags: SDNodeFlags::NoUnsignedWrap);
1831 } else {
1832 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1833 // Increment the pointer to the other half.
1834 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1835 }
1836}
1837
1838std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1839 return SplitMask(Mask, DL: SDLoc(Mask));
1840}
1841
1842std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1843 const SDLoc &DL) {
1844 SDValue MaskLo, MaskHi;
1845 EVT MaskVT = Mask.getValueType();
1846 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1847 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1848 else
1849 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1850 return std::make_pair(x&: MaskLo, y&: MaskHi);
1851}
1852
1853void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1854 SDValue LHSLo, LHSHi;
1855 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1856 SDValue RHSLo, RHSHi;
1857 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1858 SDLoc dl(N);
1859
1860 const SDNodeFlags Flags = N->getFlags();
1861 unsigned Opcode = N->getOpcode();
1862 if (N->getNumOperands() == 2) {
1863 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1864 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1865 return;
1866 }
1867
1868 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1869 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
1870 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
1871 "Expected VP opcode");
1872
1873 SDValue MaskLo, MaskHi;
1874 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1875
1876 SDValue EVLLo, EVLHi;
1877 std::tie(args&: EVLLo, args&: EVLHi) =
1878 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1879
1880 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1881 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1882 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1883 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1884}
1885
1886void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1887 SDValue &Hi) {
1888 SDValue LHSLo, LHSHi;
1889 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1890 SDValue RHSLo, RHSHi;
1891 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1892
1893 SDValue MaskLo, MaskHi, Mask = N->getOperand(Num: 2);
1894 if (Mask.getOpcode() == ISD::SETCC)
1895 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
1896 else
1897 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask);
1898
1899 SDLoc dl(N);
1900
1901 const SDNodeFlags Flags = N->getFlags();
1902 unsigned Opcode = N->getOpcode();
1903 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: MaskLo,
1904 Flags);
1905 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: MaskHi,
1906 Flags);
1907}
1908
1909void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1910 SDValue &Hi) {
1911 SDValue Op0Lo, Op0Hi;
1912 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1913 SDValue Op1Lo, Op1Hi;
1914 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1915 SDValue Op2Lo, Op2Hi;
1916 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1917 SDLoc dl(N);
1918
1919 const SDNodeFlags Flags = N->getFlags();
1920 unsigned Opcode = N->getOpcode();
1921 Lo =
1922 DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1923 Hi =
1924 DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1925}
1926
1927void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1928 LLVMContext &Ctxt = *DAG.getContext();
1929 SDLoc dl(N);
1930
1931 SDValue LHS = N->getOperand(Num: 0);
1932 SDValue RHS = N->getOperand(Num: 1);
1933
1934 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1935 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1936 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1937 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1938 } else {
1939 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1940 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1941 }
1942
1943 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1944 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1945 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1946}
1947
1948void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1949 SDValue LHSLo, LHSHi;
1950 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1951 SDValue RHSLo, RHSHi;
1952 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1953 SDLoc dl(N);
1954 SDValue Op2 = N->getOperand(Num: 2);
1955
1956 unsigned Opcode = N->getOpcode();
1957 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1958 Flags: N->getFlags());
1959 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1960 Flags: N->getFlags());
1961}
1962
1963void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1964 SDValue &Hi) {
1965 // We know the result is a vector. The input may be either a vector or a
1966 // scalar value.
1967 EVT LoVT, HiVT;
1968 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1969 SDLoc dl(N);
1970
1971 SDValue InOp = N->getOperand(Num: 0);
1972 EVT InVT = InOp.getValueType();
1973
1974 // Handle some special cases efficiently.
1975 switch (getTypeAction(VT: InVT)) {
1976 case TargetLowering::TypeLegal:
1977 case TargetLowering::TypePromoteInteger:
1978 case TargetLowering::TypeSoftPromoteHalf:
1979 case TargetLowering::TypeSoftenFloat:
1980 case TargetLowering::TypeScalarizeVector:
1981 case TargetLowering::TypeWidenVector:
1982 break;
1983 case TargetLowering::TypeExpandInteger:
1984 case TargetLowering::TypeExpandFloat:
1985 // A scalar to vector conversion, where the scalar needs expansion.
1986 // If the vector is being split in two then we can just convert the
1987 // expanded pieces.
1988 if (LoVT == HiVT) {
1989 GetExpandedOp(Op: InOp, Lo, Hi);
1990 if (DAG.getDataLayout().isBigEndian())
1991 std::swap(a&: Lo, b&: Hi);
1992 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1993 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1994 return;
1995 }
1996 break;
1997 case TargetLowering::TypeSplitVector:
1998 // If the input is a vector that needs to be split, convert each split
1999 // piece of the input now.
2000 GetSplitVector(Op: InOp, Lo, Hi);
2001 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
2002 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
2003 return;
2004 case TargetLowering::TypeScalarizeScalableVector:
2005 report_fatal_error(reason: "scalarization of scalable vectors is not supported");
2006 }
2007
2008 if (LoVT.isScalableVector()) {
2009 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
2010 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
2011 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
2012 return;
2013 }
2014
2015 // In the general case, convert the input to an integer and split it by hand.
2016 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2017 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2018 if (DAG.getDataLayout().isBigEndian())
2019 std::swap(a&: LoIntVT, b&: HiIntVT);
2020
2021 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
2022
2023 if (DAG.getDataLayout().isBigEndian())
2024 std::swap(a&: Lo, b&: Hi);
2025 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
2026 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
2027}
2028
2029void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
2030 SDValue &Hi) {
2031 SDLoc DL(N);
2032 EVT LoVT, HiVT;
2033 SDValue PtrA = N->getOperand(Num: 0);
2034 SDValue PtrB = N->getOperand(Num: 1);
2035 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2036
2037 // The lane offset for the "Lo" half of the mask is unchanged.
2038 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
2039 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
2040 /*LaneOffset=*/N4: N->getOperand(Num: 3));
2041 // The lane offset for the "Hi" half of the mask is incremented by the number
2042 // of elements in the "Lo" half.
2043 unsigned LaneOffset =
2044 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
2045 // Note: The lane offset is implicitly scalable for scalable masks.
2046 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
2047 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
2048 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
2049}
2050
2051void DAGTypeLegalizer::SplitVecRes_MASK_BEFOREFIRST(SDNode *N, SDValue &Lo,
2052 SDValue &Hi) {
2053 SDLoc DL(N);
2054 SDValue InLo, InHi;
2055 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
2056 EVT VT = InLo.getValueType();
2057 Lo = DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL, VT, Operand: InLo);
2058
2059 // hi = AnyLoActive ? all-zeros : (mask_beforefirst hi)
2060 SDValue AnyLoActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: InLo);
2061 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyLoActive, SL: DL,
2062 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
2063 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond, N2: DAG.getConstant(Val: 0, DL, VT),
2064 N3: DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL, VT, Operand: InHi));
2065}
2066
2067void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
2068 SDValue &Hi) {
2069 EVT LoVT, HiVT;
2070 SDLoc dl(N);
2071 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2072 unsigned LoNumElts = LoVT.getVectorNumElements();
2073 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
2074 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
2075
2076 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
2077 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
2078}
2079
2080void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
2081 SDValue &Hi) {
2082 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
2083 SDLoc dl(N);
2084 unsigned NumSubvectors = N->getNumOperands() / 2;
2085 if (NumSubvectors == 1) {
2086 Lo = N->getOperand(Num: 0);
2087 Hi = N->getOperand(Num: 1);
2088 return;
2089 }
2090
2091 EVT LoVT, HiVT;
2092 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2093
2094 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
2095 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
2096
2097 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
2098 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
2099}
2100
2101void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
2102 SDValue &Hi) {
2103 SDValue Vec = N->getOperand(Num: 0);
2104 SDValue Idx = N->getOperand(Num: 1);
2105 SDLoc dl(N);
2106
2107 EVT LoVT, HiVT;
2108 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2109
2110 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
2111 uint64_t IdxVal = Idx->getAsZExtVal();
2112 Hi = DAG.getNode(
2113 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
2114 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
2115}
2116
2117void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
2118 SDValue &Hi) {
2119 SDValue Vec = N->getOperand(Num: 0);
2120 SDValue SubVec = N->getOperand(Num: 1);
2121 SDValue Idx = N->getOperand(Num: 2);
2122 SDLoc dl(N);
2123 GetSplitVector(Op: Vec, Lo, Hi);
2124
2125 EVT VecVT = Vec.getValueType();
2126 EVT LoVT = Lo.getValueType();
2127 EVT SubVecVT = SubVec.getValueType();
2128 unsigned VecElems = VecVT.getVectorMinNumElements();
2129 unsigned SubElems = SubVecVT.getVectorMinNumElements();
2130 unsigned LoElems = LoVT.getVectorMinNumElements();
2131
2132 // If we know the index is in the first half, and we know the subvector
2133 // doesn't cross the boundary between the halves, we can avoid spilling the
2134 // vector, and insert into the lower half of the split vector directly.
2135 unsigned IdxVal = Idx->getAsZExtVal();
2136 if (IdxVal + SubElems <= LoElems) {
2137 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
2138 return;
2139 }
2140 // Similarly if the subvector is fully in the high half, but mind that we
2141 // can't tell whether a fixed-length subvector is fully within the high half
2142 // of a scalable vector.
2143 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
2144 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
2145 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
2146 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
2147 return;
2148 }
2149
2150 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
2151 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
2152 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
2153 if (WideSubVec.getValueType() == VecVT) {
2154 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
2155 return;
2156 }
2157 }
2158
2159 // Spill the vector to the stack.
2160 // In cases where the vector is illegal it will be broken down into parts
2161 // and stored in parts - we should use the alignment for the smallest part.
2162 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2163 SDValue StackPtr =
2164 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2165 auto &MF = DAG.getMachineFunction();
2166 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2167 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2168
2169 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2170 Alignment: SmallestAlign);
2171
2172 // Store the new subvector into the specified index.
2173 SDValue SubVecPtr =
2174 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
2175 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
2176 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2177
2178 // Load the Lo part from the stack slot.
2179 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
2180 Alignment: SmallestAlign);
2181
2182 // Increment the pointer to the other part.
2183 auto *Load = cast<LoadSDNode>(Val&: Lo);
2184 MachinePointerInfo MPI = Load->getPointerInfo();
2185 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2186
2187 // Load the Hi part from the stack slot.
2188 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2189}
2190
2191// Handle splitting an FP where the second operand does not match the first
2192// type. The second operand may be a scalar, or a vector that has exactly as
2193// many elements as the first
2194void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2195 SDValue &Hi) {
2196 SDValue LHSLo, LHSHi;
2197 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2198 SDLoc DL(N);
2199
2200 SDValue RHSLo, RHSHi;
2201 SDValue RHS = N->getOperand(Num: 1);
2202 EVT RHSVT = RHS.getValueType();
2203 if (RHSVT.isVector()) {
2204 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
2205 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
2206 else
2207 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
2208
2209 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
2210 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
2211 } else {
2212 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
2213 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
2214 }
2215}
2216
2217void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2218 SDValue &Hi) {
2219 SDLoc DL(N);
2220 SDValue ArgLo, ArgHi;
2221 SDValue Test = N->getOperand(Num: 1);
2222 SDValue FpValue = N->getOperand(Num: 0);
2223 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2224 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
2225 else
2226 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
2227 EVT LoVT, HiVT;
2228 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2229
2230 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
2231 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
2232}
2233
2234void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2235 SDValue &Hi) {
2236 SDValue LHSLo, LHSHi;
2237 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2238 SDLoc dl(N);
2239
2240 EVT LoVT, HiVT;
2241 std::tie(args&: LoVT, args&: HiVT) =
2242 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
2243
2244 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
2245 N2: DAG.getValueType(LoVT));
2246 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
2247 N2: DAG.getValueType(HiVT));
2248}
2249
2250void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2251 SDValue &Hi) {
2252 unsigned Opcode = N->getOpcode();
2253 SDValue N0 = N->getOperand(Num: 0);
2254
2255 SDLoc dl(N);
2256 SDValue InLo, InHi;
2257
2258 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
2259 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
2260 else
2261 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2262
2263 EVT InLoVT = InLo.getValueType();
2264 unsigned InNumElements = InLoVT.getVectorNumElements();
2265
2266 EVT OutLoVT, OutHiVT;
2267 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2268 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2269 assert((2 * OutNumElements) <= InNumElements &&
2270 "Illegal extend vector in reg split");
2271
2272 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2273 // input vector (i.e. we only use InLo):
2274 // OutLo will extend the first OutNumElements from InLo.
2275 // OutHi will extend the next OutNumElements from InLo.
2276
2277 // Shuffle the elements from InLo for OutHi into the bottom elements to
2278 // create a 'fake' InHi.
2279 SmallVector<int, 8> SplitHi(InNumElements, -1);
2280 for (unsigned i = 0; i != OutNumElements; ++i)
2281 SplitHi[i] = i + OutNumElements;
2282 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2283
2284 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2285 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2286}
2287
2288void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2289 SDValue &Hi) {
2290 unsigned NumOps = N->getNumOperands();
2291 SDValue Chain = N->getOperand(Num: 0);
2292 EVT LoVT, HiVT;
2293 SDLoc dl(N);
2294 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2295
2296 SmallVector<SDValue, 4> OpsLo(NumOps);
2297 SmallVector<SDValue, 4> OpsHi(NumOps);
2298
2299 // The Chain is the first operand.
2300 OpsLo[0] = Chain;
2301 OpsHi[0] = Chain;
2302
2303 // Now process the remaining operands.
2304 for (unsigned i = 1; i < NumOps; ++i) {
2305 SDValue Op = N->getOperand(Num: i);
2306 SDValue OpLo = Op;
2307 SDValue OpHi = Op;
2308
2309 EVT InVT = Op.getValueType();
2310 if (InVT.isVector()) {
2311 // If the input also splits, handle it directly for a
2312 // compile time speedup. Otherwise split it by hand.
2313 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2314 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2315 else
2316 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2317 }
2318
2319 OpsLo[i] = OpLo;
2320 OpsHi[i] = OpHi;
2321 }
2322
2323 EVT LoValueVTs[] = {LoVT, MVT::Other};
2324 EVT HiValueVTs[] = {HiVT, MVT::Other};
2325 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2326 Flags: N->getFlags());
2327 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2328 Flags: N->getFlags());
2329
2330 // Build a factor node to remember that this Op is independent of the
2331 // other one.
2332 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2333 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2334
2335 // Legalize the chain result - switch anything that used the old chain to
2336 // use the new one.
2337 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2338}
2339
2340SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2341 EVT VT = N->getValueType(ResNo: 0);
2342 if (VT.isScalableVector())
2343 report_fatal_error(reason: "Cannot unroll scalable vector strict-fp operation!");
2344
2345 SDValue Chain = N->getOperand(Num: 0);
2346 unsigned NE = VT.getVectorNumElements();
2347 EVT EltVT = VT.getVectorElementType();
2348 SDLoc dl(N);
2349
2350 SmallVector<SDValue, 8> Scalars;
2351 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2352
2353 // If ResNE is 0, fully unroll the vector op.
2354 if (ResNE == 0)
2355 ResNE = NE;
2356 else if (NE > ResNE)
2357 NE = ResNE;
2358
2359 //The results of each unrolled operation, including the chain.
2360 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2361 SmallVector<SDValue, 8> Chains;
2362
2363 unsigned i;
2364 for (i = 0; i != NE; ++i) {
2365 Operands[0] = Chain;
2366 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2367 SDValue Operand = N->getOperand(Num: j);
2368 EVT OperandVT = Operand.getValueType();
2369 if (OperandVT.isVector()) {
2370 EVT OperandEltVT = OperandVT.getVectorElementType();
2371 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2372 } else {
2373 Operands[j] = Operand;
2374 }
2375 }
2376 SDValue Scalar =
2377 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2378
2379 //Add in the scalar as well as its chain value to the
2380 //result vectors.
2381 Scalars.push_back(Elt: Scalar);
2382 Chains.push_back(Elt: Scalar.getValue(R: 1));
2383 }
2384
2385 for (; i < ResNE; ++i)
2386 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2387
2388 // Build a new factor node to connect the chain back together.
2389 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2390 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2391
2392 // Create a new BUILD_VECTOR node
2393 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2394 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2395}
2396
2397void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2398 SDValue &Lo, SDValue &Hi) {
2399 SDLoc dl(N);
2400 EVT ResVT = N->getValueType(ResNo: 0);
2401 EVT OvVT = N->getValueType(ResNo: 1);
2402 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2403 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2404 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2405
2406 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2407 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2408 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2409 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2410 } else {
2411 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2412 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2413 }
2414
2415 unsigned Opcode = N->getOpcode();
2416 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2417 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2418 SDNode *LoNode =
2419 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2420 SDNode *HiNode =
2421 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2422
2423 Lo = SDValue(LoNode, ResNo);
2424 Hi = SDValue(HiNode, ResNo);
2425
2426 // Replace the other vector result not being explicitly split here.
2427 unsigned OtherNo = 1 - ResNo;
2428 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2429 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2430 SetSplitVector(Op: SDValue(N, OtherNo),
2431 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2432 } else {
2433 SDValue OtherVal = DAG.getNode(
2434 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2435 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2436 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2437 }
2438}
2439
2440void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2441 SDValue &Hi) {
2442 SDValue Vec = N->getOperand(Num: 0);
2443 SDValue Elt = N->getOperand(Num: 1);
2444 SDValue Idx = N->getOperand(Num: 2);
2445 SDLoc dl(N);
2446 GetSplitVector(Op: Vec, Lo, Hi);
2447
2448 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2449 unsigned IdxVal = CIdx->getZExtValue();
2450 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2451 if (IdxVal < LoNumElts) {
2452 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2453 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2454 return;
2455 } else if (!Vec.getValueType().isScalableVector()) {
2456 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2457 return;
2458 }
2459 }
2460
2461 // Make the vector elements byte-addressable if they aren't already.
2462 EVT VecVT = Vec.getValueType();
2463 EVT EltVT = VecVT.getVectorElementType();
2464 if (!EltVT.isByteSized()) {
2465 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2466 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2467 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2468 // Extend the element type to match if needed.
2469 if (EltVT.bitsGT(VT: Elt.getValueType()))
2470 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2471 }
2472
2473 // Spill the vector to the stack.
2474 // In cases where the vector is illegal it will be broken down into parts
2475 // and stored in parts - we should use the alignment for the smallest part.
2476 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2477 SDValue StackPtr =
2478 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2479 auto &MF = DAG.getMachineFunction();
2480 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2481 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2482
2483 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2484 Alignment: SmallestAlign);
2485
2486 // Store the new element. This may be larger than the vector element type,
2487 // so use a truncating store.
2488 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2489 Store = DAG.getTruncStore(
2490 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2491 Alignment: commonAlignment(A: SmallestAlign,
2492 Offset: EltVT.getFixedSizeInBits() / 8));
2493
2494 EVT LoVT, HiVT;
2495 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2496
2497 // Load the Lo part from the stack slot.
2498 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2499
2500 // Increment the pointer to the other part.
2501 auto Load = cast<LoadSDNode>(Val&: Lo);
2502 MachinePointerInfo MPI = Load->getPointerInfo();
2503 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2504
2505 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2506
2507 // If we adjusted the original type, we need to truncate the results.
2508 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2509 if (LoVT != Lo.getValueType())
2510 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2511 if (HiVT != Hi.getValueType())
2512 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2513}
2514
2515void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2516 SDValue &Hi) {
2517 EVT LoVT, HiVT;
2518 SDLoc dl(N);
2519 assert(N->getValueType(0).isScalableVector() &&
2520 "Only scalable vectors are supported for STEP_VECTOR");
2521 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2522 SDValue Step = N->getOperand(Num: 0);
2523
2524 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2525
2526 // Hi = Lo + (EltCnt * Step)
2527 EVT EltVT = Step.getValueType();
2528 APInt StepVal = Step->getAsAPIntVal();
2529 SDValue StartOfHi =
2530 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2531 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2532 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2533
2534 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2535 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2536}
2537
2538void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2539 SDValue &Hi) {
2540 EVT LoVT, HiVT;
2541 SDLoc dl(N);
2542 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2543 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2544 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2545 Hi = DAG.getPOISON(VT: HiVT);
2546 } else {
2547 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2548 Hi = Lo;
2549 }
2550}
2551
2552void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2553 SDValue &Hi) {
2554 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2555 "Extended load during type legalization!");
2556 SDLoc dl(LD);
2557 EVT VT = LD->getValueType(ResNo: 0);
2558 EVT LoVT, HiVT;
2559 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2560
2561 SDValue Ch = LD->getChain();
2562 SDValue Ptr = LD->getBasePtr();
2563
2564 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2565 EVT MemIntVT =
2566 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LD->getMemoryVT().getSizeInBits());
2567 SDValue ALD = DAG.getAtomicLoad(ExtType: LD->getExtensionType(), dl, MemVT: MemIntVT, VT: IntVT,
2568 Chain: Ch, Ptr, MMO: LD->getMemOperand());
2569
2570 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2571 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2572 SDValue ExtractLo, ExtractHi;
2573 SplitInteger(Op: ALD, LoVT: LoIntVT, HiVT: HiIntVT, Lo&: ExtractLo, Hi&: ExtractHi);
2574
2575 Lo = DAG.getBitcast(VT: LoVT, V: ExtractLo);
2576 Hi = DAG.getBitcast(VT: HiVT, V: ExtractHi);
2577
2578 // Legalize the chain result - switch anything that used the old chain to
2579 // use the new one.
2580 ReplaceValueWith(From: SDValue(LD, 1), To: ALD.getValue(R: 1));
2581}
2582
2583void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2584 SDValue &Hi) {
2585 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2586 EVT LoVT, HiVT;
2587 SDLoc dl(LD);
2588 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2589
2590 ISD::LoadExtType ExtType = LD->getExtensionType();
2591 SDValue Ch = LD->getChain();
2592 SDValue Ptr = LD->getBasePtr();
2593 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
2594 EVT MemoryVT = LD->getMemoryVT();
2595 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2596 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
2597
2598 EVT LoMemVT, HiMemVT;
2599 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2600
2601 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2602 SDValue Value, NewChain;
2603 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2604 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2605 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2606 return;
2607 }
2608
2609 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2610 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2611 Metadata);
2612
2613 MachinePointerInfo MPI;
2614 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2615
2616 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2617 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
2618
2619 // Build a factor node to remember that this load is independent of the
2620 // other one.
2621 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2622 N2: Hi.getValue(R: 1));
2623
2624 // Legalize the chain result - switch anything that used the old chain to
2625 // use the new one.
2626 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2627}
2628
2629void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2630 SDValue &Hi) {
2631 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2632 EVT LoVT, HiVT;
2633 SDLoc dl(LD);
2634 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2635
2636 ISD::LoadExtType ExtType = LD->getExtensionType();
2637 SDValue Ch = LD->getChain();
2638 SDValue Ptr = LD->getBasePtr();
2639 SDValue Offset = LD->getOffset();
2640 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2641 Align Alignment = LD->getBaseAlign();
2642 SDValue Mask = LD->getMask();
2643 SDValue EVL = LD->getVectorLength();
2644 EVT MemoryVT = LD->getMemoryVT();
2645
2646 EVT LoMemVT, HiMemVT;
2647 bool HiIsEmpty = false;
2648 std::tie(args&: LoMemVT, args&: HiMemVT) =
2649 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2650
2651 // Split Mask operand
2652 SDValue MaskLo, MaskHi;
2653 if (Mask.getOpcode() == ISD::SETCC) {
2654 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2655 } else {
2656 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2657 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2658 else
2659 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2660 }
2661
2662 // Split EVL operand
2663 SDValue EVLLo, EVLHi;
2664 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2665
2666 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2667 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2668 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2669 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2670
2671 Lo =
2672 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2673 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2674
2675 if (HiIsEmpty) {
2676 // The hi vp_load has zero storage size. We therefore simply set it to
2677 // the low vp_load and rely on subsequent removal from the chain.
2678 Hi = Lo;
2679 } else {
2680 // Generate hi vp_load.
2681 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2682 IsCompressedMemory: LD->isExpandingLoad());
2683
2684 MachinePointerInfo MPI;
2685 if (LoMemVT.isScalableVector())
2686 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2687 else
2688 MPI = LD->getPointerInfo().getWithOffset(
2689 O: LoMemVT.getStoreSize().getFixedValue());
2690
2691 MMO = DAG.getMachineFunction().getMachineMemOperand(
2692 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2693 BaseAlignment: Alignment, Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2694
2695 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2696 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2697 IsExpanding: LD->isExpandingLoad());
2698 }
2699
2700 // Build a factor node to remember that this load is independent of the
2701 // other one.
2702 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2703 N2: Hi.getValue(R: 1));
2704
2705 // Legalize the chain result - switch anything that used the old chain to
2706 // use the new one.
2707 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2708}
2709
2710void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2711 SDValue &Hi) {
2712 SDLoc dl(LD);
2713 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2714
2715 SDValue Ch = LD->getChain();
2716 SDValue Ptr = LD->getBasePtr();
2717 Align Alignment = LD->getBaseAlign();
2718 SDValue Mask = LD->getMask();
2719 SDValue EVL = LD->getVectorLength();
2720
2721 // Split Mask operand
2722 SDValue MaskLo, MaskHi;
2723 if (Mask.getOpcode() == ISD::SETCC) {
2724 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2725 } else {
2726 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2727 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2728 else
2729 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2730 }
2731
2732 // Split EVL operand
2733 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2734
2735 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2736 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2737 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2738 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2739
2740 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2741
2742 // Fill the upper half with poison.
2743 Hi = DAG.getPOISON(VT: HiVT);
2744
2745 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2746 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2747}
2748
2749void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2750 SDValue &Lo, SDValue &Hi) {
2751 assert(SLD->isUnindexed() &&
2752 "Indexed VP strided load during type legalization!");
2753 assert(SLD->getOffset().isUndef() &&
2754 "Unexpected indexed variable-length load offset");
2755
2756 SDLoc DL(SLD);
2757
2758 EVT LoVT, HiVT;
2759 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2760
2761 EVT LoMemVT, HiMemVT;
2762 bool HiIsEmpty = false;
2763 std::tie(args&: LoMemVT, args&: HiMemVT) =
2764 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2765
2766 SDValue Mask = SLD->getMask();
2767 SDValue LoMask, HiMask;
2768 if (Mask.getOpcode() == ISD::SETCC) {
2769 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2770 } else {
2771 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2772 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2773 else
2774 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2775 }
2776
2777 SDValue LoEVL, HiEVL;
2778 std::tie(args&: LoEVL, args&: HiEVL) =
2779 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2780
2781 // Generate the low vp_strided_load
2782 Lo = DAG.getStridedLoadVP(
2783 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2784 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2785 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2786
2787 if (HiIsEmpty) {
2788 // The high vp_strided_load has zero storage size. We therefore simply set
2789 // it to the low vp_strided_load and rely on subsequent removal from the
2790 // chain.
2791 Hi = Lo;
2792 } else {
2793 // Generate the high vp_strided_load.
2794 // To calculate the high base address, we need to sum to the low base
2795 // address stride number of bytes for each element already loaded by low,
2796 // that is: Ptr = Ptr + (LoEVL * Stride)
2797 EVT PtrVT = SLD->getBasePtr().getValueType();
2798 SDValue Increment =
2799 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2800 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2801 SDValue Ptr =
2802 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2803
2804 Align Alignment = SLD->getBaseAlign();
2805 if (LoMemVT.isScalableVector())
2806 Alignment = commonAlignment(
2807 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2808
2809 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2810 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2811 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2812 BaseAlignment: Alignment, Metadata: MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
2813
2814 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2815 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2816 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2817 IsExpanding: SLD->isExpandingLoad());
2818 }
2819
2820 // Build a factor node to remember that this load is independent of the
2821 // other one.
2822 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2823 N2: Hi.getValue(R: 1));
2824
2825 // Legalize the chain result - switch anything that used the old chain to
2826 // use the new one.
2827 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2828}
2829
2830void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2831 SDValue &Lo, SDValue &Hi) {
2832 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2833 EVT LoVT, HiVT;
2834 SDLoc dl(MLD);
2835 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2836
2837 SDValue Ch = MLD->getChain();
2838 SDValue Ptr = MLD->getBasePtr();
2839 SDValue Offset = MLD->getOffset();
2840 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2841 SDValue Mask = MLD->getMask();
2842 SDValue PassThru = MLD->getPassThru();
2843 Align Alignment = MLD->getBaseAlign();
2844 ISD::LoadExtType ExtType = MLD->getExtensionType();
2845 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2846
2847 // Split Mask operand
2848 SDValue MaskLo, MaskHi;
2849 if (Mask.getOpcode() == ISD::SETCC) {
2850 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2851 } else {
2852 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2853 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2854 else
2855 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2856 }
2857
2858 EVT MemoryVT = MLD->getMemoryVT();
2859 EVT LoMemVT, HiMemVT;
2860 bool HiIsEmpty = false;
2861 std::tie(args&: LoMemVT, args&: HiMemVT) =
2862 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2863
2864 SDValue PassThruLo, PassThruHi;
2865 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2866 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2867 else
2868 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2869
2870 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2871 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2872 BaseAlignment: Alignment,
2873 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(), MLD->getMemCacheHint()));
2874
2875 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2876 MMO, AM: MLD->getAddressingMode(), ExtType,
2877 IsExpanding: MLD->isExpandingLoad());
2878
2879 if (HiIsEmpty) {
2880 // The hi masked load has zero storage size. We therefore simply set it to
2881 // the low masked load and rely on subsequent removal from the chain.
2882 Hi = Lo;
2883 } else {
2884 // Generate hi masked load.
2885 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2886 IsCompressedMemory: MLD->isExpandingLoad());
2887
2888 MachinePointerInfo MPI;
2889 if (LoMemVT.isScalableVector())
2890 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2891 else
2892 MPI = MLD->getPointerInfo().getWithOffset(
2893 O: LoMemVT.getStoreSize().getFixedValue());
2894
2895 MMO = DAG.getMachineFunction().getMachineMemOperand(
2896 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2897 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(),
2898 MLD->getMemCacheHint()));
2899
2900 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2901 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2902 IsExpanding: MLD->isExpandingLoad());
2903 }
2904
2905 // Build a factor node to remember that this load is independent of the
2906 // other one.
2907 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2908 N2: Hi.getValue(R: 1));
2909
2910 // Legalize the chain result - switch anything that used the old chain to
2911 // use the new one.
2912 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2913
2914}
2915
2916void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2917 SDValue &Hi, bool SplitSETCC) {
2918 EVT LoVT, HiVT;
2919 SDLoc dl(N);
2920 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2921
2922 SDValue Ch = N->getChain();
2923 SDValue Ptr = N->getBasePtr();
2924 struct Operands {
2925 SDValue Mask;
2926 SDValue Index;
2927 SDValue Scale;
2928 } Ops = [&]() -> Operands {
2929 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2930 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2931 }
2932 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2933 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2934 }();
2935
2936 EVT MemoryVT = N->getMemoryVT();
2937 Align Alignment = N->getBaseAlign();
2938
2939 // Split Mask operand
2940 SDValue MaskLo, MaskHi;
2941 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2942 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2943 } else {
2944 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2945 }
2946
2947 EVT LoMemVT, HiMemVT;
2948 // Split MemoryVT
2949 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2950
2951 SDValue IndexHi, IndexLo;
2952 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2953 TargetLowering::TypeSplitVector)
2954 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2955 else
2956 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2957
2958 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2959 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2960 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2961 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
2962
2963 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2964 SDValue PassThru = MGT->getPassThru();
2965 SDValue PassThruLo, PassThruHi;
2966 if (getTypeAction(VT: PassThru.getValueType()) ==
2967 TargetLowering::TypeSplitVector)
2968 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2969 else
2970 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2971
2972 ISD::LoadExtType ExtType = MGT->getExtensionType();
2973 ISD::MemIndexType IndexTy = MGT->getIndexType();
2974
2975 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2976 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2977 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2978
2979 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2980 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2981 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2982 } else {
2983 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2984 SDValue EVLLo, EVLHi;
2985 std::tie(args&: EVLLo, args&: EVLHi) =
2986 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2987
2988 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2989 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2990 MMO, IndexType: VPGT->getIndexType());
2991
2992 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2993 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2994 MMO, IndexType: VPGT->getIndexType());
2995 }
2996
2997 // Build a factor node to remember that this load is independent of the
2998 // other one.
2999 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
3000 N2: Hi.getValue(R: 1));
3001
3002 // Legalize the chain result - switch anything that used the old chain to
3003 // use the new one.
3004 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
3005}
3006
3007void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
3008 SDValue &Hi) {
3009 // This is not "trivial", as there is a dependency between the two subvectors.
3010 // Depending on the number of 1s in the mask, the elements from the Hi vector
3011 // need to be moved to the Lo vector. Passthru values make this even harder.
3012 // We try to use VECTOR_COMPRESS if the target has custom lowering with
3013 // smaller types and passthru is undef, as it is most likely faster than the
3014 // fully expand path. Otherwise, just do the full expansion as one "big"
3015 // operation and then extract the Lo and Hi vectors from that. This gets
3016 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
3017 SDLoc DL(N);
3018 EVT VecVT = N->getValueType(ResNo: 0);
3019
3020 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
3021 bool HasCustomLowering = false;
3022 EVT CheckVT = LoVT;
3023 while (CheckVT.getVectorMinNumElements() > 1) {
3024 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
3025 // custom lowering for illegal types. So we do the checks separately.
3026 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
3027 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
3028 HasCustomLowering = true;
3029 break;
3030 }
3031 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3032 }
3033
3034 SDValue Passthru = N->getOperand(Num: 2);
3035 if (!HasCustomLowering) {
3036 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
3037 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
3038 return;
3039 }
3040
3041 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
3042 SDValue Mask = N->getOperand(Num: 1);
3043 SDValue LoMask, HiMask;
3044 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3045 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
3046
3047 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
3048 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
3049 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
3050
3051 SDValue StackPtr = DAG.CreateStackTemporary(
3052 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
3053 MachineFunction &MF = DAG.getMachineFunction();
3054 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
3055 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
3056
3057 EVT LoMaskVT = LoMask.getValueType();
3058 assert(LoMaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
3059
3060 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
3061 EVT WideLoMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3062 EC: LoMaskVT.getVectorElementCount());
3063 SDValue WideLoMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideLoMaskVT, Operand: LoMask);
3064 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideLoMask);
3065 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
3066
3067 SDValue Chain = DAG.getEntryNode();
3068 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
3069 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
3070 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
3071
3072 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
3073 if (!Passthru.isUndef()) {
3074 // Compress the input mask so only inactive lanes of the result are replaced
3075 // by their passthrough value.
3076 EVT MaskVT = Mask.getValueType();
3077 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3078 EC: MaskVT.getVectorElementCount());
3079 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: Mask);
3080 SDValue NumActiveElts =
3081 DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
3082
3083 SDValue StepVector = DAG.getStepVector(DL, ResVT: WideMaskVT);
3084 SDValue SplatNumActiveElts = DAG.getSplat(VT: WideMaskVT, DL, Op: NumActiveElts);
3085 SDValue CompressedMask =
3086 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVector, RHS: SplatNumActiveElts, Cond: ISD::SETULT);
3087
3088 Compressed = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: CompressedMask,
3089 N2: Compressed, N3: Passthru);
3090 }
3091 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
3092}
3093
3094void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE_VAR(SDNode *N, SDValue &Lo,
3095 SDValue &Hi) {
3096 // TODO: Split into half-width shuffles when the target lowers them natively.
3097 SDValue Res = TLI.expandVECTOR_SHUFFLE_VAR(Node: N, DAG);
3098 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Res, DL: SDLoc(N));
3099}
3100
3101void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
3102 assert(N->getValueType(0).isVector() &&
3103 N->getOperand(0).getValueType().isVector() &&
3104 "Operand types must be vectors");
3105
3106 EVT LoVT, HiVT;
3107 SDLoc DL(N);
3108 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3109
3110 // If the input also splits, handle it directly. Otherwise split it by hand.
3111 SDValue LL, LH, RL, RH;
3112 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
3113 TargetLowering::TypeSplitVector)
3114 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
3115 else
3116 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
3117
3118 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
3119 TargetLowering::TypeSplitVector)
3120 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
3121 else
3122 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
3123
3124 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
3125 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
3126}
3127
3128void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
3129 SDValue &Hi) {
3130 // Get the dest types - they may not match the input types, e.g. int_to_fp.
3131 EVT LoVT, HiVT;
3132 SDLoc dl(N);
3133 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3134
3135 // If the input also splits, handle it directly for a compile time speedup.
3136 // Otherwise split it by hand.
3137 EVT InVT = N->getOperand(Num: 0).getValueType();
3138 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3139 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3140 else
3141 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3142
3143 const SDNodeFlags Flags = N->getFlags();
3144 unsigned Opcode = N->getOpcode();
3145 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
3146 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
3147 N4: N->getOperand(Num: 3), Flags);
3148 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
3149 N4: N->getOperand(Num: 3), Flags);
3150 return;
3151 }
3152
3153 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
3154 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
3155 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
3156 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
3157 } else {
3158 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
3159 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
3160 }
3161}
3162
3163void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
3164 SDValue &Hi) {
3165 SDLoc dl(N);
3166 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3167
3168 // If the input also splits, handle it directly for a compile time speedup.
3169 // Otherwise split it by hand.
3170 EVT InVT = N->getOperand(Num: 0).getValueType();
3171 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3172 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3173 else
3174 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3175
3176 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
3177 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3178 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3179 SDNodeFlags Flags = AddrSpaceCastN->getFlags();
3180 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS, Flags);
3181 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS, Flags);
3182}
3183
3184void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
3185 unsigned ResNo,
3186 SDValue &Lo,
3187 SDValue &Hi) {
3188 SDLoc dl(N);
3189 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3190 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
3191
3192 // If the input also splits, handle it directly for a compile time speedup.
3193 // Otherwise split it by hand.
3194 EVT InVT = N->getOperand(Num: 0).getValueType();
3195 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3196 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3197 else
3198 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3199
3200 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
3201 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
3202
3203 SDNode *HiNode = Hi.getNode();
3204 SDNode *LoNode = Lo.getNode();
3205
3206 // Replace the other vector result not being explicitly split here.
3207 unsigned OtherNo = 1 - ResNo;
3208 EVT OtherVT = N->getValueType(ResNo: OtherNo);
3209 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
3210 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
3211 Hi: SDValue(HiNode, OtherNo));
3212 } else {
3213 SDValue OtherVal =
3214 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
3215 N2: SDValue(HiNode, OtherNo));
3216 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
3217 }
3218}
3219
3220void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3221 SDValue &Hi) {
3222 SDLoc dl(N);
3223 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3224 EVT DestVT = N->getValueType(ResNo: 0);
3225 EVT LoVT, HiVT;
3226 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
3227
3228 // We can do better than a generic split operation if the extend is doing
3229 // more than just doubling the width of the elements and the following are
3230 // true:
3231 // - The number of vector elements is even,
3232 // - the source type is legal,
3233 // - the type of a split source is illegal,
3234 // - the type of an extended (by doubling element size) source is legal, and
3235 // - the type of that extended source when split is legal.
3236 //
3237 // This won't necessarily completely legalize the operation, but it will
3238 // more effectively move in the right direction and prevent falling down
3239 // to scalarization in many cases due to the input vector being split too
3240 // far.
3241 if (SrcVT.getVectorElementCount().isKnownEven() &&
3242 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3243 LLVMContext &Ctx = *DAG.getContext();
3244 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
3245 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
3246
3247 EVT SplitLoVT, SplitHiVT;
3248 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
3249 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
3250 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
3251 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3252 N->dump(&DAG); dbgs() << "\n");
3253 // Extend the source vector by one step.
3254 SDValue NewSrc =
3255 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
3256 // Get the low and high halves of the new, extended one step, vector.
3257 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3258 // Extend those vector halves the rest of the way.
3259 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
3260 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
3261 return;
3262 }
3263 }
3264 // Fall back to the generic unary operator splitting otherwise.
3265 SplitVecRes_UnaryOp(N, Lo, Hi);
3266}
3267
3268void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3269 SDValue &Lo, SDValue &Hi) {
3270 // The low and high parts of the original input give four input vectors.
3271 SDValue Inputs[4];
3272 SDLoc DL(N);
3273 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
3274 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
3275 EVT NewVT = Inputs[0].getValueType();
3276 unsigned NewElts = NewVT.getVectorNumElements();
3277
3278 auto &&IsConstant = [](const SDValue &N) {
3279 APInt SplatValue;
3280 return N.getResNo() == 0 && (ISD::isConstantSplatVector(V: N, SplatValue) ||
3281 ISD::isBuildVectorOfConstantSDNodes(V: N));
3282 };
3283 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3284 SDValue &Input2,
3285 ArrayRef<int> Mask) {
3286 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3287 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3288 "Expected build vector node.");
3289 EVT EltVT = NewVT.getVectorElementType();
3290 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
3291 for (unsigned I = 0; I < NewElts; ++I) {
3292 if (Mask[I] == PoisonMaskElem)
3293 continue;
3294 unsigned Idx = Mask[I];
3295 if (Idx >= NewElts)
3296 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3297 else
3298 Ops[I] = Input1.getOperand(i: Idx);
3299 // Make the type of all elements the same as the element type.
3300 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3301 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3302 }
3303 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3304 };
3305
3306 // If Lo or Hi uses elements from at most two of the four input vectors, then
3307 // express it as a vector shuffle of those two inputs. Otherwise extract the
3308 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3309 SmallVector<int> OrigMask(N->getMask());
3310 // Try to pack incoming shuffles/inputs.
3311 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3312 &DL](SmallVectorImpl<int> &Mask) {
3313 // Check if all inputs are shuffles of the same operands or non-shuffles.
3314 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3315 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3316 SDValue Input = Inputs[Idx];
3317 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3318 if (!Shuffle ||
3319 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3320 continue;
3321 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3322 .push_back(Elt: Idx);
3323 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3324 .push_back(Elt: Idx);
3325 }
3326 for (auto &P : ShufflesIdxs) {
3327 if (P.second.size() < 2)
3328 continue;
3329 // Use shuffles operands instead of shuffles themselves.
3330 // 1. Adjust mask.
3331 for (int &Idx : Mask) {
3332 if (Idx == PoisonMaskElem)
3333 continue;
3334 unsigned SrcRegIdx = Idx / NewElts;
3335 if (Inputs[SrcRegIdx].isUndef()) {
3336 Idx = PoisonMaskElem;
3337 continue;
3338 }
3339 auto *Shuffle =
3340 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3341 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3342 continue;
3343 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3344 if (MaskElt == PoisonMaskElem) {
3345 Idx = PoisonMaskElem;
3346 continue;
3347 }
3348 Idx = MaskElt % NewElts +
3349 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3350 ? 0
3351 : 1] *
3352 NewElts;
3353 }
3354 // 2. Update inputs.
3355 Inputs[P.second[0]] = P.first.first;
3356 Inputs[P.second[1]] = P.first.second;
3357 // Clear the pair data.
3358 P.second.clear();
3359 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3360 }
3361 // Check if any concat_vectors can be simplified.
3362 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3363 for (int &Idx : Mask) {
3364 if (Idx == PoisonMaskElem)
3365 continue;
3366 unsigned SrcRegIdx = Idx / NewElts;
3367 if (Inputs[SrcRegIdx].isUndef()) {
3368 Idx = PoisonMaskElem;
3369 continue;
3370 }
3371 TargetLowering::LegalizeTypeAction TypeAction =
3372 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3373 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3374 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3375 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3376 (TypeAction == TargetLowering::TypeLegal ||
3377 TypeAction == TargetLowering::TypeWidenVector))
3378 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3379 }
3380 if (UsedSubVector.count() > 1) {
3381 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3382 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3383 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3384 continue;
3385 if (Pairs.empty() || Pairs.back().size() == 2)
3386 Pairs.emplace_back();
3387 if (UsedSubVector.test(Idx: 2 * I)) {
3388 Pairs.back().emplace_back(Args&: I, Args: 0);
3389 } else {
3390 assert(UsedSubVector.test(2 * I + 1) &&
3391 "Expected to be used one of the subvectors.");
3392 Pairs.back().emplace_back(Args&: I, Args: 1);
3393 }
3394 }
3395 if (!Pairs.empty() && Pairs.front().size() > 1) {
3396 // Adjust mask.
3397 for (int &Idx : Mask) {
3398 if (Idx == PoisonMaskElem)
3399 continue;
3400 unsigned SrcRegIdx = Idx / NewElts;
3401 auto *It = find_if(
3402 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3403 return Idxs.front().first == SrcRegIdx ||
3404 Idxs.back().first == SrcRegIdx;
3405 });
3406 if (It == Pairs.end())
3407 continue;
3408 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3409 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3410 }
3411 // Adjust inputs.
3412 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3413 Inputs[Idxs.front().first] = DAG.getNode(
3414 Opcode: ISD::CONCAT_VECTORS, DL,
3415 VT: Inputs[Idxs.front().first].getValueType(),
3416 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3417 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3418 }
3419 }
3420 }
3421 bool Changed;
3422 do {
3423 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3424 // reused operands.
3425 Changed = false;
3426 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3427 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3428 if (!Shuffle)
3429 continue;
3430 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3431 continue;
3432 int Op = -1;
3433 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3434 !Shuffle->isSplat()) {
3435 Op = 0;
3436 } else if (!Inputs[I].hasOneUse() &&
3437 !Shuffle->getOperand(Num: 1).isUndef()) {
3438 // Find the only used operand, if possible.
3439 for (int &Idx : Mask) {
3440 if (Idx == PoisonMaskElem)
3441 continue;
3442 unsigned SrcRegIdx = Idx / NewElts;
3443 if (SrcRegIdx != I)
3444 continue;
3445 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3446 if (MaskElt == PoisonMaskElem) {
3447 Idx = PoisonMaskElem;
3448 continue;
3449 }
3450 int OpIdx = MaskElt / NewElts;
3451 if (Op == -1) {
3452 Op = OpIdx;
3453 continue;
3454 }
3455 if (Op != OpIdx) {
3456 Op = -1;
3457 break;
3458 }
3459 }
3460 }
3461 if (Op < 0) {
3462 // Try to check if one of the shuffle operands is used already.
3463 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3464 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3465 continue;
3466 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3467 if (It == std::end(arr&: Inputs))
3468 continue;
3469 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3470 // Found that operand is used already.
3471 // 1. Fix the mask for the reused operand.
3472 for (int &Idx : Mask) {
3473 if (Idx == PoisonMaskElem)
3474 continue;
3475 unsigned SrcRegIdx = Idx / NewElts;
3476 if (SrcRegIdx != I)
3477 continue;
3478 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3479 if (MaskElt == PoisonMaskElem) {
3480 Idx = PoisonMaskElem;
3481 continue;
3482 }
3483 int MaskIdx = MaskElt / NewElts;
3484 if (OpIdx == MaskIdx)
3485 Idx = MaskElt % NewElts + FoundOp * NewElts;
3486 }
3487 // 2. Set Op to the unused OpIdx.
3488 Op = (OpIdx + 1) % 2;
3489 break;
3490 }
3491 }
3492 if (Op >= 0) {
3493 Changed = true;
3494 Inputs[I] = Shuffle->getOperand(Num: Op);
3495 // Adjust mask.
3496 for (int &Idx : Mask) {
3497 if (Idx == PoisonMaskElem)
3498 continue;
3499 unsigned SrcRegIdx = Idx / NewElts;
3500 if (SrcRegIdx != I)
3501 continue;
3502 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3503 int OpIdx = MaskElt / NewElts;
3504 if (OpIdx != Op)
3505 continue;
3506 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3507 }
3508 }
3509 }
3510 } while (Changed);
3511 };
3512 TryPeekThroughShufflesInputs(OrigMask);
3513 // Proces unique inputs.
3514 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3515 NewElts](SmallVectorImpl<int> &Mask) {
3516 SetVector<SDValue> UniqueInputs;
3517 SetVector<SDValue> UniqueConstantInputs;
3518 for (const auto &I : Inputs) {
3519 if (IsConstant(I))
3520 UniqueConstantInputs.insert(X: I);
3521 else if (!I.isUndef())
3522 UniqueInputs.insert(X: I);
3523 }
3524 // Adjust mask in case of reused inputs. Also, need to insert constant
3525 // inputs at first, otherwise it affects the final outcome.
3526 if (UniqueInputs.size() != std::size(Inputs)) {
3527 auto &&UniqueVec = UniqueInputs.takeVector();
3528 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3529 unsigned ConstNum = UniqueConstantVec.size();
3530 for (int &Idx : Mask) {
3531 if (Idx == PoisonMaskElem)
3532 continue;
3533 unsigned SrcRegIdx = Idx / NewElts;
3534 if (Inputs[SrcRegIdx].isUndef()) {
3535 Idx = PoisonMaskElem;
3536 continue;
3537 }
3538 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3539 if (It != UniqueConstantVec.end()) {
3540 Idx = (Idx % NewElts) +
3541 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3542 assert(Idx >= 0 && "Expected defined mask idx.");
3543 continue;
3544 }
3545 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3546 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3547 Idx = (Idx % NewElts) +
3548 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3549 assert(Idx >= 0 && "Expected defined mask idx.");
3550 }
3551 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3552 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3553 }
3554 };
3555 MakeUniqueInputs(OrigMask);
3556 SDValue OrigInputs[4];
3557 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3558 for (unsigned High = 0; High < 2; ++High) {
3559 SDValue &Output = High ? Hi : Lo;
3560
3561 // Build a shuffle mask for the output, discovering on the fly which
3562 // input vectors to use as shuffle operands.
3563 unsigned FirstMaskIdx = High * NewElts;
3564 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3565 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3566 assert(!Output && "Expected default initialized initial value.");
3567 TryPeekThroughShufflesInputs(Mask);
3568 MakeUniqueInputs(Mask);
3569 SDValue TmpInputs[4];
3570 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3571 // Track changes in the output registers.
3572 int UsedIdx = -1;
3573 bool SecondIteration = false;
3574 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3575 if (UsedIdx < 0) {
3576 UsedIdx = Idx;
3577 return false;
3578 }
3579 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3580 SecondIteration = true;
3581 return SecondIteration;
3582 };
3583 processShuffleMasks(
3584 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3585 /*NumOfUsedRegs=*/1,
3586 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3587 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3588 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3589 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3590 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3591 else
3592 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3593 N2: DAG.getPOISON(VT: NewVT), Mask);
3594 Inputs[Idx] = Output;
3595 },
3596 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3597 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3598 unsigned Idx2, bool /*Unused*/) {
3599 if (AccumulateResults(Idx1)) {
3600 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3601 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3602 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3603 else
3604 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3605 N2: Inputs[Idx2], Mask);
3606 } else {
3607 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3608 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3609 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3610 else
3611 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3612 N2: TmpInputs[Idx2], Mask);
3613 }
3614 Inputs[Idx1] = Output;
3615 });
3616 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3617 }
3618}
3619
3620void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3621 EVT OVT = N->getValueType(ResNo: 0);
3622 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3623 SDValue Chain = N->getOperand(Num: 0);
3624 SDValue Ptr = N->getOperand(Num: 1);
3625 SDValue SV = N->getOperand(Num: 2);
3626 SDLoc dl(N);
3627
3628 const Align Alignment =
3629 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3630
3631 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3632 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3633 Chain = Hi.getValue(R: 1);
3634
3635 // Modified the chain - switch anything that used the old chain to use
3636 // the new one.
3637 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3638}
3639
3640void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3641 SDValue &Hi) {
3642 EVT DstVTLo, DstVTHi;
3643 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3644 SDLoc dl(N);
3645
3646 SDValue SrcLo, SrcHi;
3647 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3648 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3649 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3650 else
3651 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3652
3653 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3654 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3655}
3656
3657void DAGTypeLegalizer::SplitVecRes_VECTOR_REPEAT(SDNode *N, SDValue &Lo,
3658 SDValue &Hi) {
3659 EVT VT = N->getValueType(ResNo: 0);
3660 SDValue Src = N->getOperand(Num: 0);
3661 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
3662 assert(LoVT == HiVT && "Expected equal split types");
3663
3664 // Use smaller even/odd source vectors so their broadcasts can be
3665 // reinterleaved in the original lane order for every value of vscale.
3666 SDLoc DL(N);
3667 auto [SrcLo, SrcHi] = DAG.SplitVector(N: Src, DL);
3668 EVT SplitSrcVT = SrcLo.getValueType();
3669 SDValue Deinterleaved =
3670 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
3671 VTList: DAG.getVTList(VT1: SplitSrcVT, VT2: SplitSrcVT), N1: SrcLo, N2: SrcHi);
3672 SDValue Even =
3673 DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: LoVT, Operand: Deinterleaved.getValue(R: 0));
3674 SDValue Odd =
3675 DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: LoVT, Operand: Deinterleaved.getValue(R: 1));
3676 SDValue Interleaved = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
3677 VTList: DAG.getVTList(VT1: LoVT, VT2: LoVT), N1: Even, N2: Odd);
3678 Lo = Interleaved.getValue(R: 0);
3679 Hi = Interleaved.getValue(R: 1);
3680}
3681
3682void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3683 SDValue &Hi) {
3684 SDValue InLo, InHi;
3685 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3686 SDLoc DL(N);
3687
3688 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3689 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3690}
3691
3692void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3693 SDValue &Hi) {
3694 SDLoc DL(N);
3695
3696 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3697 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3698}
3699
3700void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3701 SDValue &Hi) {
3702 EVT VT = N->getValueType(ResNo: 0);
3703 SDValue Val = N->getOperand(Num: 0);
3704 SDValue Mask = N->getOperand(Num: 1);
3705 SDValue EVL = N->getOperand(Num: 2);
3706 SDLoc DL(N);
3707
3708 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3709 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3710 // then truncate back.
3711 EVT OrigVT = VT;
3712 if (!VT.getVectorElementType().isByteSized()) {
3713 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3714 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3715 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3716 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Val);
3717 }
3718
3719 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3720 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3721
3722 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3723 EC: VT.getVectorElementCount());
3724 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3725 EVT PtrVT = StackPtr.getValueType();
3726 auto &MF = DAG.getMachineFunction();
3727 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3728 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3729
3730 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3731 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3732 BaseAlignment: Alignment);
3733 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3734 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3735 BaseAlignment: Alignment);
3736
3737 EVL = DAG.getNode(
3738 Opcode: ISD::UMIN, DL, VT: EVL.getValueType(), N1: DAG.getFreeze(V: EVL),
3739 N2: DAG.getElementCount(DL, VT: EVL.getValueType(), EC: VT.getVectorElementCount()));
3740
3741 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3742 SDValue NumElemMinus1 =
3743 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3744 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3745 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3746 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3747 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3748 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3749
3750 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3751 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3752 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3753 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3754
3755 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3756
3757 // Truncate back if we widened above.
3758 if (OrigVT != VT)
3759 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3760
3761 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3762}
3763
3764void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3765 SDValue &Hi) {
3766 EVT VT = N->getValueType(ResNo: 0);
3767 SDValue V1 = N->getOperand(Num: 0);
3768 SDValue V2 = N->getOperand(Num: 1);
3769 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3770 SDValue Mask = N->getOperand(Num: 3);
3771 SDValue EVL1 = N->getOperand(Num: 4);
3772 SDValue EVL2 = N->getOperand(Num: 5);
3773 SDLoc DL(N);
3774
3775 // Since EVL2 is considered the real VL it gets promoted during
3776 // SelectionDAGBuilder. Promote EVL1 here if needed.
3777 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3778 EVL1 = ZExtPromotedInteger(Op: EVL1);
3779
3780 // The stack splice addresses elements by byte offset/stride, which breaks for
3781 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3782 // stride is 0. Widen to a byte integer, splice, then truncate back.
3783 EVT OrigVT = VT;
3784 if (!VT.getVectorElementType().isByteSized()) {
3785 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3786 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3787 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3788 V1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V1);
3789 V2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V2);
3790 }
3791
3792 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3793
3794 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3795 EC: VT.getVectorElementCount() * 2);
3796 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3797 EVT PtrVT = StackPtr.getValueType();
3798 auto &MF = DAG.getMachineFunction();
3799 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3800 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3801
3802 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3803 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3804 BaseAlignment: Alignment);
3805 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3806 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3807 BaseAlignment: Alignment);
3808
3809 SDValue EltByteSize =
3810 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
3811 SDValue EVL1Ptr = DAG.getZExtOrTrunc(Op: DAG.getFreeze(V: EVL1), DL, VT: PtrVT);
3812 SDValue EVL1Bytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: EVL1Ptr, N2: EltByteSize);
3813 // Clip EVL1Bytes to make sure we stay within the stack object.
3814 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
3815 EVL1Bytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: EVL1Bytes, N2: VTBytes);
3816 SDValue StackPtr2 = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: EVL1Bytes, DL);
3817 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3818
3819 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3820 SDValue StoreV1 =
3821 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3822 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3823
3824 SDValue StoreV2 =
3825 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3826 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3827
3828 SDValue Load;
3829 if (Imm >= 0) {
3830 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3831 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3832 } else {
3833 uint64_t TrailingElts = -Imm;
3834 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3835 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3836
3837 // Make sure TrailingBytes doesn't exceed the size of vec1.
3838 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3839 TrailingBytes =
3840 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3841
3842 // Calculate the start address of the spliced result.
3843 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3844 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3845 }
3846
3847 // Truncate back if we widened above.
3848 if (OrigVT != VT)
3849 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3850
3851 EVT LoVT, HiVT;
3852 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: OrigVT);
3853 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3854 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3855 Hi =
3856 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3857 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3858}
3859
3860void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3861 SDValue &Hi) {
3862 SDLoc DL(N);
3863 SDValue Acc = N->getOperand(Num: 0);
3864 SDValue Input1 = N->getOperand(Num: 1);
3865 SDValue Input2 = N->getOperand(Num: 2);
3866
3867 SDValue AccLo, AccHi;
3868 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3869 unsigned Opcode = N->getOpcode();
3870
3871 // If the input types don't need splitting, just accumulate into the
3872 // low part of the accumulator.
3873 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3874 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3875 Hi = AccHi;
3876 return;
3877 }
3878
3879 SDValue Input1Lo, Input1Hi;
3880 SDValue Input2Lo, Input2Hi;
3881 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3882 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3883 EVT ResultVT = AccLo.getValueType();
3884
3885 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3886 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3887}
3888
3889void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3890 SDValue &Hi) {
3891 SDLoc DL(N);
3892 SDValue Op0 = N->getOperand(Num: 0);
3893 SDValue Op1 = N->getOperand(Num: 1);
3894 EVT OpVT = Op0.getValueType();
3895
3896 EVT LoVT, HiVT;
3897 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3898
3899 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3900 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3901 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3902 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3903}
3904
3905void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
3906 SDValue &Hi) {
3907 SDValue SourceLo, SourceHi;
3908 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SourceLo, Hi&: SourceHi);
3909 SDValue MaskLo, MaskHi;
3910 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
3911 SDLoc DL(N);
3912
3913 Lo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskLo.getValueType(), N1: SourceLo,
3914 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
3915 Hi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskHi.getValueType(), N1: SourceHi,
3916 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
3917}
3918
3919void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3920 unsigned Factor = N->getNumOperands();
3921
3922 SmallVector<SDValue, 8> Ops(Factor * 2);
3923 for (unsigned i = 0; i != Factor; ++i) {
3924 SDValue OpLo, OpHi;
3925 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3926 Ops[i * 2] = OpLo;
3927 Ops[i * 2 + 1] = OpHi;
3928 }
3929
3930 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3931
3932 SDLoc DL(N);
3933 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3934 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3935 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3936 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3937
3938 for (unsigned i = 0; i != Factor; ++i)
3939 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3940}
3941
3942void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3943 unsigned Factor = N->getNumOperands();
3944
3945 SmallVector<SDValue, 8> Ops(Factor * 2);
3946 for (unsigned i = 0; i != Factor; ++i) {
3947 SDValue OpLo, OpHi;
3948 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3949 Ops[i] = OpLo;
3950 Ops[i + Factor] = OpHi;
3951 }
3952
3953 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3954
3955 SDLoc DL(N);
3956 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3957 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3958 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3959 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3960
3961 for (unsigned i = 0; i != Factor; ++i) {
3962 unsigned IdxLo = 2 * i;
3963 unsigned IdxHi = 2 * i + 1;
3964 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3965 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3966 }
3967}
3968
3969//===----------------------------------------------------------------------===//
3970// Operand Vector Splitting
3971//===----------------------------------------------------------------------===//
3972
3973/// This method is called when the specified operand of the specified node is
3974/// found to need vector splitting. At this point, all of the result types of
3975/// the node are known to be legal, but other operands of the node may need
3976/// legalization as well as the specified one.
3977bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3978 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3979 SDValue Res = SDValue();
3980
3981 // See if the target wants to custom split this node.
3982 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3983 return false;
3984
3985 switch (N->getOpcode()) {
3986 default:
3987#ifndef NDEBUG
3988 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3989 N->dump(&DAG);
3990 dbgs() << "\n";
3991#endif
3992 report_fatal_error(reason: "Do not know how to split this operator's "
3993 "operand!\n");
3994
3995 case ISD::STRICT_FSETCC:
3996 case ISD::STRICT_FSETCCS:
3997 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3998 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3999 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
4000 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
4001 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
4002 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
4003 case ISD::VECTOR_FIND_LAST_ACTIVE:
4004 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
4005 break;
4006 case ISD::TRUNCATE:
4007 Res = SplitVecOp_TruncateHelper(N);
4008 break;
4009 case ISD::STRICT_FP_ROUND:
4010 case ISD::FP_ROUND:
4011 case ISD::CONVERT_FROM_ARBITRARY_FP:
4012 case ISD::CONVERT_TO_ARBITRARY_FP:
4013 Res = SplitVecOp_FP_ROUND(N);
4014 break;
4015 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
4016 case ISD::STORE:
4017 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
4018 break;
4019 case ISD::ATOMIC_STORE:
4020 Res = SplitVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
4021 break;
4022 case ISD::VP_STORE:
4023 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
4024 break;
4025 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
4026 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
4027 break;
4028 case ISD::MSTORE:
4029 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
4030 break;
4031 case ISD::MSCATTER:
4032 case ISD::VP_SCATTER:
4033 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
4034 break;
4035 case ISD::MGATHER:
4036 case ISD::VP_GATHER:
4037 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
4038 break;
4039 case ISD::VSELECT:
4040 Res = SplitVecOp_VSELECT(N, OpNo);
4041 break;
4042 case ISD::MASKED_UDIV:
4043 case ISD::MASKED_SDIV:
4044 case ISD::MASKED_UREM:
4045 case ISD::MASKED_SREM:
4046 Res = SplitVecOp_MaskedBinOp(N, OpNo);
4047 break;
4048 case ISD::VECTOR_COMPRESS:
4049 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
4050 break;
4051 case ISD::VECTOR_SHUFFLE_VAR:
4052 Res = SplitVecOp_VECTOR_SHUFFLE_VAR(N, OpNo);
4053 break;
4054 case ISD::STRICT_SINT_TO_FP:
4055 case ISD::STRICT_UINT_TO_FP:
4056 case ISD::SINT_TO_FP:
4057 case ISD::UINT_TO_FP:
4058 if (N->getValueType(ResNo: 0).bitsLT(
4059 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
4060 Res = SplitVecOp_TruncateHelper(N);
4061 else
4062 Res = SplitVecOp_UnaryOp(N);
4063 break;
4064 case ISD::FP_TO_SINT_SAT:
4065 case ISD::FP_TO_UINT_SAT:
4066 Res = SplitVecOp_FP_TO_XINT_SAT(N);
4067 break;
4068 case ISD::FP_TO_SINT:
4069 case ISD::FP_TO_UINT:
4070 case ISD::STRICT_FP_TO_SINT:
4071 case ISD::STRICT_FP_TO_UINT:
4072 case ISD::STRICT_FP_EXTEND:
4073 case ISD::FP_EXTEND:
4074 case ISD::SIGN_EXTEND:
4075 case ISD::ZERO_EXTEND:
4076 case ISD::ANY_EXTEND:
4077 case ISD::FTRUNC:
4078 case ISD::LROUND:
4079 case ISD::LLROUND:
4080 case ISD::LRINT:
4081 case ISD::LLRINT:
4082 Res = SplitVecOp_UnaryOp(N);
4083 break;
4084 case ISD::FLDEXP:
4085 Res = SplitVecOp_FPOpDifferentTypes(N);
4086 break;
4087
4088 case ISD::SCMP:
4089 case ISD::UCMP:
4090 Res = SplitVecOp_CMP(N);
4091 break;
4092
4093 case ISD::FAKE_USE:
4094 Res = SplitVecOp_FAKE_USE(N);
4095 break;
4096 case ISD::ANY_EXTEND_VECTOR_INREG:
4097 case ISD::SIGN_EXTEND_VECTOR_INREG:
4098 case ISD::ZERO_EXTEND_VECTOR_INREG:
4099 Res = SplitVecOp_ExtVecInRegOp(N);
4100 break;
4101
4102 case ISD::VECREDUCE_FADD:
4103 case ISD::VECREDUCE_FMUL:
4104 case ISD::VECREDUCE_ADD:
4105 case ISD::VECREDUCE_MUL:
4106 case ISD::VECREDUCE_AND:
4107 case ISD::VECREDUCE_OR:
4108 case ISD::VECREDUCE_XOR:
4109 case ISD::VECREDUCE_SMAX:
4110 case ISD::VECREDUCE_SMIN:
4111 case ISD::VECREDUCE_UMAX:
4112 case ISD::VECREDUCE_UMIN:
4113 case ISD::VECREDUCE_FMAX:
4114 case ISD::VECREDUCE_FMIN:
4115 case ISD::VECREDUCE_FMAXIMUM:
4116 case ISD::VECREDUCE_FMINIMUM:
4117 case ISD::VECREDUCE_FMAXIMUMNUM:
4118 case ISD::VECREDUCE_FMINIMUMNUM:
4119 Res = SplitVecOp_VECREDUCE(N, OpNo);
4120 break;
4121 case ISD::VECREDUCE_SEQ_FADD:
4122 case ISD::VECREDUCE_SEQ_FMUL:
4123 Res = SplitVecOp_VECREDUCE_SEQ(N);
4124 break;
4125 case ISD::VP_REDUCE_FADD:
4126 case ISD::VP_REDUCE_SEQ_FADD:
4127 case ISD::VP_REDUCE_FMUL:
4128 case ISD::VP_REDUCE_SEQ_FMUL:
4129 case ISD::VP_REDUCE_ADD:
4130 case ISD::VP_REDUCE_MUL:
4131 case ISD::VP_REDUCE_AND:
4132 case ISD::VP_REDUCE_OR:
4133 case ISD::VP_REDUCE_XOR:
4134 case ISD::VP_REDUCE_SMAX:
4135 case ISD::VP_REDUCE_SMIN:
4136 case ISD::VP_REDUCE_UMAX:
4137 case ISD::VP_REDUCE_UMIN:
4138 case ISD::VP_REDUCE_FMAX:
4139 case ISD::VP_REDUCE_FMIN:
4140 case ISD::VP_REDUCE_FMAXIMUM:
4141 case ISD::VP_REDUCE_FMINIMUM:
4142 Res = SplitVecOp_VP_REDUCE(N, OpNo);
4143 break;
4144 case ISD::CTTZ_ELTS:
4145 case ISD::CTTZ_ELTS_ZERO_POISON:
4146 Res = SplitVecOp_CttzElts(N);
4147 break;
4148 case ISD::VP_CTTZ_ELTS:
4149 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4150 Res = SplitVecOp_VP_CttzElements(N);
4151 break;
4152 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
4153 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
4154 break;
4155 case ISD::PARTIAL_REDUCE_UMLA:
4156 case ISD::PARTIAL_REDUCE_SMLA:
4157 case ISD::PARTIAL_REDUCE_SUMLA:
4158 case ISD::PARTIAL_REDUCE_FMLA:
4159 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
4160 break;
4161 case ISD::VECTOR_MATCH:
4162 Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
4163 break;
4164 }
4165
4166 // If the result is null, the sub-method took care of registering results etc.
4167 if (!Res.getNode()) return false;
4168
4169 // If the result is N, the sub-method updated N in place. Tell the legalizer
4170 // core about this.
4171 if (Res.getNode() == N)
4172 return true;
4173
4174 if (N->isStrictFPOpcode())
4175 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4176 "Invalid operand expansion");
4177 else
4178 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4179 "Invalid operand expansion");
4180
4181 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4182 return false;
4183}
4184
4185SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
4186 SDLoc DL(N);
4187
4188 SDValue LoMask, HiMask;
4189 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
4190
4191 EVT VT = N->getValueType(ResNo: 0);
4192 EVT SplitVT = LoMask.getValueType();
4193 ElementCount SplitEC = SplitVT.getVectorElementCount();
4194
4195 // Find the last active in both the low and the high masks.
4196 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
4197 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
4198
4199 // Check if any lane is active in the high mask.
4200 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
4201 // sentinel value for "none active".
4202 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
4203 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
4204 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
4205
4206 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
4207 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
4208 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
4209 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
4210 N3: LoFind);
4211}
4212
4213SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
4214 // The only possibility for an illegal operand is the mask, since result type
4215 // legalization would have handled this node already otherwise.
4216 assert(OpNo == 0 && "Illegal operand must be mask");
4217
4218 SDValue Mask = N->getOperand(Num: 0);
4219 SDValue Src0 = N->getOperand(Num: 1);
4220 SDValue Src1 = N->getOperand(Num: 2);
4221 EVT Src0VT = Src0.getValueType();
4222 SDLoc DL(N);
4223 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4224
4225 SDValue Lo, Hi;
4226 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4227 assert(Lo.getValueType() == Hi.getValueType() &&
4228 "Lo and Hi have differing types");
4229
4230 EVT LoOpVT, HiOpVT;
4231 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
4232 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4233
4234 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4235 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
4236 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
4237 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4238
4239 SDValue LoSelect =
4240 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
4241 SDValue HiSelect =
4242 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
4243
4244 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
4245}
4246
4247SDValue DAGTypeLegalizer::SplitVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
4248 assert(OpNo == 2 && "Illegal operand must be mask");
4249
4250 SDLoc DL(N);
4251 auto [LHSLo, LHSHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
4252 auto [RHSLo, RHSHi] = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4253 SDValue MaskLo, MaskHi;
4254 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
4255
4256 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo,
4257 N2: RHSLo, N3: MaskLo, Flags: N->getFlags());
4258 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi,
4259 N2: RHSHi, N3: MaskHi, Flags: N->getFlags());
4260 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4261}
4262
4263SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4264 // The only possibility for an illegal operand is the mask, since result type
4265 // legalization would have handled this node already otherwise.
4266 assert(OpNo == 1 && "Illegal operand must be mask");
4267
4268 // To split the mask, we need to split the result type too, so we can just
4269 // reuse that logic here.
4270 SDValue Lo, Hi;
4271 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4272
4273 EVT VecVT = N->getValueType(ResNo: 0);
4274 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
4275}
4276
4277SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_SHUFFLE_VAR(SDNode *N,
4278 unsigned OpNo) {
4279 // The only possibility for an illegal operand is the mask, since result type
4280 // legalization would have handled this node already otherwise.
4281 assert(OpNo == 1 && "Illegal operand must be mask");
4282 // TODO: Legalize the mask type instead once a target lowers the node.
4283 return TLI.expandVECTOR_SHUFFLE_VAR(Node: N, DAG);
4284}
4285
4286SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4287 EVT ResVT = N->getValueType(ResNo: 0);
4288 SDValue Lo, Hi;
4289 SDLoc dl(N);
4290
4291 SDValue VecOp = N->getOperand(Num: OpNo);
4292 EVT VecVT = VecOp.getValueType();
4293 assert(VecVT.isVector() && "Can only split reduce vector operand");
4294 GetSplitVector(Op: VecOp, Lo, Hi);
4295 EVT LoOpVT, HiOpVT;
4296 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4297
4298 // Use the appropriate scalar instruction on the split subvectors before
4299 // reducing the now partially reduced smaller vector.
4300 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
4301 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
4302 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
4303}
4304
4305SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4306 EVT ResVT = N->getValueType(ResNo: 0);
4307 SDValue Lo, Hi;
4308 SDLoc dl(N);
4309
4310 SDValue AccOp = N->getOperand(Num: 0);
4311 SDValue VecOp = N->getOperand(Num: 1);
4312 SDNodeFlags Flags = N->getFlags();
4313
4314 EVT VecVT = VecOp.getValueType();
4315 assert(VecVT.isVector() && "Can only split reduce vector operand");
4316 GetSplitVector(Op: VecOp, Lo, Hi);
4317 EVT LoOpVT, HiOpVT;
4318 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4319
4320 // Reduce low half.
4321 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
4322
4323 // Reduce high half, using low half result as initial value.
4324 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
4325}
4326
4327SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4328 assert(N->isVPOpcode() && "Expected VP opcode");
4329 assert(OpNo == 1 && "Can only split reduce vector operand");
4330
4331 unsigned Opc = N->getOpcode();
4332 EVT ResVT = N->getValueType(ResNo: 0);
4333 SDValue Lo, Hi;
4334 SDLoc dl(N);
4335
4336 SDValue VecOp = N->getOperand(Num: OpNo);
4337 EVT VecVT = VecOp.getValueType();
4338 assert(VecVT.isVector() && "Can only split reduce vector operand");
4339 GetSplitVector(Op: VecOp, Lo, Hi);
4340
4341 SDValue MaskLo, MaskHi;
4342 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
4343
4344 SDValue EVLLo, EVLHi;
4345 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
4346
4347 const SDNodeFlags Flags = N->getFlags();
4348
4349 SDValue ResLo =
4350 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
4351 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
4352}
4353
4354SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4355 // The result has a legal vector type, but the input needs splitting.
4356 EVT ResVT = N->getValueType(ResNo: 0);
4357 SDValue Lo, Hi;
4358 SDLoc dl(N);
4359 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4360 EVT InVT = Lo.getValueType();
4361
4362 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4363 EC: InVT.getVectorElementCount());
4364
4365 if (N->isStrictFPOpcode()) {
4366 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4367 Ops: {N->getOperand(Num: 0), Lo});
4368 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4369 Ops: {N->getOperand(Num: 0), Hi});
4370
4371 // Build a factor node to remember that this operation is independent
4372 // of the other one.
4373 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4374 N2: Hi.getValue(R: 1));
4375
4376 // Legalize the chain result - switch anything that used the old chain to
4377 // use the new one.
4378 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4379 } else {
4380 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
4381 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
4382 }
4383
4384 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4385}
4386
4387// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4388SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4389 SDValue Lo, Hi;
4390 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4391 SDValue Chain =
4392 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
4393 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
4394}
4395
4396SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4397 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4398 // end up being split all the way down to individual components. Convert the
4399 // split pieces into integers and reassemble.
4400 EVT ResVT = N->getValueType(ResNo: 0);
4401 SDValue Lo, Hi;
4402 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4403 SDLoc dl(N);
4404
4405 if (ResVT.isScalableVector()) {
4406 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4407 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4408 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4409 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4410 }
4411
4412 Lo = BitConvertToInteger(Op: Lo);
4413 Hi = BitConvertToInteger(Op: Hi);
4414
4415 if (DAG.getDataLayout().isBigEndian())
4416 std::swap(a&: Lo, b&: Hi);
4417
4418 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4419}
4420
4421SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4422 unsigned OpNo) {
4423 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4424 // We know that the result type is legal.
4425 EVT ResVT = N->getValueType(ResNo: 0);
4426
4427 SDValue Vec = N->getOperand(Num: 0);
4428 SDValue SubVec = N->getOperand(Num: 1);
4429 SDValue Idx = N->getOperand(Num: 2);
4430 SDLoc dl(N);
4431
4432 SDValue Lo, Hi;
4433 GetSplitVector(Op: SubVec, Lo, Hi);
4434
4435 uint64_t IdxVal = Idx->getAsZExtVal();
4436 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4437
4438 SDValue FirstInsertion =
4439 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4440 SDValue SecondInsertion =
4441 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4442 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4443
4444 return SecondInsertion;
4445}
4446
4447SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4448 // We know that the extracted result type is legal.
4449 EVT SubVT = N->getValueType(ResNo: 0);
4450 SDValue Idx = N->getOperand(Num: 1);
4451 SDLoc dl(N);
4452 SDValue Lo, Hi;
4453
4454 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4455
4456 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4457 // Note: For scalable vectors, the index is scaled by vscale.
4458 ElementCount IdxVal =
4459 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4460 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4461
4462 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4463 ElementCount NumResultElts = SubVT.getVectorElementCount();
4464
4465 // If the extracted elements are all in the low half, do a simple extract.
4466 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4467 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4468
4469 unsigned LoEltsMin = LoElts.getKnownMinValue();
4470 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4471 SrcVT.isFixedLengthVector()) {
4472 // Extracted subvector crosses vector split, so we need to blend the two
4473 // halves.
4474 // TODO: May be able to emit partial extract_subvector.
4475 SmallVector<SDValue, 8> Elts;
4476 Elts.reserve(N: NumResultElts.getFixedValue());
4477
4478 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4479 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4480 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4481 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4482 /*Count=*/LoEltsMin - IdxValMin);
4483 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4484 /*Count=*/SubVT.getVectorNumElements() -
4485 Elts.size());
4486 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4487 }
4488
4489 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4490 ElementCount ExtractIdx = IdxVal - LoElts;
4491 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4492 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4493 Idx: ExtractIdx.getKnownMinValue());
4494
4495 EVT HiVT = Hi.getValueType();
4496 assert(HiVT.isFixedLengthVector() &&
4497 "Only fixed-vector extracts are supported in this case");
4498
4499 // We cannot create an extract_subvector that isn't a multiple of the
4500 // result size, which may go out of bounds for the last elements. Shuffle
4501 // the desired elements down to 0 and do a simple 0 extract.
4502 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4503 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4504 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4505
4506 SDValue Shuffle =
4507 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4508 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4509 }
4510
4511 // After this point the DAG node only permits extracting fixed-width
4512 // subvectors from scalable vectors.
4513 assert(SubVT.isFixedLengthVector() &&
4514 "Extracting scalable subvector from fixed-width unsupported");
4515
4516 // If the element type is i1 and we're not promoting the result, then we may
4517 // end up loading the wrong data since the bits are packed tightly into
4518 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4519 // type at index 4, then we will load a byte starting at index 0.
4520 if (SubVT.getScalarType() == MVT::i1)
4521 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4522 "subvector from a scalable predicate vector");
4523
4524 // Spill the vector to the stack. We should use the alignment for
4525 // the smallest part.
4526 SDValue Vec = N->getOperand(Num: 0);
4527 EVT VecVT = Vec.getValueType();
4528 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4529 SDValue StackPtr =
4530 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4531 auto &MF = DAG.getMachineFunction();
4532 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4533 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4534
4535 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4536 Alignment: SmallestAlign);
4537
4538 // Extract the subvector by loading the correct part.
4539 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4540
4541 return DAG.getLoad(
4542 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4543 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4544}
4545
4546SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4547 SDValue Vec = N->getOperand(Num: 0);
4548 SDValue Idx = N->getOperand(Num: 1);
4549 EVT VecVT = Vec.getValueType();
4550
4551 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4552 uint64_t IdxVal = Index->getZExtValue();
4553
4554 SDValue Lo, Hi;
4555 GetSplitVector(Op: Vec, Lo, Hi);
4556
4557 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4558
4559 if (IdxVal < LoElts)
4560 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4561 else if (!Vec.getValueType().isScalableVector())
4562 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4563 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4564 VT: Idx.getValueType())), 0);
4565 }
4566
4567 // See if the target wants to custom expand this node.
4568 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4569 return SDValue();
4570
4571 // Make the vector elements byte-addressable if they aren't already.
4572 SDLoc dl(N);
4573 EVT EltVT = VecVT.getVectorElementType();
4574 if (!EltVT.isByteSized()) {
4575 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4576 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4577 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4578 SDValue NewExtract =
4579 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4580 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4581 }
4582
4583 // Store the vector to the stack.
4584 // In cases where the vector is illegal it will be broken down into parts
4585 // and stored in parts - we should use the alignment for the smallest part.
4586 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4587 SDValue StackPtr =
4588 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4589 auto &MF = DAG.getMachineFunction();
4590 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4591 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4592 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4593 Alignment: SmallestAlign);
4594
4595 // Load back the required element.
4596 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4597
4598 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4599 // type, leaving the high bits undefined. But it can't truncate.
4600 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4601
4602 return DAG.getExtLoad(
4603 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4604 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4605 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4606}
4607
4608SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4609 SDValue Lo, Hi;
4610
4611 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4612 // splitting the result has the same effect as splitting the input operand.
4613 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4614
4615 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4616}
4617
4618SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4619 (void)OpNo;
4620 SDValue Lo, Hi;
4621 SplitVecRes_Gather(N, Lo, Hi);
4622
4623 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4624 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4625 return SDValue();
4626}
4627
4628SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4629 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4630 SDValue Ch = N->getChain();
4631 SDValue Ptr = N->getBasePtr();
4632 SDValue Offset = N->getOffset();
4633 assert(Offset.isUndef() && "Unexpected VP store offset");
4634 SDValue Mask = N->getMask();
4635 SDValue EVL = N->getVectorLength();
4636 SDValue Data = N->getValue();
4637 Align Alignment = N->getBaseAlign();
4638 SDLoc DL(N);
4639
4640 SDValue DataLo, DataHi;
4641 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4642 // Split Data operand
4643 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4644 else
4645 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4646
4647 // Split Mask operand
4648 SDValue MaskLo, MaskHi;
4649 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4650 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4651 } else {
4652 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4653 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4654 else
4655 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4656 }
4657
4658 EVT MemoryVT = N->getMemoryVT();
4659 EVT LoMemVT, HiMemVT;
4660 bool HiIsEmpty = false;
4661 std::tie(args&: LoMemVT, args&: HiMemVT) =
4662 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4663
4664 // Split EVL
4665 SDValue EVLLo, EVLHi;
4666 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4667
4668 SDValue Lo, Hi;
4669 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4670 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4671 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4672 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4673
4674 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4675 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4676 IsCompressing: N->isCompressingStore());
4677
4678 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4679 if (HiIsEmpty)
4680 return Lo;
4681
4682 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4683 IsCompressedMemory: N->isCompressingStore());
4684
4685 MachinePointerInfo MPI;
4686 if (LoMemVT.isScalableVector()) {
4687 Alignment = commonAlignment(A: Alignment,
4688 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4689 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4690 } else
4691 MPI = N->getPointerInfo().getWithOffset(
4692 O: LoMemVT.getStoreSize().getFixedValue());
4693
4694 MMO = DAG.getMachineFunction().getMachineMemOperand(
4695 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4696 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4697
4698 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4699 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4700 IsCompressing: N->isCompressingStore());
4701
4702 // Build a factor node to remember that this store is independent of the
4703 // other one.
4704 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4705}
4706
4707SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4708 unsigned OpNo) {
4709 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4710 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4711
4712 SDLoc DL(N);
4713
4714 SDValue Data = N->getValue();
4715 SDValue LoData, HiData;
4716 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4717 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4718 else
4719 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4720
4721 EVT LoMemVT, HiMemVT;
4722 bool HiIsEmpty = false;
4723 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4724 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4725
4726 SDValue Mask = N->getMask();
4727 SDValue LoMask, HiMask;
4728 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4729 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4730 else if (getTypeAction(VT: Mask.getValueType()) ==
4731 TargetLowering::TypeSplitVector)
4732 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4733 else
4734 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4735
4736 SDValue LoEVL, HiEVL;
4737 std::tie(args&: LoEVL, args&: HiEVL) =
4738 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4739
4740 // Generate the low vp_strided_store
4741 SDValue Lo = DAG.getStridedStoreVP(
4742 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4743 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4744 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4745
4746 // If the high vp_strided_store has zero storage size, only the low
4747 // vp_strided_store is needed.
4748 if (HiIsEmpty)
4749 return Lo;
4750
4751 // Generate the high vp_strided_store.
4752 // To calculate the high base address, we need to sum to the low base
4753 // address stride number of bytes for each element already stored by low,
4754 // that is: Ptr = Ptr + (LoEVL * Stride)
4755 EVT PtrVT = N->getBasePtr().getValueType();
4756 SDValue Increment =
4757 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4758 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4759 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4760
4761 Align Alignment = N->getBaseAlign();
4762 if (LoMemVT.isScalableVector())
4763 Alignment = commonAlignment(A: Alignment,
4764 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4765
4766 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4767 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4768 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4769 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4770
4771 SDValue Hi = DAG.getStridedStoreVP(
4772 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4773 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4774 IsCompressing: N->isCompressingStore());
4775
4776 // Build a factor node to remember that this store is independent of the
4777 // other one.
4778 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4779}
4780
4781SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4782 unsigned OpNo) {
4783 assert(N->isUnindexed() && "Indexed masked store of vector?");
4784 SDValue Ch = N->getChain();
4785 SDValue Ptr = N->getBasePtr();
4786 SDValue Offset = N->getOffset();
4787 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4788 SDValue Mask = N->getMask();
4789 SDValue Data = N->getValue();
4790 Align Alignment = N->getBaseAlign();
4791 SDLoc DL(N);
4792
4793 SDValue DataLo, DataHi;
4794 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4795 // Split Data operand
4796 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4797 else
4798 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4799
4800 // Split Mask operand
4801 SDValue MaskLo, MaskHi;
4802 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4803 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4804 } else {
4805 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4806 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4807 else
4808 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4809 }
4810
4811 EVT MemoryVT = N->getMemoryVT();
4812 EVT LoMemVT, HiMemVT;
4813 bool HiIsEmpty = false;
4814 std::tie(args&: LoMemVT, args&: HiMemVT) =
4815 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4816
4817 SDValue Lo, Hi, Res;
4818 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4819 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4820 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4821 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4822
4823 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4824 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4825 IsCompressing: N->isCompressingStore());
4826
4827 if (HiIsEmpty) {
4828 // The hi masked store has zero storage size.
4829 // Only the lo masked store is needed.
4830 Res = Lo;
4831 } else {
4832
4833 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4834 IsCompressedMemory: N->isCompressingStore());
4835
4836 MachinePointerInfo MPI;
4837 if (LoMemVT.isScalableVector()) {
4838 Alignment = commonAlignment(
4839 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4840 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4841 } else
4842 MPI = N->getPointerInfo().getWithOffset(
4843 O: LoMemVT.getStoreSize().getFixedValue());
4844
4845 MMO = DAG.getMachineFunction().getMachineMemOperand(
4846 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4847 BaseAlignment: Alignment,
4848 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4849
4850 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4851 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4852 IsCompressing: N->isCompressingStore());
4853
4854 // Build a factor node to remember that this store is independent of the
4855 // other one.
4856 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4857 }
4858
4859 return Res;
4860}
4861
4862SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4863 SDValue Ch = N->getChain();
4864 SDValue Ptr = N->getBasePtr();
4865 EVT MemoryVT = N->getMemoryVT();
4866 Align Alignment = N->getBaseAlign();
4867 SDLoc DL(N);
4868 struct Operands {
4869 SDValue Mask;
4870 SDValue Index;
4871 SDValue Scale;
4872 SDValue Data;
4873 } Ops = [&]() -> Operands {
4874 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4875 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4876 .Data: MSC->getValue()};
4877 }
4878 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4879 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4880 .Data: VPSC->getValue()};
4881 }();
4882 // Split all operands
4883
4884 EVT LoMemVT, HiMemVT;
4885 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4886
4887 SDValue DataLo, DataHi;
4888 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4889 // Split Data operand
4890 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4891 else
4892 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4893
4894 // Split Mask operand
4895 SDValue MaskLo, MaskHi;
4896 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4897 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4898 } else {
4899 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4900 }
4901
4902 SDValue IndexHi, IndexLo;
4903 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4904 TargetLowering::TypeSplitVector)
4905 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4906 else
4907 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4908
4909 SDValue Lo;
4910 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4911 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4912 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4913 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4914
4915 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4916 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4917 Lo =
4918 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4919 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4920
4921 // The order of the Scatter operation after split is well defined. The "Hi"
4922 // part comes after the "Lo". So these two operations should be chained one
4923 // after another.
4924 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4925 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4926 MMO, IndexType: MSC->getIndexType(),
4927 IsTruncating: MSC->isTruncatingStore());
4928 }
4929 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4930 SDValue EVLLo, EVLHi;
4931 std::tie(args&: EVLLo, args&: EVLHi) =
4932 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4933
4934 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4935 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4936 IndexType: VPSC->getIndexType());
4937
4938 // The order of the Scatter operation after split is well defined. The "Hi"
4939 // part comes after the "Lo". So these two operations should be chained one
4940 // after another.
4941 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4942 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4943 IndexType: VPSC->getIndexType());
4944}
4945
4946SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4947 assert(N->isUnindexed() && "Indexed store of vector?");
4948 assert(OpNo == 1 && "Can only split the stored value");
4949 SDLoc DL(N);
4950
4951 bool isTruncating = N->isTruncatingStore();
4952 SDValue Ch = N->getChain();
4953 SDValue Ptr = N->getBasePtr();
4954 EVT MemoryVT = N->getMemoryVT();
4955 Align Alignment = N->getBaseAlign();
4956 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4957 MMOMetadata Metadata = N->getMMOMetadataForSubAccess();
4958 SDValue Lo, Hi;
4959 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4960
4961 EVT LoMemVT, HiMemVT;
4962 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4963
4964 // Scalarize if the split halves are not byte-sized.
4965 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4966 return TLI.scalarizeVectorStore(ST: N, DAG);
4967
4968 if (isTruncating)
4969 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4970 Alignment, MMOFlags, Metadata);
4971 else
4972 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4973 Metadata);
4974
4975 MachinePointerInfo MPI;
4976 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4977
4978 if (isTruncating)
4979 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, SVT: HiMemVT, Alignment, MMOFlags,
4980 Metadata);
4981 else
4982 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, Metadata);
4983
4984 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4985}
4986
4987SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4988 SDLoc DL(N);
4989 LLVMContext &Ctx = *DAG.getContext();
4990 SDValue StVal = N->getVal();
4991 EVT VT = StVal.getValueType();
4992 EVT MemIntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: N->getMemoryVT().getSizeInBits());
4993
4994 // The store needs a single value spanning the full memory width. If the
4995 // value can be held in a legal vector register, keep it there and extract
4996 // the low integer element of the memory width. This lets the store be issued
4997 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4998 // bitcasting the split vector straight to a scalar integer, which would
4999 // reassemble the value element by element in GPRs.
5000 //
5001 // Reinterpret the value as a same-shaped integer vector first: an FP element
5002 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
5003 // integer-of-element-size form does. Ask the target which legal vector type
5004 // it widens to.
5005 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
5006 EVT IntEltVT = IntVecVT.getVectorElementType();
5007 EVT WideVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT: IntVecVT);
5008 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(VT: MemIntVT) &&
5009 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
5010 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
5011 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
5012 SDValue Wide = ModifyToType(InOp: DAG.getBitcast(VT: IntVecVT, V: StVal), NVT: WideVT);
5013 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
5014 EVT MemVecVT = EVT::getVectorVT(Context&: Ctx, VT: MemIntVT, NumElements: NumMemElts);
5015 SDValue Elt = DAG.getExtractVectorElt(DL, VT: MemIntVT,
5016 Vec: DAG.getBitcast(VT: MemVecVT, V: Wide), Idx: 0);
5017 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: Elt,
5018 Val: N->getBasePtr(), MMO: N->getMemOperand());
5019 }
5020
5021 // Otherwise issue a single atomic store of an integer that spans the full
5022 // memory width. Bitcasting the (illegal) vector value to that integer lets
5023 // the type legalizer further legalize the BITCAST input as needed, while the
5024 // ATOMIC_STORE itself uses only the legal integer type.
5025 EVT IntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits());
5026 SDValue AsInt = DAG.getBitcast(VT: IntVT, V: StVal);
5027 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: AsInt,
5028 Val: N->getBasePtr(), MMO: N->getMemOperand());
5029}
5030
5031SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
5032 SDLoc DL(N);
5033
5034 // The input operands all must have the same type, and we know the result
5035 // type is valid. Convert this to a buildvector which extracts all the
5036 // input elements.
5037 // TODO: If the input elements are power-two vectors, we could convert this to
5038 // a new CONCAT_VECTORS node with elements that are half-wide.
5039 SmallVector<SDValue, 32> Elts;
5040 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
5041 for (const SDValue &Op : N->op_values()) {
5042 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
5043 i != e; ++i) {
5044 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
5045 }
5046 }
5047
5048 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
5049}
5050
5051SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
5052 // The result type is legal, but the input type is illegal. If splitting
5053 // ends up with the result type of each half still being legal, just
5054 // do that. If, however, that would result in an illegal result type,
5055 // we can try to get more clever with power-two vectors. Specifically,
5056 // split the input type, but also widen the result element size, then
5057 // concatenate the halves and truncate again. For example, consider a target
5058 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
5059 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
5060 // %inlo = v4i32 extract_subvector %in, 0
5061 // %inhi = v4i32 extract_subvector %in, 4
5062 // %lo16 = v4i16 trunc v4i32 %inlo
5063 // %hi16 = v4i16 trunc v4i32 %inhi
5064 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
5065 // %res = v8i8 trunc v8i16 %in16
5066 //
5067 // Without this transform, the original truncate would end up being
5068 // scalarized, which is pretty much always a last resort.
5069 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
5070 SDValue InVec = N->getOperand(Num: OpNo);
5071 EVT InVT = InVec->getValueType(ResNo: 0);
5072 EVT OutVT = N->getValueType(ResNo: 0);
5073 ElementCount NumElements = OutVT.getVectorElementCount();
5074 bool IsFloat = OutVT.isFloatingPoint();
5075
5076 unsigned InElementSize = InVT.getScalarSizeInBits();
5077 unsigned OutElementSize = OutVT.getScalarSizeInBits();
5078
5079 // Determine the split output VT. If its legal we can just split dirctly.
5080 EVT LoOutVT, HiOutVT;
5081 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
5082 assert(LoOutVT == HiOutVT && "Unequal split?");
5083
5084 // If the input elements are only 1/2 the width of the result elements,
5085 // just use the normal splitting. Our trick only work if there's room
5086 // to split more than once.
5087 if (isTypeLegal(VT: LoOutVT) || InElementSize <= OutElementSize * 2 ||
5088 (IsFloat && !isPowerOf2_32(Value: InElementSize)))
5089 return SplitVecOp_UnaryOp(N);
5090 SDLoc DL(N);
5091
5092 // Don't touch if this will be scalarized.
5093 EVT FinalVT = InVT;
5094 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
5095 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
5096
5097 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
5098 return SplitVecOp_UnaryOp(N);
5099
5100 // Get the split input vector.
5101 SDValue InLoVec, InHiVec;
5102 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
5103
5104 // Truncate them to 1/2 the element size.
5105 //
5106 // This assumes the number of elements is a power of two; any vector that
5107 // isn't should be widened, not split.
5108 EVT HalfElementVT = IsFloat ?
5109 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
5110 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
5111 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
5112 EC: NumElements.divideCoefficientBy(RHS: 2));
5113
5114 SDValue HalfLo;
5115 SDValue HalfHi;
5116 SDValue Chain;
5117 if (N->isStrictFPOpcode()) {
5118 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
5119 Ops: {N->getOperand(Num: 0), InLoVec});
5120 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
5121 Ops: {N->getOperand(Num: 0), InHiVec});
5122 // Legalize the chain result - switch anything that used the old chain to
5123 // use the new one.
5124 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
5125 N2: HalfHi.getValue(R: 1));
5126 } else {
5127 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
5128 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
5129 }
5130
5131 // Concatenate them to get the full intermediate truncation result.
5132 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
5133 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
5134 N2: HalfHi);
5135 // Now finish up by truncating all the way down to the original result
5136 // type. This should normally be something that ends up being legal directly,
5137 // but in theory if a target has very wide vectors and an annoyingly
5138 // restricted set of legal types, this split can chain to build things up.
5139
5140 if (N->isStrictFPOpcode()) {
5141 SDValue Res = DAG.getNode(
5142 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
5143 Ops: {Chain, InterVec,
5144 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
5145 // Relink the chain
5146 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
5147 return Res;
5148 }
5149
5150 return IsFloat
5151 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
5152 N2: DAG.getTargetConstant(
5153 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
5154 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
5155}
5156
5157SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
5158 unsigned Opc = N->getOpcode();
5159 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
5160 assert(N->getValueType(0).isVector() &&
5161 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
5162 "Operand types must be vectors");
5163 // The result has a legal vector type, but the input needs splitting.
5164 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
5165 SDLoc DL(N);
5166 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
5167 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
5168
5169 EVT VT = N->getValueType(ResNo: 0);
5170 EVT PartResVT = getSetCCResultType(VT: Lo0.getValueType());
5171
5172 if (Opc == ISD::SETCC) {
5173 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
5174 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
5175 } else {
5176 assert(isStrict && "unexpected node");
5177 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
5178 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
5179 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
5180 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
5181 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5182 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
5183 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5184 }
5185
5186 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
5187 SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: LoRes, N2: HiRes);
5188 if (VT == ConcatVT)
5189 return Con;
5190
5191 EVT OpVT = N->getOperand(Num: 0).getValueType();
5192 ISD::NodeType ExtendCode =
5193 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
5194 return DAG.getExtOrTrunc(Op: Con, DL, VT, Opcode: ExtendCode);
5195}
5196
5197
5198SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
5199 // The result has a legal vector type, but the input needs splitting.
5200 EVT ResVT = N->getValueType(ResNo: 0);
5201 SDValue Lo, Hi;
5202 SDLoc DL(N);
5203 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
5204 EVT InVT = Lo.getValueType();
5205
5206 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5207 EC: InVT.getVectorElementCount());
5208
5209 if (N->isStrictFPOpcode()) {
5210 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5211 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
5212 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5213 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
5214 // Legalize the chain result - switch anything that used the old chain to
5215 // use the new one.
5216 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5217 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
5218 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5219 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
5220 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1),
5221 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5222 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1),
5223 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5224 } else {
5225 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
5226 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
5227 }
5228
5229 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
5230}
5231
5232// Split a vector type in an FP binary operation where the second operand has a
5233// different type from the first.
5234//
5235// The result (and the first input) has a legal vector type, but the second
5236// input needs splitting.
5237SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5238 SDLoc DL(N);
5239
5240 EVT LHSLoVT, LHSHiVT;
5241 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5242
5243 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
5244 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
5245
5246 SDValue LHSLo, LHSHi;
5247 std::tie(args&: LHSLo, args&: LHSHi) =
5248 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
5249
5250 SDValue RHSLo, RHSHi;
5251 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
5252
5253 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
5254 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
5255
5256 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
5257}
5258
5259SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5260 LLVMContext &Ctxt = *DAG.getContext();
5261 SDLoc dl(N);
5262
5263 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5264 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
5265 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
5266
5267 EVT ResVT = N->getValueType(ResNo: 0);
5268 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5269 EVT NewResVT =
5270 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
5271
5272 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
5273 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
5274
5275 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5276}
5277
5278SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5279 EVT ResVT = N->getValueType(ResNo: 0);
5280 SDValue Lo, Hi;
5281 SDLoc dl(N);
5282 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
5283 EVT InVT = Lo.getValueType();
5284
5285 EVT NewResVT =
5286 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5287 EC: InVT.getVectorElementCount());
5288
5289 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
5290 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
5291
5292 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5293}
5294
5295SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5296 SDLoc DL(N);
5297 EVT ResVT = N->getValueType(ResNo: 0);
5298
5299 SDValue Lo, Hi;
5300 SDValue VecOp = N->getOperand(Num: 0);
5301 GetSplitVector(Op: VecOp, Lo, Hi);
5302
5303 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5304 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5305 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
5306 SDValue VL =
5307 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
5308 SDValue ResLoNotVL =
5309 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
5310 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
5311 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
5312 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
5313}
5314
5315SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5316 SDLoc DL(N);
5317 EVT ResVT = N->getValueType(ResNo: 0);
5318
5319 SDValue Lo, Hi;
5320 SDValue VecOp = N->getOperand(Num: 0);
5321 GetSplitVector(Op: VecOp, Lo, Hi);
5322
5323 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
5324 auto [EVLLo, EVLHi] =
5325 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
5326 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
5327
5328 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5329 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5330 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5331 SDValue ResLoNotEVL =
5332 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
5333 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5334 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
5335 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
5336}
5337
5338SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5339 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
5340 SDLoc DL(HG);
5341 SDValue Inc = HG->getInc();
5342 SDValue Ptr = HG->getBasePtr();
5343 SDValue Scale = HG->getScale();
5344 SDValue IntID = HG->getIntID();
5345 EVT MemVT = HG->getMemoryVT();
5346 MachineMemOperand *MMO = HG->getMemOperand();
5347 ISD::MemIndexType IndexType = HG->getIndexType();
5348
5349 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5350 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
5351 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
5352 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5353 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
5354 Ops: OpsLo, MMO, IndexType);
5355 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5356 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
5357 MMO, IndexType);
5358}
5359
5360SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
5361 SDLoc DL(N);
5362
5363 if (OpNo == 0) {
5364 EVT LoResVT, HiResVT;
5365 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5366 SDValue SourceLo, SourceHi;
5367 std::tie(args&: SourceLo, args&: SourceHi) = DAG.SplitVectorOperand(N, OpNo: 0);
5368 SDValue MaskLo, MaskHi;
5369 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVectorOperand(N, OpNo: 2);
5370
5371 SDValue MatchLo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: LoResVT, N1: SourceLo,
5372 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
5373 SDValue MatchHi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: HiResVT, N1: SourceHi,
5374 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
5375 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo,
5376 N2: MatchHi);
5377 }
5378
5379 // Note: The Mask (OpNo == 2) should be widened with the result.
5380 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
5381
5382 SDValue NeedleLo, NeedleHi;
5383 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: NeedleLo, Hi&: NeedleHi);
5384
5385 SDValue MatchLo =
5386 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5387 N2: NeedleLo, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5388 SDValue MatchHi =
5389 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5390 N2: NeedleHi, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5391 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo, N2: MatchHi);
5392}
5393
5394SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5395 SDValue Acc = N->getOperand(Num: 0);
5396 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5397 "Accumulator should already be a legal type, and shouldn't need "
5398 "further splitting");
5399
5400 SDLoc DL(N);
5401 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5402 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
5403 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
5404 unsigned Opcode = N->getOpcode();
5405 EVT ResultVT = Acc.getValueType();
5406
5407 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
5408 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
5409}
5410
5411//===----------------------------------------------------------------------===//
5412// Result Vector Widening
5413//===----------------------------------------------------------------------===//
5414
5415void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5416 unsigned WidenResNo) {
5417 unsigned NumResults = N->getNumValues();
5418 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5419 if (ResNo == WidenResNo)
5420 continue;
5421 EVT ResVT = N->getValueType(ResNo);
5422 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
5423 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
5424 } else {
5425 SDLoc DL(N);
5426 SDValue ResVal =
5427 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
5428 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
5429 }
5430 }
5431}
5432
5433void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5434 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5435
5436 // See if the target wants to custom widen this node.
5437 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
5438 return;
5439
5440 SDValue Res = SDValue();
5441
5442 auto unrollExpandedOp = [&]() {
5443 // We're going to widen this vector op to a legal type by padding with undef
5444 // elements. If the wide vector op is eventually going to be expanded to
5445 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5446 // libcalls on the undef elements.
5447
5448 // The lane count comes from the result being widened, but operation
5449 // legality is queried using the corresponding result 0 type.
5450 EVT ResVT = N->getValueType(ResNo);
5451 EVT WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5452 if (WideResVT.isScalableVector())
5453 return false;
5454
5455 EVT VT0 = N->getValueType(ResNo: 0);
5456 EVT WideVecVT = VT0.changeVectorElementCount(
5457 Context&: *DAG.getContext(), EC: WideResVT.getVectorElementCount());
5458
5459 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
5460 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT0.getScalarType())) {
5461 SDValue Unrolled =
5462 DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
5463 Res = Unrolled.getValue(R: ResNo);
5464 if (N->getNumValues() > 1)
5465 ReplaceOtherWidenResults(N, WidenNode: Unrolled.getNode(), WidenResNo: ResNo);
5466 return true;
5467 }
5468 return false;
5469 };
5470
5471 switch (N->getOpcode()) {
5472 default:
5473#ifndef NDEBUG
5474 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5475 N->dump(&DAG);
5476 dbgs() << "\n";
5477#endif
5478 report_fatal_error(reason: "do not know how to widen the result of this operator!");
5479
5480 case ISD::LOOP_DEPENDENCE_RAW_MASK:
5481 case ISD::LOOP_DEPENDENCE_WAR_MASK:
5482 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5483 break;
5484 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5485 case ISD::ADDRSPACECAST:
5486 Res = WidenVecRes_ADDRSPACECAST(N);
5487 break;
5488 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5489 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5490 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5491 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5492 case ISD::INSERT_SUBVECTOR:
5493 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5494 break;
5495 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5496 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5497 case ISD::ATOMIC_LOAD:
5498 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5499 break;
5500 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5501 case ISD::STEP_VECTOR:
5502 case ISD::SPLAT_VECTOR:
5503 case ISD::SCALAR_TO_VECTOR:
5504 Res = WidenVecRes_ScalarOp(N);
5505 break;
5506 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5507 case ISD::VSELECT:
5508 case ISD::SELECT:
5509 case ISD::VP_MERGE:
5510 Res = WidenVecRes_Select(N);
5511 break;
5512 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5513 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5514 case ISD::POISON:
5515 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5516 case ISD::VECTOR_SHUFFLE:
5517 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5518 break;
5519 case ISD::VP_LOAD:
5520 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5521 break;
5522 case ISD::VP_LOAD_FF:
5523 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5524 break;
5525 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5526 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5527 break;
5528 case ISD::VECTOR_COMPRESS:
5529 Res = WidenVecRes_VECTOR_COMPRESS(N);
5530 break;
5531 case ISD::VECTOR_SHUFFLE_VAR:
5532 Res = WidenVecRes_VECTOR_SHUFFLE_VAR(N);
5533 break;
5534 case ISD::MLOAD:
5535 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5536 break;
5537 case ISD::MGATHER:
5538 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5539 break;
5540 case ISD::VP_GATHER:
5541 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5542 break;
5543 case ISD::VECTOR_REVERSE:
5544 Res = WidenVecRes_VECTOR_REVERSE(N);
5545 break;
5546 case ISD::GET_ACTIVE_LANE_MASK:
5547 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5548 break;
5549 case ISD::VECTOR_INTERLEAVE:
5550 WidenVecRes_VECTOR_INTERLEAVE(N);
5551 break;
5552 case ISD::VECTOR_MATCH:
5553 Res = WidenVecRes_VECTOR_MATCH(N);
5554 break;
5555 case ISD::VECTOR_DEINTERLEAVE:
5556 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5557 break;
5558
5559 case ISD::ADD:
5560 case ISD::AND:
5561 case ISD::MUL:
5562 case ISD::MULHS:
5563 case ISD::MULHU:
5564 case ISD::ABDS:
5565 case ISD::ABDU:
5566 case ISD::OR:
5567 case ISD::SUB:
5568 case ISD::XOR:
5569 case ISD::SHL:
5570 case ISD::SRA:
5571 case ISD::SRL:
5572 case ISD::CLMUL:
5573 case ISD::CLMULR:
5574 case ISD::CLMULH:
5575 case ISD::PEXT:
5576 case ISD::PDEP:
5577 case ISD::FMINNUM:
5578 case ISD::FMINNUM_IEEE:
5579 case ISD::FMAXNUM:
5580 case ISD::FMAXNUM_IEEE:
5581 case ISD::FMINIMUM:
5582 case ISD::FMAXIMUM:
5583 case ISD::FMINIMUMNUM:
5584 case ISD::FMAXIMUMNUM:
5585 case ISD::SMIN:
5586 case ISD::SMAX:
5587 case ISD::UMIN:
5588 case ISD::UMAX:
5589 case ISD::UADDSAT:
5590 case ISD::SADDSAT:
5591 case ISD::USUBSAT:
5592 case ISD::SSUBSAT:
5593 case ISD::SSHLSAT:
5594 case ISD::USHLSAT:
5595 case ISD::ROTL:
5596 case ISD::ROTR:
5597 case ISD::AVGFLOORS:
5598 case ISD::AVGFLOORU:
5599 case ISD::AVGCEILS:
5600 case ISD::AVGCEILU:
5601 case ISD::AVGSUBFLOORS:
5602 case ISD::AVGSUBFLOORU:
5603 // Vector-predicated binary op widening. Note that -- unlike the
5604 // unpredicated versions -- we don't have to worry about trapping on
5605 // operations like UDIV, FADD, etc., as we pass on the original vector
5606 // length parameter. This means the widened elements containing garbage
5607 // aren't active.
5608 case ISD::VP_SDIV:
5609 case ISD::VP_UDIV:
5610 case ISD::VP_SREM:
5611 case ISD::VP_UREM:
5612 Res = WidenVecRes_Binary(N);
5613 break;
5614
5615 case ISD::MASKED_UDIV:
5616 case ISD::MASKED_SDIV:
5617 case ISD::MASKED_UREM:
5618 case ISD::MASKED_SREM:
5619 Res = WidenVecRes_MaskedBinary(N);
5620 break;
5621
5622 case ISD::SCMP:
5623 case ISD::UCMP:
5624 Res = WidenVecRes_CMP(N);
5625 break;
5626
5627 case ISD::FPOW:
5628 case ISD::FATAN2:
5629 case ISD::FREM:
5630 if (unrollExpandedOp())
5631 break;
5632 // If the target has custom/legal support for the scalar FP intrinsic ops
5633 // (they are probably not destined to become libcalls), then widen those
5634 // like any other binary ops.
5635 [[fallthrough]];
5636
5637 case ISD::FADD:
5638 case ISD::FMUL:
5639 case ISD::FSUB:
5640 case ISD::FDIV:
5641 case ISD::SDIV:
5642 case ISD::UDIV:
5643 case ISD::SREM:
5644 case ISD::UREM:
5645 Res = WidenVecRes_BinaryCanTrap(N);
5646 break;
5647
5648 case ISD::SMULFIX:
5649 case ISD::SMULFIXSAT:
5650 case ISD::UMULFIX:
5651 case ISD::UMULFIXSAT:
5652 // These are binary operations, but with an extra operand that shouldn't
5653 // be widened (the scale).
5654 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5655 break;
5656
5657#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5658 case ISD::STRICT_##DAGN:
5659#include "llvm/IR/ConstrainedOps.def"
5660 Res = WidenVecRes_StrictFP(N);
5661 break;
5662
5663 case ISD::UADDO:
5664 case ISD::SADDO:
5665 case ISD::USUBO:
5666 case ISD::SSUBO:
5667 case ISD::UMULO:
5668 case ISD::SMULO:
5669 Res = WidenVecRes_OverflowOp(N, ResNo);
5670 break;
5671
5672 case ISD::FCOPYSIGN:
5673 Res = WidenVecRes_FCOPYSIGN(N);
5674 break;
5675
5676 case ISD::IS_FPCLASS:
5677 case ISD::FPTRUNC_ROUND:
5678 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5679 break;
5680
5681 case ISD::FLDEXP:
5682 case ISD::FPOWI:
5683 if (!unrollExpandedOp())
5684 Res = WidenVecRes_ExpOp(N);
5685 break;
5686
5687 case ISD::ANY_EXTEND_VECTOR_INREG:
5688 case ISD::SIGN_EXTEND_VECTOR_INREG:
5689 case ISD::ZERO_EXTEND_VECTOR_INREG:
5690 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5691 break;
5692
5693 case ISD::ANY_EXTEND:
5694 case ISD::FP_EXTEND:
5695 case ISD::FP_ROUND:
5696 case ISD::FP_TO_SINT:
5697 case ISD::FP_TO_UINT:
5698 case ISD::SIGN_EXTEND:
5699 case ISD::SINT_TO_FP:
5700 case ISD::TRUNCATE:
5701 case ISD::UINT_TO_FP:
5702 case ISD::ZERO_EXTEND:
5703 case ISD::CONVERT_FROM_ARBITRARY_FP:
5704 case ISD::CONVERT_TO_ARBITRARY_FP:
5705 Res = WidenVecRes_Convert(N);
5706 break;
5707
5708 case ISD::FP_TO_SINT_SAT:
5709 case ISD::FP_TO_UINT_SAT:
5710 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5711 break;
5712
5713 case ISD::LRINT:
5714 case ISD::LLRINT:
5715 case ISD::LROUND:
5716 case ISD::LLROUND:
5717 Res = WidenVecRes_XROUND(N);
5718 break;
5719
5720 case ISD::FACOS:
5721 case ISD::FASIN:
5722 case ISD::FATAN:
5723 case ISD::FCEIL:
5724 case ISD::FCOS:
5725 case ISD::FCOSH:
5726 case ISD::FEXP:
5727 case ISD::FEXP2:
5728 case ISD::FEXP10:
5729 case ISD::FFLOOR:
5730 case ISD::FLOG:
5731 case ISD::FLOG10:
5732 case ISD::FLOG2:
5733 case ISD::FNEARBYINT:
5734 case ISD::FRINT:
5735 case ISD::FROUND:
5736 case ISD::FROUNDEVEN:
5737 case ISD::FSIN:
5738 case ISD::FSINH:
5739 case ISD::FSQRT:
5740 case ISD::FTAN:
5741 case ISD::FTANH:
5742 case ISD::FTRUNC:
5743 if (unrollExpandedOp())
5744 break;
5745 // If the target has custom/legal support for the scalar FP intrinsic ops
5746 // (they are probably not destined to become libcalls), then widen those
5747 // like any other unary ops.
5748 [[fallthrough]];
5749
5750 case ISD::ABS:
5751 case ISD::ABS_MIN_POISON:
5752 case ISD::BITREVERSE:
5753 case ISD::BSWAP:
5754 case ISD::CTLZ:
5755 case ISD::CTLZ_ZERO_POISON:
5756 case ISD::CTPOP:
5757 case ISD::CTTZ:
5758 case ISD::CTTZ_ZERO_POISON:
5759 case ISD::FNEG:
5760 case ISD::FABS:
5761 case ISD::FREEZE:
5762 case ISD::ARITH_FENCE:
5763 case ISD::FCANONICALIZE:
5764 case ISD::AssertNoFPClass:
5765 case ISD::MASK_BEFOREFIRST:
5766 Res = WidenVecRes_Unary(N);
5767 break;
5768 case ISD::FMA:
5769 case ISD::FSHL:
5770 case ISD::FSHR:
5771 Res = WidenVecRes_Ternary(N);
5772 break;
5773 case ISD::FMODF:
5774 case ISD::FFREXP:
5775 case ISD::FSINCOS:
5776 case ISD::FSINCOSPI: {
5777 if (!unrollExpandedOp())
5778 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5779 break;
5780 }
5781 case ISD::PARTIAL_REDUCE_UMLA:
5782 case ISD::PARTIAL_REDUCE_SMLA:
5783 case ISD::PARTIAL_REDUCE_SUMLA:
5784 case ISD::PARTIAL_REDUCE_FMLA:
5785 Res = WidenVecRes_PARTIAL_REDUCE_MLA(N);
5786 break;
5787 }
5788
5789 // If Res is null, the sub-method took care of registering the result.
5790 if (Res.getNode())
5791 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5792}
5793
5794SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5795 // Ternary op widening.
5796 SDLoc dl(N);
5797 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5798 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5799 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5800 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5801 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5802}
5803
5804SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5805 // Binary op widening.
5806 SDLoc dl(N);
5807 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5808 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5809 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5810 if (N->getNumOperands() == 2)
5811 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5812 Flags: N->getFlags());
5813
5814 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5815 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
5816 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
5817 "Expected VP opcode");
5818
5819 SDValue Mask =
5820 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5821 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5822 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5823}
5824
5825SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5826 SDLoc dl(N);
5827 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5828 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5829 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5830 SDValue Mask = N->getOperand(Num: 2);
5831 EVT WideMaskVT = WidenVT.changeVectorElementType(
5832 Context&: *DAG.getContext(), EltVT: Mask.getValueType().getVectorElementType());
5833 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeros=*/FillWithZeroes: true);
5834 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask,
5835 Flags: N->getFlags());
5836}
5837
5838SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5839 LLVMContext &Ctxt = *DAG.getContext();
5840 SDLoc dl(N);
5841
5842 SDValue LHS = N->getOperand(Num: 0);
5843 SDValue RHS = N->getOperand(Num: 1);
5844 EVT OpVT = LHS.getValueType();
5845 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5846 LHS = GetWidenedVector(Op: LHS);
5847 RHS = GetWidenedVector(Op: RHS);
5848 OpVT = LHS.getValueType();
5849 }
5850
5851 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5852 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5853 if (WidenResEC == OpVT.getVectorElementCount()) {
5854 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5855 }
5856
5857 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5858}
5859
5860SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5861 // Binary op widening, but with an extra operand that shouldn't be widened.
5862 SDLoc dl(N);
5863 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5864 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5865 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5866 SDValue InOp3 = N->getOperand(Num: 2);
5867 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5868 Flags: N->getFlags());
5869}
5870
5871// Given a vector of operations that have been broken up to widen, see
5872// if we can collect them together into the next widest legal VT. This
5873// implementation is trap-safe.
5874static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5875 SmallVectorImpl<SDValue> &ConcatOps,
5876 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5877 EVT WidenVT) {
5878 // Check to see if we have a single operation with the widen type.
5879 if (ConcatEnd == 1) {
5880 VT = ConcatOps[0].getValueType();
5881 if (VT == WidenVT)
5882 return ConcatOps[0];
5883 }
5884
5885 SDLoc dl(ConcatOps[0]);
5886 EVT WidenEltVT = WidenVT.getVectorElementType();
5887
5888 // while (Some element of ConcatOps is not of type MaxVT) {
5889 // From the end of ConcatOps, collect elements of the same type and put
5890 // them into an op of the next larger supported type
5891 // }
5892 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5893 int Idx = ConcatEnd - 1;
5894 VT = ConcatOps[Idx--].getValueType();
5895 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5896 Idx--;
5897
5898 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5899 EVT NextVT;
5900 do {
5901 NextSize *= 2;
5902 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5903 } while (!TLI.isTypeLegal(VT: NextVT));
5904
5905 if (!VT.isVector()) {
5906 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5907 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5908 unsigned NumToInsert = ConcatEnd - Idx - 1;
5909 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5910 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5911 ConcatOps[Idx+1] = VecOp;
5912 ConcatEnd = Idx + 2;
5913 } else {
5914 // Vector type, create a CONCAT_VECTORS of type NextVT
5915 SDValue undefVec = DAG.getPOISON(VT);
5916 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5917 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5918 unsigned RealVals = ConcatEnd - Idx - 1;
5919 unsigned SubConcatEnd = 0;
5920 unsigned SubConcatIdx = Idx + 1;
5921 while (SubConcatEnd < RealVals)
5922 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5923 while (SubConcatEnd < OpsToConcat)
5924 SubConcatOps[SubConcatEnd++] = undefVec;
5925 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5926 VT: NextVT, Ops: SubConcatOps);
5927 ConcatEnd = SubConcatIdx + 1;
5928 }
5929 }
5930
5931 // Check to see if we have a single operation with the widen type.
5932 if (ConcatEnd == 1) {
5933 VT = ConcatOps[0].getValueType();
5934 if (VT == WidenVT)
5935 return ConcatOps[0];
5936 }
5937
5938 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5939 unsigned NumOps = WidenVT.getVectorElementCount().getKnownScalarFactor(
5940 RHS: MaxVT.getVectorElementCount());
5941 if (NumOps != ConcatEnd ) {
5942 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5943 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5944 ConcatOps[j] = UndefVal;
5945 }
5946 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5947 Ops: ArrayRef(ConcatOps.data(), NumOps));
5948}
5949
5950SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5951 // Binary op widening for operations that can trap.
5952 unsigned Opcode = N->getOpcode();
5953 SDLoc dl(N);
5954 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5955 EVT VT = WidenVT;
5956 ElementCount NumElts = VT.getVectorElementCount();
5957 const SDNodeFlags Flags = N->getFlags();
5958 while (!TLI.isTypeLegal(VT) && NumElts.getKnownMinValue() != 1) {
5959 NumElts = NumElts.divideCoefficientBy(RHS: 2);
5960 VT = VT.changeVectorElementCount(Context&: *DAG.getContext(), EC: NumElts);
5961 }
5962
5963 bool FoundLegalVectorType = TLI.isTypeLegal(VT) && !NumElts.isScalar();
5964 if (FoundLegalVectorType && DAG.isSafeToSpeculativelyExecute(Opcode)) {
5965 // Operation can be speculatively executed so just widen as normal.
5966 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5967 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5968 return DAG.getNode(Opcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5969 }
5970
5971 // Generate a vp.op if it is custom/legal for the target. This avoids need
5972 // to split and tile the subvectors (below), because the inactive lanes can
5973 // simply be disabled. To avoid possible recursion, only do this if the
5974 // widened mask type is legal.
5975 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5976 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5977 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5978 EC: WidenVT.getVectorElementCount());
5979 TLI.isTypeLegal(VT: WideMaskVT)) {
5980 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5981 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5982 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5983 SDValue EVL =
5984 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5985 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5986 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5987 Flags);
5988 }
5989 }
5990
5991 // No legal vector version so unroll the vector operation and then widen.
5992 if (NumElts.isScalar())
5993 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5994
5995 if (!TLI.isTypeLegal(VT))
5996 report_fatal_error(reason: "Cannot widen scalable vector trapping binop!");
5997
5998 // Since the operation can trap, apply operation on the original vector.
5999 EVT MaxVT = VT;
6000 EVT EltVT = VT.getVectorElementType();
6001 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6002 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
6003 ElementCount CurNumElts = N->getValueType(ResNo: 0).getVectorElementCount();
6004
6005 SmallVector<SDValue, 16> ConcatOps(CurNumElts.getKnownMinValue());
6006 unsigned ConcatEnd = 0; // Current ConcatOps index.
6007 int Idx = 0; // Current Idx into input vectors.
6008
6009 // NumElts := greatest legal vector size (at most WidenVT)
6010 // while (orig. vector has unhandled elements) {
6011 // take munches of size NumElts from the beginning and add to ConcatOps
6012 // NumElts := next smaller supported vector size or 1
6013 // }
6014 for (;;) {
6015 while (ElementCount::isKnownGE(LHS: CurNumElts, RHS: NumElts)) {
6016 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
6017 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
6018 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
6019 // Subvector indices are implicitly scaled by vscale.
6020 Idx += NumElts.getKnownMinValue();
6021 CurNumElts -= NumElts;
6022 }
6023
6024 if (CurNumElts.isZero())
6025 break;
6026
6027 // Find the next smaller legal vector type.
6028 do {
6029 NumElts = NumElts.divideCoefficientBy(RHS: 2);
6030 VT = VT.changeVectorElementCount(Context&: *DAG.getContext(), EC: NumElts);
6031 } while (!TLI.isTypeLegal(VT) && NumElts.getKnownMinValue() != 1);
6032
6033 if (NumElts.isScalar()) {
6034 // Scalarize the non-type-legal remainder.
6035 for (unsigned i = 0; i != CurNumElts.getFixedValue(); ++i, ++Idx) {
6036 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp1, Idx);
6037 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp2, Idx);
6038 ConcatOps[ConcatEnd++] =
6039 DAG.getNode(Opcode, DL: dl, VT: EltVT, N1: EOp1, N2: EOp2, Flags);
6040 }
6041
6042 break;
6043 }
6044
6045 if (!TLI.isTypeLegal(VT))
6046 report_fatal_error(reason: "scalarization of scalable vectors is not supported");
6047 }
6048
6049 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
6050}
6051
6052SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
6053 switch (N->getOpcode()) {
6054 case ISD::STRICT_FSETCC:
6055 case ISD::STRICT_FSETCCS:
6056 return WidenVecRes_STRICT_FSETCC(N);
6057 case ISD::STRICT_FP_EXTEND:
6058 case ISD::STRICT_FP_ROUND:
6059 case ISD::STRICT_FP_TO_SINT:
6060 case ISD::STRICT_FP_TO_UINT:
6061 case ISD::STRICT_SINT_TO_FP:
6062 case ISD::STRICT_UINT_TO_FP:
6063 return WidenVecRes_Convert_StrictFP(N);
6064 default:
6065 break;
6066 }
6067
6068 // StrictFP op widening for operations that can trap.
6069 unsigned NumOpers = N->getNumOperands();
6070 unsigned Opcode = N->getOpcode();
6071 SDLoc dl(N);
6072 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6073 EVT WidenEltVT = WidenVT.getVectorElementType();
6074 EVT VT = WidenVT;
6075 unsigned NumElts = VT.getVectorNumElements();
6076 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
6077 NumElts = NumElts / 2;
6078 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
6079 }
6080
6081 // No legal vector version so unroll the vector operation and then widen.
6082 if (NumElts == 1)
6083 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
6084
6085 // Since the operation can trap, apply operation on the original vector.
6086 EVT MaxVT = VT;
6087 SmallVector<SDValue, 4> InOps;
6088 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
6089
6090 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
6091 SmallVector<SDValue, 16> Chains;
6092 unsigned ConcatEnd = 0; // Current ConcatOps index.
6093 int Idx = 0; // Current Idx into input vectors.
6094
6095 // The Chain is the first operand.
6096 InOps.push_back(Elt: N->getOperand(Num: 0));
6097
6098 // Now process the remaining operands.
6099 for (unsigned i = 1; i < NumOpers; ++i) {
6100 SDValue Oper = N->getOperand(Num: i);
6101
6102 EVT OpVT = Oper.getValueType();
6103 if (OpVT.isVector()) {
6104 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
6105 Oper = GetWidenedVector(Op: Oper);
6106 else {
6107 EVT WideOpVT =
6108 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
6109 EC: WidenVT.getVectorElementCount());
6110 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
6111 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
6112 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
6113 }
6114 }
6115
6116 InOps.push_back(Elt: Oper);
6117 }
6118
6119 // NumElts := greatest legal vector size (at most WidenVT)
6120 // while (orig. vector has unhandled elements) {
6121 // take munches of size NumElts from the beginning and add to ConcatOps
6122 // NumElts := next smaller supported vector size or 1
6123 // }
6124 while (CurNumElts != 0) {
6125 while (CurNumElts >= NumElts) {
6126 SmallVector<SDValue, 4> EOps;
6127
6128 for (unsigned i = 0; i < NumOpers; ++i) {
6129 SDValue Op = InOps[i];
6130
6131 EVT OpVT = Op.getValueType();
6132 if (OpVT.isVector()) {
6133 EVT OpExtractVT =
6134 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
6135 EC: VT.getVectorElementCount());
6136 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
6137 }
6138
6139 EOps.push_back(Elt: Op);
6140 }
6141
6142 EVT OperVT[] = {VT, MVT::Other};
6143 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
6144 ConcatOps[ConcatEnd++] = Oper;
6145 Chains.push_back(Elt: Oper.getValue(R: 1));
6146 Idx += NumElts;
6147 CurNumElts -= NumElts;
6148 }
6149 do {
6150 NumElts = NumElts / 2;
6151 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
6152 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
6153
6154 if (NumElts == 1) {
6155 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
6156 SmallVector<SDValue, 4> EOps;
6157
6158 for (unsigned i = 0; i < NumOpers; ++i) {
6159 SDValue Op = InOps[i];
6160
6161 EVT OpVT = Op.getValueType();
6162 if (OpVT.isVector())
6163 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
6164 Idx);
6165
6166 EOps.push_back(Elt: Op);
6167 }
6168
6169 EVT WidenVT[] = {WidenEltVT, MVT::Other};
6170 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
6171 ConcatOps[ConcatEnd++] = Oper;
6172 Chains.push_back(Elt: Oper.getValue(R: 1));
6173 }
6174 CurNumElts = 0;
6175 }
6176 }
6177
6178 // Build a factor node to remember all the Ops that have been created.
6179 SDValue NewChain;
6180 if (Chains.size() == 1)
6181 NewChain = Chains[0];
6182 else
6183 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
6184 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6185
6186 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
6187}
6188
6189SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
6190 SDLoc DL(N);
6191 EVT ResVT = N->getValueType(ResNo: 0);
6192 EVT OvVT = N->getValueType(ResNo: 1);
6193 EVT WideResVT, WideOvVT;
6194 SDValue WideLHS, WideRHS;
6195
6196 // TODO: This might result in a widen/split loop.
6197 if (ResNo == 0) {
6198 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
6199 WideOvVT = EVT::getVectorVT(
6200 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
6201 NumElements: WideResVT.getVectorNumElements());
6202
6203 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6204 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
6205 } else {
6206 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
6207 WideResVT = EVT::getVectorVT(
6208 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
6209 NumElements: WideOvVT.getVectorNumElements());
6210
6211 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6212 SDValue Poison = DAG.getPOISON(VT: WideResVT);
6213
6214 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6215 N2: N->getOperand(Num: 0), N3: Zero);
6216 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6217 N2: N->getOperand(Num: 1), N3: Zero);
6218 }
6219
6220 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
6221 SDNode *WideNode = DAG.getNode(
6222 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
6223
6224 // Replace the other vector result not being explicitly widened here.
6225 unsigned OtherNo = 1 - ResNo;
6226 EVT OtherVT = N->getValueType(ResNo: OtherNo);
6227 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
6228 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
6229 } else {
6230 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6231 SDValue OtherVal = DAG.getNode(
6232 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
6233 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
6234 }
6235
6236 return SDValue(WideNode, ResNo);
6237}
6238
6239SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
6240 LLVMContext &Ctx = *DAG.getContext();
6241 SDValue InOp = N->getOperand(Num: 0);
6242 SDLoc DL(N);
6243
6244 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
6245 ElementCount WidenEC = WidenVT.getVectorElementCount();
6246
6247 EVT InVT = InOp.getValueType();
6248
6249 unsigned Opcode = N->getOpcode();
6250 const SDNodeFlags Flags = N->getFlags();
6251
6252 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
6253 // not equal that of WidenVT.
6254 if (N->getOpcode() == ISD::ZERO_EXTEND &&
6255 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
6256 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
6257 WidenVT.getScalarSizeInBits()) {
6258 InOp = ZExtPromotedInteger(Op: InOp);
6259 InVT = InOp.getValueType();
6260 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
6261 Opcode = ISD::TRUNCATE;
6262 }
6263
6264 EVT InEltVT = InVT.getVectorElementType();
6265 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
6266 ElementCount InVTEC = InVT.getVectorElementCount();
6267
6268 // Helper to build node with all scalar trailing operands.
6269 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6270 if (N->getNumOperands() == 1)
6271 return DAG.getNode(Opcode, DL, VT, Operand: Op, Flags);
6272 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6273 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
6274 N4: N->getOperand(Num: 3), Flags);
6275 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), Flags);
6276 };
6277
6278 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6279 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6280 InVT = InOp.getValueType();
6281 InVTEC = InVT.getVectorElementCount();
6282 if (InVTEC == WidenEC)
6283 return MakeConvertNode(WidenVT, InOp);
6284 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6285 // If both input and result vector types are of same width, extend
6286 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6287 // accepts fewer elements in the result than in the input.
6288 if (Opcode == ISD::ANY_EXTEND)
6289 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6290 if (Opcode == ISD::SIGN_EXTEND)
6291 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6292 if (Opcode == ISD::ZERO_EXTEND)
6293 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6294 }
6295
6296 // For TRUNCATE, try to widen using the legal EC of the input type instead
6297 // if the legalisation action for that intermediate type is not widening.
6298 // E.g. for trunc nxv1i64 -> nxv1i8 where
6299 // - nxv1i64 input gets widened to nxv2i64
6300 // - nxv1i8 output gets widened to nxv16i8
6301 // Then one can try widening the result to nxv2i8 (instead of going all the
6302 // way to nxv16i8) if this later allows type promotion.
6303 EVT MidResVT =
6304 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
6305 if (N->getOpcode() == ISD::TRUNCATE &&
6306 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
6307 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
6308 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
6309 }
6310 }
6311
6312 if (TLI.isTypeLegal(VT: InWidenVT)) {
6313 // Because the result and the input are different vector types, widening
6314 // the result could create a legal type but widening the input might make
6315 // it an illegal type that might lead to repeatedly splitting the input
6316 // and then widening it. To avoid this, we widen the input only if
6317 // it results in a legal type.
6318 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
6319 // Widen the input and call convert on the widened input vector.
6320 unsigned NumConcat =
6321 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6322 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
6323 Ops[0] = InOp;
6324 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
6325 return MakeConvertNode(WidenVT, InVec);
6326 }
6327
6328 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
6329 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
6330 // Extract the input and convert the shorten input vector.
6331 return MakeConvertNode(WidenVT, InVal);
6332 }
6333 }
6334
6335 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6336 EVT EltVT = WidenVT.getVectorElementType();
6337 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
6338 // Use the original element count so we don't do more scalar opts than
6339 // necessary.
6340 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6341 for (unsigned i=0; i < MinElts; ++i) {
6342 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6343 Ops[i] = MakeConvertNode(EltVT, Val);
6344 }
6345
6346 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6347}
6348
6349SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6350 SDLoc dl(N);
6351 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6352 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6353
6354 SDValue Src = N->getOperand(Num: 0);
6355 EVT SrcVT = Src.getValueType();
6356
6357 // Also widen the input.
6358 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6359 Src = GetWidenedVector(Op: Src);
6360 SrcVT = Src.getValueType();
6361 }
6362
6363 // Input and output not widened to the same size, give up.
6364 if (WidenNumElts != SrcVT.getVectorElementCount())
6365 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6366
6367 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
6368}
6369
6370SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6371 SDLoc dl(N);
6372 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6373 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6374
6375 SDValue Src = N->getOperand(Num: 0);
6376 EVT SrcVT = Src.getValueType();
6377
6378 // Also widen the input.
6379 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6380 Src = GetWidenedVector(Op: Src);
6381 SrcVT = Src.getValueType();
6382 }
6383
6384 // Input and output not widened to the same size, give up.
6385 if (WidenNumElts != SrcVT.getVectorElementCount())
6386 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6387
6388 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
6389}
6390
6391SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6392 SDValue InOp = N->getOperand(Num: 1);
6393 SDLoc DL(N);
6394 SmallVector<SDValue, 4> NewOps(N->ops());
6395
6396 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6397 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6398
6399 EVT InVT = InOp.getValueType();
6400 EVT InEltVT = InVT.getVectorElementType();
6401
6402 unsigned Opcode = N->getOpcode();
6403
6404 // FIXME: Optimizations need to be implemented here.
6405
6406 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6407 EVT EltVT = WidenVT.getVectorElementType();
6408 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
6409 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
6410 SmallVector<SDValue, 32> OpChains;
6411 // Use the original element count so we don't do more scalar opts than
6412 // necessary.
6413 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6414 for (unsigned i=0; i < MinElts; ++i) {
6415 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6416 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
6417 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6418 }
6419 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
6420 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6421
6422 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6423}
6424
6425SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6426 unsigned Opcode = N->getOpcode();
6427 SDValue InOp = N->getOperand(Num: 0);
6428 SDLoc DL(N);
6429
6430 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6431 EVT WidenSVT = WidenVT.getVectorElementType();
6432 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6433
6434 EVT InVT = InOp.getValueType();
6435 EVT InSVT = InVT.getVectorElementType();
6436 unsigned InVTNumElts = InVT.getVectorNumElements();
6437
6438 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6439 InOp = GetWidenedVector(Op: InOp);
6440 InVT = InOp.getValueType();
6441 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6442 switch (Opcode) {
6443 case ISD::ANY_EXTEND_VECTOR_INREG:
6444 case ISD::SIGN_EXTEND_VECTOR_INREG:
6445 case ISD::ZERO_EXTEND_VECTOR_INREG:
6446 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
6447 }
6448 }
6449 }
6450
6451 // Unroll, extend the scalars and rebuild the vector.
6452 SmallVector<SDValue, 16> Ops;
6453 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
6454 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
6455 switch (Opcode) {
6456 case ISD::ANY_EXTEND_VECTOR_INREG:
6457 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
6458 break;
6459 case ISD::SIGN_EXTEND_VECTOR_INREG:
6460 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
6461 break;
6462 case ISD::ZERO_EXTEND_VECTOR_INREG:
6463 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
6464 break;
6465 default:
6466 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6467 }
6468 Ops.push_back(Elt: Val);
6469 }
6470
6471 while (Ops.size() != WidenNumElts)
6472 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
6473
6474 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6475}
6476
6477SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6478 // If this is an FCOPYSIGN with same input types, we can treat it as a
6479 // normal binary op.
6480 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
6481 return WidenVecRes_Binary(N);
6482
6483 // If the types are different, fall back to unrolling.
6484 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6485 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6486}
6487
6488/// Result and first source operand are different scalar types, but must have
6489/// the same number of elements. There is an additional control argument which
6490/// should be passed through unchanged.
6491SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6492 SDValue FpValue = N->getOperand(Num: 0);
6493 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6494 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6495 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6496 SDValue Arg = GetWidenedVector(Op: FpValue);
6497 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6498 Flags: N->getFlags());
6499}
6500
6501SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6502 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6503 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6504 SDValue RHS = N->getOperand(Num: 1);
6505 EVT ExpVT = RHS.getValueType();
6506 SDValue ExpOp = RHS;
6507 if (ExpVT.isVector()) {
6508 EVT WideExpVT = WidenVT.changeVectorElementType(
6509 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6510 if (WideExpVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ExpVT))
6511 ExpOp = GetWidenedVector(Op: RHS);
6512 else
6513 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6514 }
6515
6516 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6517}
6518
6519SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6520 // Unary op widening.
6521 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6522 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6523 if (N->getNumOperands() == 1)
6524 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6525 assert(N->getOpcode() == ISD::AssertNoFPClass && "unexpected opcode");
6526 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1),
6527 Flags: N->getFlags());
6528}
6529
6530SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6531 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6532 EVT ExtVT = EVT::getVectorVT(
6533 Context&: *DAG.getContext(),
6534 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(),
6535 EC: WidenVT.getVectorElementCount());
6536 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6537 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6538 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6539}
6540
6541SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6542 unsigned ResNo) {
6543 EVT VT0 = N->getValueType(ResNo: 0);
6544 EVT VT1 = N->getValueType(ResNo: 1);
6545
6546 assert(VT0.isVector() && VT1.isVector() &&
6547 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6548 "expected both results to be vectors of matching element count");
6549
6550 LLVMContext &Ctx = *DAG.getContext();
6551 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6552
6553 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6554 ElementCount WidenEC = WidenVT.getVectorElementCount();
6555
6556 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6557 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6558
6559 SDNode *WidenNode =
6560 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6561 .getNode();
6562
6563 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6564 return SDValue(WidenNode, ResNo);
6565}
6566
6567SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6568 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6569 return GetWidenedVector(Op: WidenVec);
6570}
6571
6572SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6573 SDLoc DL(N);
6574 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6575 ElementCount WidenEC = WidenVT.getVectorElementCount();
6576 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6577
6578 // The source has the same number of elements as the result, so widen it to
6579 // match WidenVT. It only lives in the widened-vector map if it is itself
6580 // widened; otherwise pad it up to the widened element count
6581 // when it is illegal.
6582 SDValue InOp = N->getOperand(Num: 0);
6583 EVT InVT = InOp.getValueType();
6584 TargetLowering::LegalizeTypeAction InAction = getTypeAction(VT: InVT);
6585 if (InAction == TargetLowering::TypeWidenVector) {
6586 InOp = GetWidenedVector(Op: InOp);
6587 } else if (InAction != TargetLowering::TypeLegal) {
6588 EVT InWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6589 VT: InVT.getVectorElementType(), EC: WidenEC);
6590 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: InWidenVT), SubVec: InOp, Idx: 0);
6591 }
6592
6593 return DAG.getAddrSpaceCast(
6594 dl: DL, VT: WidenVT, Ptr: InOp, SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6595 DestAS: AddrSpaceCastN->getDestAddressSpace(), Flags: AddrSpaceCastN->getFlags());
6596}
6597
6598SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6599 SDValue InOp = N->getOperand(Num: 0);
6600 EVT InVT = InOp.getValueType();
6601 EVT VT = N->getValueType(ResNo: 0);
6602 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6603 SDLoc dl(N);
6604
6605 switch (getTypeAction(VT: InVT)) {
6606 case TargetLowering::TypeLegal:
6607 break;
6608 case TargetLowering::TypeScalarizeScalableVector:
6609 report_fatal_error(reason: "scalarization of scalable vectors is not supported");
6610 case TargetLowering::TypePromoteInteger: {
6611 // If the incoming type is a vector that is being promoted, then
6612 // we know that the elements are arranged differently and that we
6613 // must perform the conversion using a stack slot.
6614 if (InVT.isVector())
6615 break;
6616
6617 // If the InOp is promoted to the same size, convert it. Otherwise,
6618 // fall out of the switch and widen the promoted input.
6619 SDValue NInOp = GetPromotedInteger(Op: InOp);
6620 EVT NInVT = NInOp.getValueType();
6621 if (WidenVT.bitsEq(VT: NInVT)) {
6622 // For big endian targets we need to shift the input integer or the
6623 // interesting bits will end up at the wrong place.
6624 if (DAG.getDataLayout().isBigEndian()) {
6625 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6626 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6627 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6628 }
6629 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6630 }
6631 InOp = NInOp;
6632 InVT = NInVT;
6633 break;
6634 }
6635 case TargetLowering::TypeSoftenFloat:
6636 case TargetLowering::TypeSoftPromoteHalf:
6637 case TargetLowering::TypeExpandInteger:
6638 case TargetLowering::TypeExpandFloat:
6639 case TargetLowering::TypeScalarizeVector:
6640 case TargetLowering::TypeSplitVector:
6641 break;
6642 case TargetLowering::TypeWidenVector:
6643 // If the InOp is widened to the same size, convert it. Otherwise, fall
6644 // out of the switch and widen the widened input.
6645 InOp = GetWidenedVector(Op: InOp);
6646 InVT = InOp.getValueType();
6647 if (WidenVT.bitsEq(VT: InVT))
6648 // The input widens to the same size. Convert to the widen value.
6649 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6650 break;
6651 }
6652
6653 unsigned WidenSize = WidenVT.getSizeInBits();
6654 unsigned InSize = InVT.getSizeInBits();
6655 unsigned InScalarSize = InVT.getScalarSizeInBits();
6656 // x86mmx is not an acceptable vector element type, so don't try.
6657 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6658 // Determine new input vector type. The new input vector type will use
6659 // the same element type (if its a vector) or use the input type as a
6660 // vector. It is the same size as the type to widen to.
6661 EVT NewInVT;
6662 unsigned NewNumParts = WidenSize / InSize;
6663 if (InVT.isVector()) {
6664 EVT InEltVT = InVT.getVectorElementType();
6665 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6666 NumElements: WidenSize / InEltVT.getSizeInBits());
6667 } else {
6668 // For big endian systems, using the promoted input scalar type
6669 // to produce the scalar_to_vector would put the desired bits into
6670 // the least significant byte(s) of the wider element zero. This
6671 // will mean that the users of the result vector are using incorrect
6672 // bits. Use the original input type instead. Although either input
6673 // type can be used on little endian systems, for consistency we
6674 // use the original type there as well.
6675 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6676 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6677 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6678 }
6679
6680 if (TLI.isTypeLegal(VT: NewInVT)) {
6681 SDValue NewVec;
6682 if (InVT.isVector()) {
6683 // Because the result and the input are different vector types, widening
6684 // the result could create a legal type but widening the input might
6685 // make it an illegal type that might lead to repeatedly splitting the
6686 // input and then widening it. To avoid this, we widen the input only if
6687 // it results in a legal type.
6688 if (WidenSize % InSize == 0) {
6689 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6690 Ops[0] = InOp;
6691
6692 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6693 } else {
6694 SmallVector<SDValue, 16> Ops;
6695 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6696 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6697 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6698
6699 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6700 }
6701 } else {
6702 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6703 }
6704 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6705 }
6706 }
6707
6708 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6709}
6710
6711SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6712 return DAG.getNode(
6713 Opcode: N->getOpcode(), DL: SDLoc(N),
6714 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6715 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6716}
6717
6718SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6719 SDLoc dl(N);
6720 // Build a vector with poison for the new nodes.
6721 EVT VT = N->getValueType(ResNo: 0);
6722
6723 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6724 // type. The POISONs need to have the same type as the existing operands.
6725 EVT EltVT = N->getOperand(Num: 0).getValueType();
6726 unsigned NumElts = VT.getVectorNumElements();
6727
6728 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6729 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6730
6731 SmallVector<SDValue, 16> NewOps(N->ops());
6732 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6733 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6734
6735 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6736}
6737
6738SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6739 EVT InVT = N->getOperand(Num: 0).getValueType();
6740 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6741 SDLoc dl(N);
6742 unsigned NumOperands = N->getNumOperands();
6743
6744 bool InputWidened = false; // Indicates we need to widen the input.
6745 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6746 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6747 unsigned NumInElts = InVT.getVectorMinNumElements();
6748 if (WidenNumElts % NumInElts == 0) {
6749 // Add undef vectors to widen to correct length.
6750 unsigned NumConcat = WidenNumElts / NumInElts;
6751 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6752 SmallVector<SDValue, 16> Ops(NumConcat);
6753 for (unsigned i=0; i < NumOperands; ++i)
6754 Ops[i] = N->getOperand(Num: i);
6755 for (unsigned i = NumOperands; i != NumConcat; ++i)
6756 Ops[i] = UndefVal;
6757 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6758 }
6759 } else {
6760 InputWidened = true;
6761 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6762 // The inputs and the result are widen to the same value.
6763 unsigned i;
6764 for (i=1; i < NumOperands; ++i)
6765 if (!N->getOperand(Num: i).isUndef())
6766 break;
6767
6768 if (i == NumOperands)
6769 // Everything but the first operand is an UNDEF so just return the
6770 // widened first operand.
6771 return GetWidenedVector(Op: N->getOperand(Num: 0));
6772
6773 if (NumOperands == 2) {
6774 assert(!WidenVT.isScalableVector() &&
6775 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6776 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6777 unsigned NumInElts = InVT.getVectorNumElements();
6778
6779 // Replace concat of two operands with a shuffle.
6780 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6781 for (unsigned i = 0; i < NumInElts; ++i) {
6782 MaskOps[i] = i;
6783 MaskOps[i + NumInElts] = i + WidenNumElts;
6784 }
6785 return DAG.getVectorShuffle(VT: WidenVT, dl,
6786 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6787 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6788 Mask: MaskOps);
6789 }
6790 }
6791 }
6792
6793 if (WidenVT.isScalableVector()) {
6794 SDValue WideVec = DAG.getPOISON(VT: WidenVT);
6795 unsigned NumInElts = InVT.getVectorMinNumElements();
6796 for (unsigned I = 0; I < NumOperands; ++I)
6797 WideVec =
6798 DAG.getInsertSubvector(DL: dl, Vec: WideVec, SubVec: N->getOperand(Num: I), Idx: I * NumInElts);
6799 return WideVec;
6800 }
6801
6802 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6803 unsigned NumInElts = InVT.getVectorNumElements();
6804
6805 // Fall back to use extracts and build vector.
6806 EVT EltVT = WidenVT.getVectorElementType();
6807 SmallVector<SDValue, 16> Ops(WidenNumElts);
6808 unsigned Idx = 0;
6809 for (unsigned i=0; i < NumOperands; ++i) {
6810 SDValue InOp = N->getOperand(Num: i);
6811 if (InputWidened)
6812 InOp = GetWidenedVector(Op: InOp);
6813 for (unsigned j = 0; j < NumInElts; ++j)
6814 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6815 }
6816 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6817 for (; Idx < WidenNumElts; ++Idx)
6818 Ops[Idx] = UndefVal;
6819 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6820}
6821
6822SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6823 EVT VT = N->getValueType(ResNo: 0);
6824 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6825 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6826 SDValue InOp2 = N->getOperand(Num: 1);
6827 SDValue Idx = N->getOperand(Num: 2);
6828 SDLoc dl(N);
6829 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6830}
6831
6832SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6833 EVT VT = N->getValueType(ResNo: 0);
6834 EVT EltVT = VT.getVectorElementType();
6835 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6836 SDValue InOp = N->getOperand(Num: 0);
6837 SDValue Idx = N->getOperand(Num: 1);
6838 SDLoc dl(N);
6839
6840 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6841 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6842 InOp = GetWidenedVector(Op: InOp);
6843
6844 EVT InVT = InOp.getValueType();
6845
6846 // Check if we can just return the input vector after widening.
6847 uint64_t IdxVal = Idx->getAsZExtVal();
6848 if (IdxVal == 0 && InVT == WidenVT)
6849 return InOp;
6850
6851 // Check if we can extract from the vector.
6852 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6853 unsigned InNumElts = InVT.getVectorMinNumElements();
6854 unsigned VTNumElts = VT.getVectorMinNumElements();
6855 assert(IdxVal % VTNumElts == 0 &&
6856 "Expected Idx to be a multiple of subvector minimum vector length");
6857 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6858 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6859
6860 if (VT.isScalableVector()) {
6861 // Try to split the operation up into smaller extracts and concat the
6862 // results together, e.g.
6863 // nxv6i64 extract_subvector(nxv12i64, 6)
6864 // <->
6865 // nxv8i64 concat(
6866 // nxv2i64 extract_subvector(nxv16i64, 6)
6867 // nxv2i64 extract_subvector(nxv16i64, 8)
6868 // nxv2i64 extract_subvector(nxv16i64, 10)
6869 // undef)
6870 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6871 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6872 "down type's element count");
6873 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6874 EC: ElementCount::getScalable(MinVal: GCD));
6875 // Avoid recursion around e.g. nxv1i8.
6876 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6877 SmallVector<SDValue> Parts;
6878 unsigned I = 0;
6879 for (; I < VTNumElts / GCD; ++I)
6880 Parts.push_back(
6881 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6882 for (; I < WidenNumElts / GCD; ++I)
6883 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6884
6885 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6886 }
6887
6888 // Fallback to extracting through memory.
6889
6890 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6891 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6892 MachineFunction &MF = DAG.getMachineFunction();
6893 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6894 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6895
6896 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6897 PtrInfo, F: MachineMemOperand::MOStore,
6898 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6899 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6900 PtrInfo, F: MachineMemOperand::MOLoad,
6901 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6902
6903 // Write out the input vector.
6904 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6905
6906 // Build a mask to match the length of the non-widened result.
6907 SDValue Mask =
6908 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6909
6910 // Read back the sub-vector setting the remaining lanes to poison.
6911 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6912 return DAG.getMaskedLoad(
6913 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6914 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6915 }
6916
6917 // We could try widening the input to the right length but for now, extract
6918 // the original elements, fill the rest with undefs and build a vector.
6919 SmallVector<SDValue, 16> Ops(WidenNumElts);
6920 unsigned i;
6921 for (i = 0; i < VTNumElts; ++i)
6922 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6923
6924 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6925 for (; i < WidenNumElts; ++i)
6926 Ops[i] = UndefVal;
6927 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6928}
6929
6930SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6931 SDValue InOp = ModifyToType(
6932 InOp: N->getOperand(Num: 0),
6933 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6934 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6935 N2: N->getOperand(Num: 1));
6936}
6937
6938SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6939 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6940 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6941 VT: InOp.getValueType(), N1: InOp,
6942 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6943}
6944
6945/// Either return the same load or provide appropriate casts
6946/// from the load and return that.
6947static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6948 TypeSize LdWidth, TypeSize FirstVTWidth,
6949 SDLoc dl, SelectionDAG &DAG) {
6950 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6951 "Load width must be less than or equal to first value type width");
6952 TypeSize WidenWidth = WidenVT.getSizeInBits();
6953 if (!FirstVT.isVector()) {
6954 unsigned NumElts =
6955 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6956 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6957 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6958 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6959 }
6960 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6961 return LdOp;
6962}
6963
6964/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6965/// widened value so it can be issued in a single atomic store.
6966static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6967 TypeSize FirstVTWidth, const SDLoc &dl,
6968 SelectionDAG &DAG) {
6969 TypeSize WidenWidth = WidenVT.getSizeInBits();
6970 if (!FirstVT.isVector()) {
6971 unsigned NumElts =
6972 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6973 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6974 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: StVal);
6975 return DAG.getExtractVectorElt(DL: dl, VT: FirstVT, Vec: VecOp, Idx: 0);
6976 }
6977 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6978 return StVal;
6979}
6980
6981static std::optional<EVT> findMemType(SelectionDAG &DAG,
6982 const TargetLowering &TLI, unsigned Width,
6983 EVT WidenVT, unsigned Align,
6984 unsigned WidenEx);
6985
6986SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6987 EVT WidenVT =
6988 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6989 EVT LdVT = LD->getMemoryVT();
6990 SDLoc dl(LD);
6991
6992 // Load information
6993 SDValue Chain = LD->getChain();
6994 SDValue BasePtr = LD->getBasePtr();
6995
6996 TypeSize LdWidth = LdVT.getSizeInBits();
6997 TypeSize WidenWidth = WidenVT.getSizeInBits();
6998 TypeSize WidthDiff = WidenWidth - LdWidth;
6999
7000 // Find the vector type that can load from.
7001 std::optional<EVT> FirstVT =
7002 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
7003 WidenEx: WidthDiff.getKnownMinValue());
7004
7005 if (!FirstVT)
7006 return SDValue();
7007
7008 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
7009
7010 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
7011 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
7012
7013 // Load the element with one instruction.
7014 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
7015 FirstVTWidth, dl, DAG);
7016
7017 // Modified the chain - switch anything that used the old chain to use
7018 // the new one.
7019 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
7020 return Result;
7021}
7022
7023SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
7024 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
7025 ISD::LoadExtType ExtType = LD->getExtensionType();
7026
7027 // A vector must always be stored in memory as-is, i.e. without any padding
7028 // between the elements, since various code depend on it, e.g. in the
7029 // handling of a bitcast of a vector type to int, which may be done with a
7030 // vector store followed by an integer load. A vector that does not have
7031 // elements that are byte-sized must therefore be stored as an integer
7032 // built out of the extracted vector elements.
7033 if (!LD->getMemoryVT().isByteSized()) {
7034 SDValue Value, NewChain;
7035 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
7036 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
7037 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
7038 return SDValue();
7039 }
7040
7041 // Generate a vector-predicated load if it is custom/legal on the target. To
7042 // avoid possible recursion, only do this if the widened mask type is legal.
7043 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
7044 // removed from the IR by the ExpandVectorPredication pass but we're
7045 // reintroducing them here.
7046 EVT VT = LD->getValueType(ResNo: 0);
7047 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7048 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
7049
7050 if (ExtType == ISD::NON_EXTLOAD &&
7051 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
7052 TLI.isTypeLegal(VT: WideMaskVT)) {
7053 SDLoc DL(N);
7054 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
7055 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
7056 EC: VT.getVectorElementCount());
7057 SDValue NewLoad =
7058 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
7059 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
7060 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
7061
7062 // Modified the chain - switch anything that used the old chain to use
7063 // the new one.
7064 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7065
7066 return NewLoad;
7067 }
7068
7069 SDValue Result;
7070 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
7071 if (ExtType != ISD::NON_EXTLOAD)
7072 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
7073 else
7074 Result = GenWidenVectorLoads(LdChain, LD);
7075
7076 if (Result) {
7077 // If we generate a single load, we can use that for the chain. Otherwise,
7078 // build a factor node to remember the multiple loads are independent and
7079 // chain to that.
7080 SDValue NewChain;
7081 if (LdChain.size() == 1)
7082 NewChain = LdChain[0];
7083 else
7084 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
7085
7086 // Modified the chain - switch anything that used the old chain to use
7087 // the new one.
7088 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7089
7090 return Result;
7091 }
7092
7093 if (VT.isVector()) {
7094 // If all else fails replace the load with a wide masked load.
7095 SDLoc DL(N);
7096 SDValue Mask =
7097 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
7098
7099 SDValue NewLoad = DAG.getMaskedLoad(
7100 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
7101 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
7102 AM: LD->getAddressingMode(), LD->getExtensionType());
7103
7104 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7105 return NewLoad;
7106 }
7107
7108 report_fatal_error(reason: "Unable to widen vector load");
7109}
7110
7111SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
7112 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7113 SDValue Mask = N->getMask();
7114 SDValue EVL = N->getVectorLength();
7115 ISD::LoadExtType ExtType = N->getExtensionType();
7116 SDLoc dl(N);
7117
7118 // The mask should be widened as well
7119 assert(getTypeAction(Mask.getValueType()) ==
7120 TargetLowering::TypeWidenVector &&
7121 "Unable to widen binary VP op");
7122 Mask = GetWidenedVector(Op: Mask);
7123 assert(Mask.getValueType().getVectorElementCount() ==
7124 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
7125 .getVectorElementCount() &&
7126 "Unable to widen vector load");
7127
7128 SDValue Res =
7129 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
7130 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
7131 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
7132 // Legalize the chain result - switch anything that used the old chain to
7133 // use the new one.
7134 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7135 return Res;
7136}
7137
7138SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
7139 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7140 SDValue Mask = N->getMask();
7141 SDValue EVL = N->getVectorLength();
7142 SDLoc dl(N);
7143
7144 // The mask should be widened as well
7145 assert(getTypeAction(Mask.getValueType()) ==
7146 TargetLowering::TypeWidenVector &&
7147 "Unable to widen binary VP op");
7148 Mask = GetWidenedVector(Op: Mask);
7149 assert(Mask.getValueType().getVectorElementCount() ==
7150 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
7151 .getVectorElementCount() &&
7152 "Unable to widen vector load");
7153
7154 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
7155 Mask, EVL, MMO: N->getMemOperand());
7156 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7157 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
7158 return Res;
7159}
7160
7161SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
7162 SDLoc DL(N);
7163
7164 // The mask should be widened as well
7165 SDValue Mask = N->getMask();
7166 assert(getTypeAction(Mask.getValueType()) ==
7167 TargetLowering::TypeWidenVector &&
7168 "Unable to widen VP strided load");
7169 Mask = GetWidenedVector(Op: Mask);
7170
7171 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7172 assert(Mask.getValueType().getVectorElementCount() ==
7173 WidenVT.getVectorElementCount() &&
7174 "Data and mask vectors should have the same number of elements");
7175
7176 SDValue Res = DAG.getStridedLoadVP(
7177 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
7178 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
7179 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
7180 IsExpanding: N->isExpandingLoad());
7181
7182 // Legalize the chain result - switch anything that used the old chain to
7183 // use the new one.
7184 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7185 return Res;
7186}
7187
7188SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
7189 SDValue Vec = N->getOperand(Num: 0);
7190 SDValue Mask = N->getOperand(Num: 1);
7191 SDValue Passthru = N->getOperand(Num: 2);
7192 EVT WideVecVT =
7193 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
7194 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7195 VT: Mask.getValueType().getVectorElementType(),
7196 EC: WideVecVT.getVectorElementCount());
7197
7198 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
7199 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
7200 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
7201 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
7202 N2: WideMask, N3: WidePassthru);
7203}
7204
7205SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE_VAR(SDNode *N) {
7206 // Only indices that were out of range can select the new source elements.
7207 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7208 EVT WideVT = Src.getValueType();
7209 EVT WideMaskVT = WideVT.changeVectorElementType(
7210 Context&: *DAG.getContext(),
7211 EltVT: N->getOperand(Num: 1).getValueType().getVectorElementType());
7212 SDValue Mask = ModifyToType(InOp: N->getOperand(Num: 1), NVT: WideMaskVT);
7213 return DAG.getNode(Opcode: ISD::VECTOR_SHUFFLE_VAR, DL: SDLoc(N), VT: WideVT, N1: Src, N2: Mask);
7214}
7215
7216SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
7217 EVT VT = N->getValueType(ResNo: 0);
7218 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7219 SDValue Mask = N->getMask();
7220 EVT MaskVT = Mask.getValueType();
7221 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7222 ISD::LoadExtType ExtType = N->getExtensionType();
7223 SDLoc dl(N);
7224
7225 EVT WideMaskVT =
7226 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
7227 EC: WidenVT.getVectorElementCount());
7228
7229 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
7230 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
7231 TLI.isTypeLegal(VT: WideMaskVT) &&
7232 // If there is a passthru, we shouldn't use vp.load. However,
7233 // type legalizer will struggle on masked.load with
7234 // scalable vectors, so for scalable vectors, we still use vp.load
7235 // but manually merge the load result with the passthru using vp.select.
7236 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
7237 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7238 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7239 EC: VT.getVectorElementCount());
7240 SDValue NewLoad =
7241 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
7242 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
7243 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
7244 SDValue NewVal = NewLoad;
7245
7246 // Manually merge with vselect
7247 if (!N->getPassThru()->isUndef()) {
7248 assert(WidenVT.isScalableVector());
7249 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
7250 // The lanes past EVL are poison.
7251 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
7252 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
7253 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
7254 }
7255
7256 // Modified the chain - switch anything that used the old chain to use
7257 // the new one.
7258 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7259
7260 return NewVal;
7261 }
7262
7263 // The mask should be widened as well
7264 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7265
7266 SDValue Res = DAG.getMaskedLoad(
7267 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
7268 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
7269 ExtType, IsExpanding: N->isExpandingLoad());
7270 // Legalize the chain result - switch anything that used the old chain to
7271 // use the new one.
7272 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7273 return Res;
7274}
7275
7276SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
7277
7278 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7279 SDValue Mask = N->getMask();
7280 EVT MaskVT = Mask.getValueType();
7281 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7282 SDValue Scale = N->getScale();
7283 ElementCount WideEC = WideVT.getVectorElementCount();
7284 SDLoc dl(N);
7285
7286 // The mask should be widened as well
7287 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7288 VT: MaskVT.getVectorElementType(), EC: WideEC);
7289 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7290
7291 // Widen the Index operand
7292 SDValue Index = N->getIndex();
7293 EVT WideIndexVT = EVT::getVectorVT(
7294 Context&: *DAG.getContext(), VT: Index.getValueType().getScalarType(), EC: WideEC);
7295 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7296 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7297 Scale };
7298
7299 // Widen the MemoryType
7300 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7301 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7302 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
7303 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
7304 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
7305
7306 // Legalize the chain result - switch anything that used the old chain to
7307 // use the new one.
7308 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7309 return Res;
7310}
7311
7312SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7313 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7314 SDValue Mask = N->getMask();
7315 SDValue Scale = N->getScale();
7316 ElementCount WideEC = WideVT.getVectorElementCount();
7317 SDLoc dl(N);
7318
7319 SDValue Index = GetWidenedVector(Op: N->getIndex());
7320 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7321 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7322 Mask = GetWidenedMask(Mask, EC: WideEC);
7323
7324 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7325 Mask, N->getVectorLength()};
7326 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
7327 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
7328
7329 // Legalize the chain result - switch anything that used the old chain to
7330 // use the new one.
7331 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7332 return Res;
7333}
7334
7335SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7336 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7337 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
7338}
7339
7340// Return true is this is a SETCC node or a strict version of it.
7341static inline bool isSETCCOp(unsigned Opcode) {
7342 switch (Opcode) {
7343 case ISD::SETCC:
7344 case ISD::STRICT_FSETCC:
7345 case ISD::STRICT_FSETCCS:
7346 return true;
7347 }
7348 return false;
7349}
7350
7351// Return true if this is a node that could have two SETCCs as operands.
7352static inline bool isLogicalMaskOp(unsigned Opcode) {
7353 switch (Opcode) {
7354 case ISD::AND:
7355 case ISD::OR:
7356 case ISD::XOR:
7357 return true;
7358 }
7359 return false;
7360}
7361
7362// If N is a SETCC or a strict variant of it, return the type
7363// of the compare operands.
7364static inline EVT getSETCCOperandType(SDValue N) {
7365 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7366 return N->getOperand(Num: OpNo).getValueType();
7367}
7368
7369// This is used just for the assert in convertMask(). Check that this either
7370// a SETCC or a previously handled SETCC by convertMask().
7371#ifndef NDEBUG
7372static inline bool isSETCCorConvertedSETCC(SDValue N) {
7373 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7374 N = N.getOperand(0);
7375 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7376 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7377 if (!N->getOperand(i)->isUndef())
7378 return false;
7379 N = N.getOperand(0);
7380 }
7381
7382 if (N.getOpcode() == ISD::TRUNCATE)
7383 N = N.getOperand(0);
7384 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7385 N = N.getOperand(0);
7386
7387 if (isLogicalMaskOp(N.getOpcode()))
7388 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7389 isSETCCorConvertedSETCC(N.getOperand(1));
7390
7391 return (isSETCCOp(N.getOpcode()) || ISD::isBuildVectorOfConstantSDNodes(N));
7392}
7393#endif
7394
7395// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7396// to ToMaskVT if needed with vector extension or truncation.
7397SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7398 EVT ToMaskVT) {
7399 // Called from convertMaskTree for SETCC leaf nodes. Re-creates the SETCC with
7400 // result type MaskVT, then sign-extends/truncates and pads to ToMaskVT.
7401 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7402
7403 // Make a new Mask node, with a legal result VT.
7404 SDValue Mask;
7405 SmallVector<SDValue, 4> Ops;
7406 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7407 Ops.push_back(Elt: InMask->getOperand(Num: i));
7408 if (InMask->isStrictFPOpcode()) {
7409 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
7410 ResultTys: { MaskVT, MVT::Other }, Ops);
7411 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
7412 }
7413 else
7414 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
7415 Flags: InMask->getFlags());
7416
7417 return adjustMaskToType(Mask, ToMaskVT);
7418}
7419
7420// Adjust element width (sign-extend/truncate) and element count
7421// (extract/concat) of Mask to match ToMaskVT.
7422SDValue DAGTypeLegalizer::adjustMaskToType(SDValue Mask, EVT ToMaskVT) {
7423 LLVMContext &Ctx = *DAG.getContext();
7424 EVT MaskVT = Mask.getValueType();
7425 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7426 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7427 if (MaskScalarBits < ToMaskScalBits) {
7428 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7429 NumElements: MaskVT.getVectorNumElements());
7430 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
7431 } else if (MaskScalarBits > ToMaskScalBits) {
7432 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7433 NumElements: MaskVT.getVectorNumElements());
7434 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
7435 }
7436
7437 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7438 ToMaskVT.getScalarSizeInBits() &&
7439 "Mask should have the right element size by now.");
7440
7441 // Adjust Mask to the right number of elements.
7442 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
7443 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7444 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
7445 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7446 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7447 EVT SubVT = Mask->getValueType(ResNo: 0);
7448 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
7449 SubOps[0] = Mask;
7450 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
7451 }
7452
7453 assert((Mask->getValueType(0) == ToMaskVT) &&
7454 "A mask of ToMaskVT should have been produced by now.");
7455
7456 return Mask;
7457}
7458
7459// Adjust both operands to a common intermediate mask type, picking a scalar
7460// width that minimizes extend/truncate overhead given the final target ToVT.
7461EVT DAGTypeLegalizer::unifyMaskTypes(SDValue &Op0, bool IsOpLenient0,
7462 SDValue &Op1, bool IsOpLenient1,
7463 EVT ToVT) {
7464 assert(Op0.getValueType().getVectorNumElements() ==
7465 Op1.getValueType().getVectorNumElements() &&
7466 "unifyMaskTypes only handles scalar width differences");
7467
7468 // If only one of the operands lenient type-wise, we can simply
7469 // adjust its type to the other operand's type assuming that this
7470 // adjustment can be folded away.
7471 //
7472 // NOTE: We essentially rely on the fact that further optimizations
7473 // do spot redundant casts in "lenient" cases. If at some
7474 // point we decide that we want to do better here, we can
7475 // postpone converting lenient sub-trees right away and postpone
7476 // it to the moment when we know the best fitting integer type
7477 // to materialize them, and do it there.
7478 if (IsOpLenient0 != IsOpLenient1) {
7479 SDValue *LenientOp, *NonLenientOp;
7480 if (IsOpLenient0) {
7481 LenientOp = &Op0;
7482 NonLenientOp = &Op1;
7483 } else {
7484 LenientOp = &Op1;
7485 NonLenientOp = &Op0;
7486 }
7487 EVT OpVT = NonLenientOp->getValueType();
7488 *LenientOp = adjustMaskToType(Mask: *LenientOp, ToMaskVT: OpVT);
7489 return OpVT;
7490 }
7491
7492 unsigned Bits0 = Op0.getScalarValueSizeInBits();
7493 unsigned Bits1 = Op1.getScalarValueSizeInBits();
7494 unsigned NarrowBits = std::min(a: Bits0, b: Bits1);
7495 unsigned WideBits = std::max(a: Bits0, b: Bits1);
7496 unsigned ToBits = ToVT.getScalarSizeInBits();
7497 unsigned IntBits = NarrowBits == WideBits ? NarrowBits
7498 : ToBits >= WideBits ? WideBits
7499 : ToBits <= NarrowBits ? NarrowBits
7500 : ToBits;
7501 EVT OpVT = Op0.getValueType().changeVectorElementType(
7502 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntBits));
7503 Op0 = adjustMaskToType(Mask: Op0, ToMaskVT: OpVT);
7504 Op1 = adjustMaskToType(Mask: Op1, ToMaskVT: OpVT);
7505 return OpVT;
7506}
7507
7508std::pair<SDValue, bool>
7509DAGTypeLegalizer::convertMaskTreeImpl(SDValue V, EVT ToVT, unsigned Depth) {
7510 // The main idea is to recursively traverse VSELECT's mask that needs
7511 // widening to see if we can avoid unnecessary casts. The problem usually
7512 // stems from a simple fact that SETCC might naturally produce results
7513 // not in i1 (as we model it in LLVM IR) and we can continue using that
7514 // type until we have to switch it up. Another important aspect is that
7515 // "all ones" and "all zeros" constants can be materialized at any type,
7516 // so we can try to utilize that to keep SETCC results at their natural
7517 // types as much as possible.
7518 //
7519 // The algorithm traverses the mask-producing tree of operations that
7520 // retain "mask-vector"-ness of the input (i.e. it remains a vector of
7521 // -1s and 0s).
7522 //
7523 // SETCC1 SETCC2 CONST1 SETCC3 CONST2 CONST3
7524 // | / | / | /
7525 // | / | / | /
7526 // |_____/ |______/ |______/
7527 // | * choose the | * choose SETCC3 | * keep it as final type
7528 // | most fitting | type | but consider it subject to
7529 // | type | / change
7530 // | | /
7531 // | |______________/
7532 // | | * choose SETCC3 type
7533 // | /
7534 // | /
7535 // |_____________/
7536 // | * choose the most fitting type
7537 // | and then cast to the final desired type
7538 // |
7539 // VSELECT
7540 //
7541 if (Depth >= DAG.MaxRecursionDepth)
7542 return {};
7543
7544 // Bail out when encounter the vector element count mismatch.
7545 // It potentially can be just an assertion, but we deliberately try to
7546 // be overly conservative here.
7547 if (V.getValueType().getVectorNumElements() != ToVT.getVectorNumElements())
7548 return {};
7549
7550 unsigned Opcode = V.getOpcode();
7551
7552 // Base case: SETCC produces the mask at its natural type.
7553 if (isSETCCOp(Opcode)) {
7554 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: V));
7555 return {convertMask(InMask: V, MaskVT, ToMaskVT: MaskVT), /*IsTypeLenient=*/false};
7556 }
7557
7558 SDLoc DL(V);
7559
7560 // Base case: all-zeros or all-ones BUILD_VECTOR. Type-lenient since these are
7561 // invariant under sign-extend/truncate.
7562 if (ISD::isBuildVectorAllZeros(V))
7563 return {DAG.getConstant(Val: 0, DL, VT: ToVT), /*IsTypeLenient=*/true};
7564 if (ISD::isBuildVectorAllOnes(V))
7565 return {DAG.getAllOnesConstant(DL, VT: ToVT), /*IsTypeLenient=*/true};
7566
7567 // Logical operations (AND/OR/XOR): try picking the best fitting width out
7568 // of children's element widths.
7569 if (isLogicalMaskOp(Opcode)) {
7570 auto [Op0, IsLenientOp0] =
7571 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7572 if (!Op0)
7573 return {};
7574 auto [Op1, IsLenientOp1] =
7575 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7576 if (!Op1)
7577 return {};
7578 EVT OpVT = unifyMaskTypes(Op0, IsOpLenient0: IsLenientOp0, Op1, IsOpLenient1: IsLenientOp1, ToVT);
7579 return {DAG.getNode(Opcode, DL, VT: OpVT, N1: Op0, N2: Op1),
7580 IsLenientOp0 && IsLenientOp1};
7581 }
7582
7583 // FREEZE: widen the operand and re-wrap.
7584 if (Opcode == ISD::FREEZE) {
7585 auto [Inner, IsTypeLenient] =
7586 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7587 if (!Inner)
7588 return {};
7589 return {DAG.getNode(Opcode: ISD::FREEZE, DL, VT: Inner.getValueType(), Operand: Inner),
7590 IsTypeLenient};
7591 }
7592
7593 // Vector shuffle: try inferring the best fitting width from operands.
7594 if (Opcode == ISD::VECTOR_SHUFFLE) {
7595 auto *Shuf = cast<ShuffleVectorSDNode>(Val&: V);
7596 auto [Op0, IsLenientOp0] =
7597 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7598 if (!Op0)
7599 return {};
7600 if (V.getOperand(i: 1).isUndef()) {
7601 EVT OpVT = Op0.getValueType();
7602 return {DAG.getVectorShuffle(VT: OpVT, dl: DL, N1: Op0, N2: DAG.getUNDEF(VT: OpVT),
7603 Mask: Shuf->getMask()),
7604 IsLenientOp0};
7605 }
7606 auto [Op1, IsLenientOp1] =
7607 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7608 if (!Op1)
7609 return {};
7610 EVT OpVT = unifyMaskTypes(Op0, IsOpLenient0: IsLenientOp0, Op1, IsOpLenient1: IsLenientOp1, ToVT);
7611 return {DAG.getVectorShuffle(VT: OpVT, dl: DL, N1: Op0, N2: Op1, Mask: Shuf->getMask()),
7612 IsLenientOp0 && IsLenientOp1};
7613 }
7614
7615 // SELECT/VSELECT: try inferring the best fitting width from operands.
7616 if (Opcode == ISD::SELECT || Opcode == ISD::VSELECT) {
7617 auto [Op1, IsLenientOp1] =
7618 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7619 if (!Op1)
7620 return {};
7621 auto [Op2, IsLenientOp2] =
7622 convertMaskTreeImpl(V: V.getOperand(i: 2), ToVT, Depth: Depth + 1);
7623 if (!Op2)
7624 return {};
7625 EVT OpVT = unifyMaskTypes(Op0&: Op1, IsOpLenient0: IsLenientOp1, Op1&: Op2, IsOpLenient1: IsLenientOp2, ToVT);
7626
7627 // We deliberately skip traversing/modifying VSELECT's mask because
7628 //
7629 // a. We only change bitwidth of the operands and it shouldn't affect
7630 // condition on its own.
7631 //
7632 // b. This VSELECT's mask can be widened in an independent traversal
7633 // if needed.
7634 SDValue Cond = V.getOperand(i: 0);
7635 return {DAG.getNode(Opcode, DL, VT: OpVT, N1: Cond, N2: Op1, N3: Op2),
7636 IsLenientOp1 && IsLenientOp2};
7637 }
7638
7639 return {};
7640}
7641
7642SDValue DAGTypeLegalizer::convertMaskTree(SDValue V, EVT ToVT) {
7643 // In general, we are converting from <N x i1> into <M x iW>.
7644 // This would mean that during the tree traversal we need to pay
7645 // attention to both bitwidth and element count, which can be error-prone.
7646 //
7647 // Instead, we split the task in two, we first widen the type of the tree
7648 // and then change the element count.
7649 EVT MaskTreeVT = ToVT.changeVectorElementCount(
7650 Context&: *DAG.getContext(), EC: V.getValueType().getVectorElementCount());
7651 auto [Result, _] = convertMaskTreeImpl(V, ToVT: MaskTreeVT);
7652 if (!Result)
7653 return Result;
7654 return adjustMaskToType(Mask: Result, ToMaskVT: ToVT);
7655}
7656
7657// This method tries to handle some special cases for the vselect mask
7658// and if needed adjusting the mask vector type to match that of the VSELECT.
7659// Without it, many cases end up with scalarization of the SETCC, with many
7660// unnecessary instructions.
7661SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7662 LLVMContext &Ctx = *DAG.getContext();
7663 SDValue Cond = N->getOperand(Num: 0);
7664
7665 if (N->getOpcode() != ISD::VSELECT)
7666 return SDValue();
7667
7668 // If this is a splitted VSELECT that was previously already handled, do
7669 // nothing.
7670 EVT CondVT = Cond->getValueType(ResNo: 0);
7671 if (CondVT.getScalarSizeInBits() != 1)
7672 return SDValue();
7673
7674 EVT VSelVT = N->getValueType(ResNo: 0);
7675
7676 // This method can't handle scalable vector types.
7677 // FIXME: This support could be added in the future.
7678 if (VSelVT.isScalableVector())
7679 return SDValue();
7680
7681 // Only handle vector types which are a power of 2.
7682 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
7683 return SDValue();
7684
7685 // Don't touch if this will be scalarized.
7686 EVT FinalVT = VSelVT;
7687 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
7688 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
7689
7690 if (FinalVT.getVectorNumElements() == 1)
7691 return SDValue();
7692
7693 // If there is support for an i1 vector mask, don't touch.
7694 if (isSETCCOp(Opcode: Cond.getOpcode())) {
7695 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
7696 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
7697 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
7698 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
7699 if (SetCCResVT.getScalarSizeInBits() == 1)
7700 return SDValue();
7701 } else if (CondVT.getScalarType() == MVT::i1) {
7702 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7703 // don't touch.
7704 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
7705 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
7706
7707 if (CondVT.getScalarType() == MVT::i1)
7708 return SDValue();
7709 }
7710
7711 // Widen the vselect result type if needed.
7712 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
7713 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
7714
7715 // The mask of the VSELECT should have integer elements.
7716 EVT ToMaskVT = VSelVT;
7717 if (!ToMaskVT.getScalarType().isInteger())
7718 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7719
7720 // Try to recursively widen the mask expression tree to the target type.
7721 return convertMaskTree(V: Cond, ToVT: ToMaskVT);
7722}
7723
7724SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7725 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7726 ElementCount WidenEC = WidenVT.getVectorElementCount();
7727
7728 SDValue Cond1 = N->getOperand(Num: 0);
7729 EVT CondVT = Cond1.getValueType();
7730 unsigned Opcode = N->getOpcode();
7731 if (CondVT.isVector()) {
7732 if (SDValue WideCond = WidenVSELECTMask(N)) {
7733 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7734 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7735 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7736 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7737 }
7738
7739 EVT CondEltVT = CondVT.getVectorElementType();
7740 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7741 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7742 Cond1 = GetWidenedVector(Op: Cond1);
7743
7744 // If we have to split the condition there is no point in widening the
7745 // select. This would result in an cycle of widening the select ->
7746 // widening the condition operand -> splitting the condition operand ->
7747 // splitting the select -> widening the select. Instead split this select
7748 // further and widen the resulting type.
7749 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7750 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7751 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7752 return Res;
7753 }
7754
7755 if (Cond1.getValueType() != CondWidenVT)
7756 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7757 }
7758
7759 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7760 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7761 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7762 if (Opcode == ISD::VP_MERGE)
7763 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7764 N4: N->getOperand(Num: 3));
7765 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7766}
7767
7768SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7769 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7770 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7771 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7772 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7773 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7774}
7775
7776SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7777 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7778 return DAG.getUNDEF(VT: WidenVT);
7779}
7780
7781SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7782 EVT VT = N->getValueType(ResNo: 0);
7783 SDLoc dl(N);
7784
7785 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7786 unsigned NumElts = VT.getVectorNumElements();
7787 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7788
7789 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7790 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7791
7792 // Adjust mask based on new input vector length.
7793 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7794 for (unsigned i = 0; i != NumElts; ++i) {
7795 int Idx = N->getMaskElt(Idx: i);
7796 if (Idx < (int)NumElts)
7797 NewMask[i] = Idx;
7798 else
7799 NewMask[i] = Idx - NumElts + WidenNumElts;
7800 }
7801 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7802}
7803
7804SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7805 EVT VT = N->getValueType(ResNo: 0);
7806 EVT EltVT = VT.getVectorElementType();
7807 SDLoc dl(N);
7808
7809 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7810 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7811 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7812
7813 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7814 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7815 unsigned VTNumElts = VT.getVectorMinNumElements();
7816 unsigned IdxVal = WidenNumElts - VTNumElts;
7817
7818 if (VT.isScalableVector()) {
7819 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7820 // results together, e.g.(nxv6i64 -> nxv8i64)
7821 // nxv8i64 vector_reverse
7822 // <->
7823 // nxv8i64 concat(
7824 // nxv2i64 extract_subvector(nxv8i64, 2)
7825 // nxv2i64 extract_subvector(nxv8i64, 4)
7826 // nxv2i64 extract_subvector(nxv8i64, 6)
7827 // nxv2i64 undef)
7828
7829 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7830 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7831 EC: ElementCount::getScalable(MinVal: GCD));
7832 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7833 "down type's element count");
7834 SmallVector<SDValue> Parts;
7835 unsigned i = 0;
7836 for (; i < VTNumElts / GCD; ++i)
7837 Parts.push_back(
7838 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7839 for (; i < WidenNumElts / GCD; ++i)
7840 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7841
7842 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7843 }
7844
7845 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7846 // fixed-vectors.
7847 SmallVector<int, 16> Mask(WidenNumElts, -1);
7848 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7849
7850 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7851 Mask);
7852}
7853
7854SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7855 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7856 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7857}
7858
7859void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
7860 EVT VT = N->getValueType(ResNo: 0);
7861 EVT EltVT = VT.getVectorElementType();
7862 ElementCount OrigEC = VT.getVectorElementCount();
7863 unsigned Factor = N->getNumOperands();
7864 SDLoc DL(N);
7865
7866 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7867 ElementCount WidenEC = WidenVT.getVectorElementCount();
7868
7869 SmallVector<SDValue, 8> WidenOps(Factor);
7870 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7871 WidenOps[Idx] = GetWidenedVector(Op: N->getOperand(Num: Idx));
7872
7873 SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
7874 SDValue Interleaved =
7875 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: WidenVTs, Ops: WidenOps);
7876
7877 EVT PackedWidenVT =
7878 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7879 SmallVector<SDValue, 8> Slices(Factor);
7880 for (unsigned Idx = 0; Idx != Factor; ++Idx)
7881 Slices[Idx] = Interleaved.getValue(R: Idx);
7882
7883 SDValue Packed = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PackedWidenVT, Ops: Slices);
7884
7885 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7886 SDValue Narrow = DAG.getExtractSubvector(DL, VT, Vec: Packed,
7887 Idx: OrigEC.getKnownMinValue() * Idx);
7888 SDValue Wide =
7889 DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: Narrow, /*Idx=*/0U);
7890 SetWidenedVector(Op: SDValue(N, Idx), Result: Wide);
7891 }
7892}
7893
7894SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
7895 SDLoc DL(N);
7896 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7897 EVT SourceVT = N->getOperand(Num: 0).getValueType();
7898 EVT WideSourceVT =
7899 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SourceVT.getVectorElementType(),
7900 EC: WidenVT.getVectorElementCount());
7901
7902 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
7903 SubVec: N->getOperand(Num: 0), Idx: 0);
7904 SDValue WideMask = DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT),
7905 SubVec: N->getOperand(Num: 2), Idx: 0);
7906 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
7907 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
7908}
7909
7910void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7911 EVT VT = N->getValueType(ResNo: 0);
7912 EVT EltVT = VT.getVectorElementType();
7913 ElementCount OrigEC = VT.getVectorElementCount();
7914 unsigned Factor = N->getNumOperands();
7915 SDLoc DL(N);
7916
7917 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7918 ElementCount WidenEC = WidenVT.getVectorElementCount();
7919 // We cannot just use the widened operands directly: since they might be
7920 // individually widened, using them directly will result in de-interleaving
7921 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7922 // not concat the widened operands but the original ones to effectively
7923 // generate a "packed" concated and widened vector, before extracting new
7924 // operand vectors with the widened type.
7925 EVT PackedWidenVT =
7926 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7927 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: OrigEC * Factor);
7928 SDValue ConcatOp = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops: N->ops());
7929 SDValue PackedWidenVec = DAG.getInsertSubvector(
7930 DL, Vec: DAG.getUNDEF(VT: PackedWidenVT), SubVec: ConcatOp, /*Idx=*/0U);
7931
7932 // Extract the new widened operand vectors.
7933 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7934 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7935 NewOps[Idx] = DAG.getExtractSubvector(DL, VT: WidenVT, Vec: PackedWidenVec,
7936 Idx: WidenEC.getKnownMinValue() * Idx);
7937 }
7938
7939 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7940 SDValue NewRes = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: NewVTs, Ops: NewOps);
7941 // Set the widened results manually.
7942 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7943 SetWidenedVector(Op: SDValue(N, Idx), Result: NewRes.getValue(R: Idx));
7944}
7945
7946SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7947 assert(N->getValueType(0).isVector() &&
7948 N->getOperand(0).getValueType().isVector() &&
7949 "Operands must be vectors");
7950 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7951 ElementCount WidenEC = WidenVT.getVectorElementCount();
7952
7953 SDValue InOp1 = N->getOperand(Num: 0);
7954 EVT InVT = InOp1.getValueType();
7955 assert(InVT.isVector() && "can not widen non-vector type");
7956 EVT WidenInVT =
7957 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7958
7959 // The input and output types often differ here, and it could be that while
7960 // we'd prefer to widen the result type, the input operands have been split.
7961 // In this case, we also need to split the result of this node as well.
7962 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7963 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7964 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7965 return Res;
7966 }
7967
7968 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7969 SDValue InOp2 = N->getOperand(Num: 1);
7970 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7971 InOp1 = GetWidenedVector(Op: InOp1);
7972 InOp2 = GetWidenedVector(Op: InOp2);
7973 } else {
7974 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7975 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7976 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7977 N2: InOp1, N3: ZeroIdx);
7978 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7979 N2: InOp2, N3: ZeroIdx);
7980 }
7981
7982 // Assume that the input and output will be widen appropriately. If not,
7983 // we will have to unroll it at some point.
7984 assert(InOp1.getValueType() == WidenInVT &&
7985 InOp2.getValueType() == WidenInVT &&
7986 "Input not widened to expected type!");
7987 (void)WidenInVT;
7988 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7989 N3: N->getOperand(Num: 2));
7990}
7991
7992SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7993 assert(N->getValueType(0).isVector() &&
7994 N->getOperand(1).getValueType().isVector() &&
7995 "Operands must be vectors");
7996 EVT VT = N->getValueType(ResNo: 0);
7997 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7998 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7999 unsigned NumElts = VT.getVectorNumElements();
8000 EVT EltVT = VT.getVectorElementType();
8001
8002 SDLoc dl(N);
8003 SDValue Chain = N->getOperand(Num: 0);
8004 SDValue LHS = N->getOperand(Num: 1);
8005 SDValue RHS = N->getOperand(Num: 2);
8006 SDValue CC = N->getOperand(Num: 3);
8007 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8008
8009 // Fully unroll and reassemble.
8010 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
8011 SmallVector<SDValue, 8> Chains(NumElts);
8012 for (unsigned i = 0; i != NumElts; ++i) {
8013 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8014 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8015
8016 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8017 Ops: {Chain, LHSElem, RHSElem, CC});
8018 Chains[i] = Scalars[i].getValue(R: 1);
8019 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8020 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8021 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8022 }
8023
8024 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8025 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8026
8027 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
8028}
8029
8030SDValue DAGTypeLegalizer::WidenVecRes_PARTIAL_REDUCE_MLA(SDNode *N) {
8031 SDLoc DL(N);
8032 EVT VT = N->getValueType(ResNo: 0);
8033
8034 // Expand, then widen the result.
8035 SDValue Expanded = TLI.expandPartialReduceMLA(Node: N, DAG);
8036 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
8037 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WideVT), SubVec: Expanded, Idx: 0);
8038}
8039
8040//===----------------------------------------------------------------------===//
8041// Widen Vector Operand
8042//===----------------------------------------------------------------------===//
8043bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
8044 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
8045 SDValue Res = SDValue();
8046
8047 // See if the target wants to custom widen this node.
8048 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
8049 return false;
8050
8051 switch (N->getOpcode()) {
8052 default:
8053#ifndef NDEBUG
8054 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
8055 N->dump(&DAG);
8056 dbgs() << "\n";
8057#endif
8058 report_fatal_error(reason: "do not know how to widen this operator's operand!");
8059
8060 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
8061 case ISD::FAKE_USE:
8062 Res = WidenVecOp_FAKE_USE(N);
8063 break;
8064 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
8065 case ISD::VECTOR_REPEAT:
8066 Res = WidenVecOp_VECTOR_REPEAT(N);
8067 break;
8068 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
8069 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
8070 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
8071 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
8072 case ISD::ATOMIC_STORE:
8073 Res = WidenVecOp_ATOMIC_STORE(ST: cast<AtomicSDNode>(Val: N));
8074 break;
8075 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
8076 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8077 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
8078 break;
8079 case ISD::ANY_EXTEND_VECTOR_INREG:
8080 case ISD::SIGN_EXTEND_VECTOR_INREG:
8081 case ISD::ZERO_EXTEND_VECTOR_INREG:
8082 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
8083 break;
8084 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
8085 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
8086 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
8087 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
8088 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
8089 case ISD::STRICT_FSETCC:
8090 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
8091 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
8092 case ISD::FLDEXP:
8093 case ISD::FCOPYSIGN:
8094 case ISD::LROUND:
8095 case ISD::LLROUND:
8096 case ISD::LRINT:
8097 case ISD::LLRINT:
8098 Res = WidenVecOp_UnrollVectorOp(N);
8099 break;
8100 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
8101
8102 case ISD::ANY_EXTEND:
8103 case ISD::SIGN_EXTEND:
8104 case ISD::ZERO_EXTEND:
8105 Res = WidenVecOp_EXTEND(N);
8106 break;
8107
8108 case ISD::SCMP:
8109 case ISD::UCMP:
8110 Res = WidenVecOp_CMP(N);
8111 break;
8112
8113 case ISD::FP_EXTEND:
8114 case ISD::STRICT_FP_EXTEND:
8115 case ISD::FP_ROUND:
8116 case ISD::STRICT_FP_ROUND:
8117 case ISD::FP_TO_SINT:
8118 case ISD::STRICT_FP_TO_SINT:
8119 case ISD::FP_TO_UINT:
8120 case ISD::STRICT_FP_TO_UINT:
8121 case ISD::SINT_TO_FP:
8122 case ISD::STRICT_SINT_TO_FP:
8123 case ISD::UINT_TO_FP:
8124 case ISD::STRICT_UINT_TO_FP:
8125 case ISD::TRUNCATE:
8126 case ISD::CONVERT_FROM_ARBITRARY_FP:
8127 case ISD::CONVERT_TO_ARBITRARY_FP:
8128 Res = WidenVecOp_Convert(N);
8129 break;
8130
8131 case ISD::FP_TO_SINT_SAT:
8132 case ISD::FP_TO_UINT_SAT:
8133 Res = WidenVecOp_FP_TO_XINT_SAT(N);
8134 break;
8135
8136 case ISD::VECREDUCE_FADD:
8137 case ISD::VECREDUCE_FMUL:
8138 case ISD::VECREDUCE_ADD:
8139 case ISD::VECREDUCE_MUL:
8140 case ISD::VECREDUCE_AND:
8141 case ISD::VECREDUCE_OR:
8142 case ISD::VECREDUCE_XOR:
8143 case ISD::VECREDUCE_SMAX:
8144 case ISD::VECREDUCE_SMIN:
8145 case ISD::VECREDUCE_UMAX:
8146 case ISD::VECREDUCE_UMIN:
8147 case ISD::VECREDUCE_FMAX:
8148 case ISD::VECREDUCE_FMIN:
8149 case ISD::VECREDUCE_FMAXIMUM:
8150 case ISD::VECREDUCE_FMINIMUM:
8151 case ISD::VECREDUCE_FMAXIMUMNUM:
8152 case ISD::VECREDUCE_FMINIMUMNUM:
8153 Res = WidenVecOp_VECREDUCE(N);
8154 break;
8155 case ISD::VECREDUCE_SEQ_FADD:
8156 case ISD::VECREDUCE_SEQ_FMUL:
8157 Res = WidenVecOp_VECREDUCE_SEQ(N);
8158 break;
8159 case ISD::VP_REDUCE_FADD:
8160 case ISD::VP_REDUCE_SEQ_FADD:
8161 case ISD::VP_REDUCE_FMUL:
8162 case ISD::VP_REDUCE_SEQ_FMUL:
8163 case ISD::VP_REDUCE_ADD:
8164 case ISD::VP_REDUCE_MUL:
8165 case ISD::VP_REDUCE_AND:
8166 case ISD::VP_REDUCE_OR:
8167 case ISD::VP_REDUCE_XOR:
8168 case ISD::VP_REDUCE_SMAX:
8169 case ISD::VP_REDUCE_SMIN:
8170 case ISD::VP_REDUCE_UMAX:
8171 case ISD::VP_REDUCE_UMIN:
8172 case ISD::VP_REDUCE_FMAX:
8173 case ISD::VP_REDUCE_FMIN:
8174 case ISD::VP_REDUCE_FMAXIMUM:
8175 case ISD::VP_REDUCE_FMINIMUM:
8176 Res = WidenVecOp_VP_REDUCE(N);
8177 break;
8178 case ISD::CTTZ_ELTS:
8179 case ISD::CTTZ_ELTS_ZERO_POISON:
8180 Res = WidenVecOp_CttzElements(N);
8181 break;
8182 case ISD::VP_CTTZ_ELTS:
8183 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8184 Res = WidenVecOp_VP_CttzElements(N);
8185 break;
8186 case ISD::VECTOR_FIND_LAST_ACTIVE:
8187 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
8188 break;
8189 case ISD::VECTOR_SHUFFLE_VAR:
8190 Res = WidenVecOp_VECTOR_SHUFFLE_VAR(N, OpNo);
8191 break;
8192 case ISD::VECTOR_MATCH:
8193 Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
8194 break;
8195 case ISD::PARTIAL_REDUCE_UMLA:
8196 case ISD::PARTIAL_REDUCE_SMLA:
8197 case ISD::PARTIAL_REDUCE_SUMLA:
8198 case ISD::PARTIAL_REDUCE_FMLA:
8199 Res = WidenVecOp_PARTIAL_REDUCE_MLA(N);
8200 break;
8201 }
8202
8203 // If Res is null, the sub-method took care of registering the result.
8204 if (!Res.getNode()) return false;
8205
8206 // If the result is N, the sub-method updated N in place. Tell the legalizer
8207 // core about this.
8208 if (Res.getNode() == N)
8209 return true;
8210
8211
8212 if (N->isStrictFPOpcode())
8213 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
8214 "Invalid operand expansion");
8215 else
8216 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
8217 "Invalid operand expansion");
8218
8219 ReplaceValueWith(From: SDValue(N, 0), To: Res);
8220 return false;
8221}
8222
8223SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
8224 SDLoc DL(N);
8225 EVT VT = N->getValueType(ResNo: 0);
8226
8227 SDValue InOp = N->getOperand(Num: 0);
8228 assert(getTypeAction(InOp.getValueType()) ==
8229 TargetLowering::TypeWidenVector &&
8230 "Unexpected type action");
8231 InOp = GetWidenedVector(Op: InOp);
8232 assert(VT.getVectorNumElements() <
8233 InOp.getValueType().getVectorNumElements() &&
8234 "Input wasn't widened!");
8235
8236 // We may need to further widen the operand until it has the same total
8237 // vector size as the result.
8238 EVT InVT = InOp.getValueType();
8239 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
8240 EVT InEltVT = InVT.getVectorElementType();
8241 for (EVT FixedVT : MVT::vector_valuetypes()) {
8242 EVT FixedEltVT = FixedVT.getVectorElementType();
8243 if (TLI.isTypeLegal(VT: FixedVT) &&
8244 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
8245 FixedEltVT == InEltVT) {
8246 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
8247 "Not enough elements in the fixed type for the operand!");
8248 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
8249 "We can't have the same type as we started with!");
8250 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
8251 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
8252 else
8253 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
8254 break;
8255 }
8256 }
8257 InVT = InOp.getValueType();
8258 if (InVT.getSizeInBits() != VT.getSizeInBits())
8259 // We couldn't find a legal vector type that was a widening of the input
8260 // and could be extended in-register to the result type, so we have to
8261 // scalarize.
8262 return WidenVecOp_Convert(N);
8263 }
8264
8265 // Use special DAG nodes to represent the operation of extending the
8266 // low lanes.
8267 switch (N->getOpcode()) {
8268 default:
8269 llvm_unreachable("Extend legalization on extend operation!");
8270 case ISD::ANY_EXTEND:
8271 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8272 case ISD::SIGN_EXTEND:
8273 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8274 case ISD::ZERO_EXTEND:
8275 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8276 }
8277}
8278
8279SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
8280 SDLoc dl(N);
8281
8282 EVT OpVT = N->getOperand(Num: 0).getValueType();
8283 EVT ResVT = N->getValueType(ResNo: 0);
8284 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
8285 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8286
8287 // 1. EXTRACT_SUBVECTOR
8288 // 2. SIGN_EXTEND/ZERO_EXTEND
8289 // 3. CMP
8290 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
8291 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
8292
8293 // At this point the result type is guaranteed to be valid, so we can use it
8294 // as the operand type by extending it appropriately
8295 ISD::NodeType ExtendOpcode =
8296 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8297 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
8298 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
8299
8300 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
8301}
8302
8303SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
8304 // The result (and first input) is legal, but the second input is illegal.
8305 // We can't do much to fix that, so just unroll and let the extracts off of
8306 // the second input be widened as needed later.
8307 return DAG.UnrollVectorOp(N);
8308}
8309
8310SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
8311 SDLoc DL(N);
8312 EVT ResultVT = N->getValueType(ResNo: 0);
8313 SDValue Test = N->getOperand(Num: 1);
8314 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
8315
8316 // Process this node similarly to SETCC.
8317 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
8318 if (ResultVT.getScalarType() == MVT::i1)
8319 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8320 NumElements: WideResultVT.getVectorNumElements());
8321
8322 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
8323 Ops: {WideArg, Test}, Flags: N->getFlags());
8324
8325 // Extract the needed results from the result vector.
8326 EVT ResVT =
8327 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
8328 NumElements: ResultVT.getVectorNumElements());
8329 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
8330
8331 EVT OpVT = N->getOperand(Num: 0).getValueType();
8332 ISD::NodeType ExtendCode =
8333 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8334 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
8335}
8336
8337SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
8338 // Since the result is legal and the input is illegal.
8339 EVT VT = N->getValueType(ResNo: 0);
8340 EVT EltVT = VT.getVectorElementType();
8341 SDLoc dl(N);
8342 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
8343 assert(getTypeAction(InOp.getValueType()) ==
8344 TargetLowering::TypeWidenVector &&
8345 "Unexpected type action");
8346 InOp = GetWidenedVector(Op: InOp);
8347 EVT InVT = InOp.getValueType();
8348 unsigned Opcode = N->getOpcode();
8349
8350 // Helper to build a convert node with all scalar trailing operands.
8351 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
8352 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
8353 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
8354 N4: N->getOperand(Num: 3));
8355 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
8356 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1));
8357 return DAG.getNode(Opcode, DL: dl, VT, Operand: Op);
8358 };
8359
8360 // See if a widened result type would be legal, if so widen the node.
8361 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
8362 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
8363 EC: InVT.getVectorElementCount());
8364 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
8365 SDValue Res;
8366 if (N->isStrictFPOpcode()) {
8367 if (Opcode == ISD::STRICT_FP_ROUND)
8368 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
8369 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
8370 else
8371 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
8372 Ops: { N->getOperand(Num: 0), InOp });
8373 // Legalize the chain result - switch anything that used the old chain to
8374 // use the new one.
8375 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8376 } else {
8377 Res = MakeConvertNode(WideVT, InOp);
8378 }
8379 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
8380 }
8381
8382 EVT InEltVT = InVT.getVectorElementType();
8383
8384 // Unroll the convert into some scalar code and create a nasty build vector.
8385 unsigned NumElts = VT.getVectorNumElements();
8386 SmallVector<SDValue, 16> Ops(NumElts);
8387 if (N->isStrictFPOpcode()) {
8388 SmallVector<SDValue, 4> NewOps(N->ops());
8389 SmallVector<SDValue, 32> OpChains;
8390 for (unsigned i=0; i < NumElts; ++i) {
8391 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
8392 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
8393 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
8394 }
8395 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
8396 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8397 } else {
8398 for (unsigned i = 0; i < NumElts; ++i) {
8399 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
8400 Ops[i] = MakeConvertNode(EltVT, Elt);
8401 }
8402 }
8403
8404 return DAG.getBuildVector(VT, DL: dl, Ops);
8405}
8406
8407SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
8408 EVT DstVT = N->getValueType(ResNo: 0);
8409 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
8410 EVT SrcVT = Src.getValueType();
8411 ElementCount WideNumElts = SrcVT.getVectorElementCount();
8412 SDLoc dl(N);
8413
8414 // See if a widened result type would be legal, if so widen the node.
8415 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8416 VT: DstVT.getVectorElementType(), EC: WideNumElts);
8417 if (TLI.isTypeLegal(VT: WideDstVT)) {
8418 SDValue Res =
8419 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
8420 return DAG.getNode(
8421 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
8422 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
8423 }
8424
8425 // Give up and unroll.
8426 return DAG.UnrollVectorOp(N);
8427}
8428
8429SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
8430 EVT VT = N->getValueType(ResNo: 0);
8431 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8432 EVT InWidenVT = InOp.getValueType();
8433 SDLoc dl(N);
8434
8435 // Check if we can convert between two legal vector types and extract.
8436 TypeSize InWidenSize = InWidenVT.getSizeInBits();
8437 TypeSize Size = VT.getSizeInBits();
8438 // x86mmx is not an acceptable vector element type, so don't try.
8439 if (!VT.isVector() && VT != MVT::x86mmx &&
8440 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
8441 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
8442 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
8443 if (TLI.isTypeLegal(VT: NewVT)) {
8444 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8445 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
8446 }
8447 }
8448
8449 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
8450 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
8451 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
8452 // having to copy via memory.
8453 if (VT.isVector()) {
8454 EVT EltVT = VT.getVectorElementType();
8455 unsigned EltSize = EltVT.getFixedSizeInBits();
8456 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
8457 ElementCount NewNumElts =
8458 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
8459 .divideCoefficientBy(RHS: EltSize);
8460 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
8461 if (TLI.isTypeLegal(VT: NewVT)) {
8462 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8463 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
8464 }
8465 }
8466 }
8467
8468 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
8469}
8470
8471// Vectors with sizes that are not powers of 2 need to be widened to the
8472// next largest power of 2. For example, we may get a vector of 3 32-bit
8473// integers or of 6 16-bit integers, both of which have to be widened to a
8474// 128-bit vector.
8475SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8476 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
8477 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
8478 N2: WidenedOp);
8479}
8480
8481SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8482 EVT VT = N->getValueType(ResNo: 0);
8483 EVT EltVT = VT.getVectorElementType();
8484 EVT InVT = N->getOperand(Num: 0).getValueType();
8485 SDLoc dl(N);
8486
8487 // If the widen width for this operand is the same as the width of the concat
8488 // and all but the first operand is undef, just use the widened operand.
8489 unsigned NumOperands = N->getNumOperands();
8490 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
8491 unsigned i;
8492 for (i = 1; i < NumOperands; ++i)
8493 if (!N->getOperand(Num: i).isUndef())
8494 break;
8495
8496 if (i == NumOperands)
8497 return GetWidenedVector(Op: N->getOperand(Num: 0));
8498 }
8499
8500 if (VT.isScalableVector()) {
8501 SDValue Result = DAG.getPOISON(VT);
8502 unsigned NumInElts = InVT.getVectorMinNumElements();
8503 for (unsigned i = 0; i < NumOperands; ++i) {
8504 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: i));
8505 if (InOp.getValueType() != InVT)
8506 InOp = DAG.getExtractSubvector(DL: dl, VT: InVT, Vec: InOp, Idx: 0);
8507 Result = DAG.getInsertSubvector(DL: dl, Vec: Result, SubVec: InOp, Idx: i * NumInElts);
8508 }
8509 return Result;
8510 }
8511
8512 // Otherwise, fall back to a nasty build vector.
8513 unsigned NumElts = VT.getVectorNumElements();
8514 SmallVector<SDValue, 16> Ops(NumElts);
8515
8516 unsigned NumInElts = InVT.getVectorNumElements();
8517
8518 unsigned Idx = 0;
8519 for (unsigned i=0; i < NumOperands; ++i) {
8520 SDValue InOp = N->getOperand(Num: i);
8521 assert(getTypeAction(InOp.getValueType()) ==
8522 TargetLowering::TypeWidenVector &&
8523 "Unexpected type action");
8524 InOp = GetWidenedVector(Op: InOp);
8525 for (unsigned j = 0; j < NumInElts; ++j)
8526 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
8527 }
8528 return DAG.getBuildVector(VT, DL: dl, Ops);
8529}
8530
8531SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_REPEAT(SDNode *N) {
8532 SDLoc DL(N);
8533 EVT VT = N->getValueType(ResNo: 0);
8534 SDValue Src = N->getOperand(Num: 0);
8535 EVT SrcVT = Src.getValueType();
8536 EVT WidenedSrcVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SrcVT);
8537
8538 if (!WidenedSrcVT.getVectorElementCount().hasKnownScalarFactor(
8539 RHS: SrcVT.getVectorElementCount()))
8540 report_fatal_error(
8541 reason: "Cannot widen VECTOR_REPEAT operand to an ElementCount that's not "
8542 "a known scalar multiple of the input ElementCount.");
8543
8544 // Repeat the original source because the extra lanes of its widened value
8545 // are unspecified.
8546 unsigned NumConcat =
8547 WidenedSrcVT.getVectorNumElements() / SrcVT.getVectorNumElements();
8548 SmallVector<SDValue, 8> Ops(NumConcat, Src);
8549 SDValue WidenedSrc = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WidenedSrcVT, Ops);
8550 EVT WidenedVT = VT.changeVectorElementCount(
8551 Context&: *DAG.getContext(),
8552 EC: ElementCount::getScalable(MinVal: WidenedSrcVT.getVectorNumElements()));
8553 SDValue Widened = DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: WidenedVT, Operand: WidenedSrc);
8554 return DAG.getExtractSubvector(DL, VT, Vec: Widened, Idx: 0);
8555}
8556
8557SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8558 EVT VT = N->getValueType(ResNo: 0);
8559 SDValue SubVec = N->getOperand(Num: 1);
8560 SDValue InVec = N->getOperand(Num: 0);
8561
8562 EVT OrigVT = SubVec.getValueType();
8563 SubVec = GetWidenedVector(Op: SubVec);
8564 EVT SubVT = SubVec.getValueType();
8565
8566 // Whether or not all the elements of the widened SubVec will be inserted into
8567 // valid indices of VT.
8568 bool IndicesValid = false;
8569 // If we statically know that VT can fit SubVT, the indices are valid.
8570 if (VT.knownBitsGE(VT: SubVT))
8571 IndicesValid = true;
8572 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8573 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8574 // we know the minimum vscale we can work out if it's valid ourselves.
8575 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8576 Kind: Attribute::VScaleRange);
8577 if (Attr.isValid()) {
8578 unsigned VScaleMin = Attr.getVScaleRangeMin();
8579 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8580 SubVT.getFixedSizeInBits())
8581 IndicesValid = true;
8582 }
8583 }
8584
8585 if (!IndicesValid)
8586 report_fatal_error(
8587 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
8588
8589 SDLoc DL(N);
8590
8591 // We need to make sure that the indices are still valid, otherwise we might
8592 // widen what was previously well-defined to something undefined.
8593 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
8594 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
8595 N3: N->getOperand(Num: 2));
8596
8597 if (OrigVT.isScalableVector()) {
8598 // When the widened types match, overwriting the start of a vector is
8599 // effectively a merge operation that can be implement as a vselect.
8600 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
8601 SDValue Mask =
8602 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
8603 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
8604 }
8605
8606 // Fallback to inserting through memory.
8607 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8608 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
8609 MachineFunction &MF = DAG.getMachineFunction();
8610 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
8611 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
8612
8613 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8614 PtrInfo, F: MachineMemOperand::MOStore,
8615 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8616 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8617 PtrInfo, F: MachineMemOperand::MOLoad,
8618 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8619
8620 // Write out the vector being inserting into.
8621 SDValue Ch =
8622 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
8623
8624 // Build a mask to match the length of the sub-vector.
8625 SDValue Mask =
8626 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
8627
8628 // Overwrite the sub-vector at the required offset.
8629 SDValue SubVecPtr =
8630 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
8631 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
8632 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
8633 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
8634
8635 // Read back the result.
8636 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
8637 }
8638
8639 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8640 // with a series of INSERT_VECTOR_ELT
8641 unsigned Idx = N->getConstantOperandVal(Num: 2);
8642
8643 SDValue InsertElt = InVec;
8644 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8645 SDValue ExtractElt =
8646 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
8647 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
8648 }
8649
8650 return InsertElt;
8651}
8652
8653SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8654 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8655 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
8656 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8657}
8658
8659SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8660 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8661 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
8662 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8663}
8664
8665SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8666 SDLoc DL(N);
8667 EVT ResVT = N->getValueType(ResNo: 0);
8668
8669 // Widen the input as requested by the legalizer.
8670 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8671 EVT WideInVT = WideInOp.getValueType();
8672
8673 // Simple case: if widened input is still smaller than or equal to result,
8674 // just use it directly.
8675 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8676 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
8677
8678 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8679 // If widening makes the input larger than the original result, widen the
8680 // result to match, then extract back down.
8681 EVT ResEltVT = ResVT.getVectorElementType();
8682 unsigned EltBits = ResEltVT.getSizeInBits();
8683 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8684 "Widened input size must be a multiple of result element size");
8685
8686 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8687 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
8688
8689 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
8690 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
8691}
8692
8693SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8694 // We have to widen the value, but we want only to store the original
8695 // vector type.
8696 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
8697
8698 if (!ST->getMemoryVT().getScalarType().isByteSized())
8699 return TLI.scalarizeVectorStore(ST, DAG);
8700
8701 if (ST->isTruncatingStore())
8702 return TLI.scalarizeVectorStore(ST, DAG);
8703
8704 // Generate a vector-predicated store if it is custom/legal on the target.
8705 // To avoid possible recursion, only do this if the widened mask type is
8706 // legal.
8707 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8708 // removed from the IR by the ExpandVectorPredication pass but we're
8709 // reintroducing them here.
8710 SDValue StVal = ST->getValue();
8711 EVT StVT = StVal.getValueType();
8712 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
8713 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
8714
8715 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8716 TLI.isTypeLegal(VT: WideMaskVT)) {
8717 // Widen the value.
8718 SDLoc DL(N);
8719 StVal = GetWidenedVector(Op: StVal);
8720 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
8721 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
8722 EC: StVT.getVectorElementCount());
8723 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
8724 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
8725 AM: ST->getAddressingMode());
8726 }
8727
8728 SmallVector<SDValue, 16> StChain;
8729 if (GenWidenVectorStores(StChain, ST)) {
8730 if (StChain.size() == 1)
8731 return StChain[0];
8732
8733 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
8734 }
8735
8736 if (StVT.isVector()) {
8737 // If all else fails replace the store with a wide masked store.
8738 SDLoc DL(N);
8739 SDValue WideStVal = GetWidenedVector(Op: StVal);
8740 SDValue Mask =
8741 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
8742
8743 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
8744 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
8745 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
8746 IsTruncating: ST->isTruncatingStore());
8747 }
8748
8749 report_fatal_error(reason: "Unable to widen vector store");
8750}
8751
8752SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8753 EVT StVT = ST->getMemoryVT();
8754 SDLoc dl(ST);
8755
8756 SDValue StVal = GetWidenedVector(Op: ST->getVal());
8757 EVT WidenVT = StVal.getValueType();
8758
8759 TypeSize StWidth = StVT.getSizeInBits();
8760 TypeSize WidenWidth = WidenVT.getSizeInBits();
8761 TypeSize WidthDiff = WidenWidth - StWidth;
8762
8763 // Find the vector type that can store the original memory width in one
8764 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8765 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8766 // implies a 4-byte movl, writing undef bytes past its object).
8767 std::optional<EVT> FirstVT =
8768 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/Align: 0,
8769 WidenEx: WidthDiff.getKnownMinValue());
8770 if (!FirstVT)
8771 return SDValue();
8772
8773 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8774
8775 SDValue StOp =
8776 coerceStoredValue(StVal, FirstVT: *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8777
8778 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: *FirstVT, Chain: ST->getChain(), Ptr: StOp,
8779 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
8780}
8781
8782SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8783 assert((OpNo == 1 || OpNo == 3) &&
8784 "Can widen only data or mask operand of vp_store");
8785 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
8786 SDValue Mask = ST->getMask();
8787 SDValue StVal = ST->getValue();
8788 SDLoc dl(N);
8789
8790 if (OpNo == 1) {
8791 // Widen the value.
8792 StVal = GetWidenedVector(Op: StVal);
8793
8794 // We only handle the case where the mask needs widening to an
8795 // identically-sized type as the vector inputs.
8796 assert(getTypeAction(Mask.getValueType()) ==
8797 TargetLowering::TypeWidenVector &&
8798 "Unable to widen VP store");
8799 Mask = GetWidenedVector(Op: Mask);
8800 } else {
8801 Mask = GetWidenedVector(Op: Mask);
8802
8803 // We only handle the case where the stored value needs widening to an
8804 // identically-sized type as the mask.
8805 assert(getTypeAction(StVal.getValueType()) ==
8806 TargetLowering::TypeWidenVector &&
8807 "Unable to widen VP store");
8808 StVal = GetWidenedVector(Op: StVal);
8809 }
8810
8811 assert(Mask.getValueType().getVectorElementCount() ==
8812 StVal.getValueType().getVectorElementCount() &&
8813 "Mask and data vectors should have the same number of elements");
8814 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
8815 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
8816 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
8817 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
8818 IsCompressing: ST->isCompressingStore());
8819}
8820
8821SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8822 unsigned OpNo) {
8823 assert((OpNo == 1 || OpNo == 4) &&
8824 "Can widen only data or mask operand of vp_strided_store");
8825 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
8826 SDValue Mask = SST->getMask();
8827 SDValue StVal = SST->getValue();
8828 SDLoc DL(N);
8829
8830 if (OpNo == 1)
8831 assert(getTypeAction(Mask.getValueType()) ==
8832 TargetLowering::TypeWidenVector &&
8833 "Unable to widen VP strided store");
8834 else
8835 assert(getTypeAction(StVal.getValueType()) ==
8836 TargetLowering::TypeWidenVector &&
8837 "Unable to widen VP strided store");
8838
8839 StVal = GetWidenedVector(Op: StVal);
8840 Mask = GetWidenedVector(Op: Mask);
8841
8842 assert(StVal.getValueType().getVectorElementCount() ==
8843 Mask.getValueType().getVectorElementCount() &&
8844 "Data and mask vectors should have the same number of elements");
8845
8846 return DAG.getStridedStoreVP(
8847 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
8848 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
8849 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
8850 IsCompressing: SST->isCompressingStore());
8851}
8852
8853SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8854 assert((OpNo == 1 || OpNo == 4) &&
8855 "Can widen only data or mask operand of mstore");
8856 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
8857 SDValue Mask = MST->getMask();
8858 EVT MaskVT = Mask.getValueType();
8859 SDValue StVal = MST->getValue();
8860 EVT VT = StVal.getValueType();
8861 SDLoc dl(N);
8862
8863 EVT WideVT, WideMaskVT;
8864 if (OpNo == 1) {
8865 // Widen the value.
8866 StVal = GetWidenedVector(Op: StVal);
8867
8868 WideVT = StVal.getValueType();
8869 WideMaskVT =
8870 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8871 EC: WideVT.getVectorElementCount());
8872 } else {
8873 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8874
8875 EVT ValueVT = StVal.getValueType();
8876 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8877 EC: WideMaskVT.getVectorElementCount());
8878 }
8879
8880 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8881 TLI.isTypeLegal(VT: WideMaskVT) && !MST->isCompressingStore()) {
8882 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8883 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8884 EC: VT.getVectorElementCount());
8885 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8886 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8887 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8888 }
8889
8890 if (OpNo == 1) {
8891 // The mask should be widened as well.
8892 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8893 } else {
8894 // Widen the mask.
8895 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8896
8897 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8898 }
8899
8900 assert(Mask.getValueType().getVectorElementCount() ==
8901 StVal.getValueType().getVectorElementCount() &&
8902 "Mask and data vectors should have the same number of elements");
8903 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8904 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8905 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8906 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8907}
8908
8909SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8910 assert(OpNo == 4 && "Can widen only the index of mgather");
8911 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8912 SDValue DataOp = MG->getPassThru();
8913 SDValue Mask = MG->getMask();
8914 SDValue Scale = MG->getScale();
8915
8916 // Just widen the index. It's allowed to have extra elements.
8917 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8918
8919 SDLoc dl(N);
8920 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8921 Scale};
8922 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8923 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8924 ExtTy: MG->getExtensionType());
8925 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8926 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8927 return SDValue();
8928}
8929
8930SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8931 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8932 SDValue DataOp = MSC->getValue();
8933 SDValue Mask = MSC->getMask();
8934 SDValue Index = MSC->getIndex();
8935 SDValue Scale = MSC->getScale();
8936 EVT WideMemVT = MSC->getMemoryVT();
8937
8938 if (OpNo == 1) {
8939 DataOp = GetWidenedVector(Op: DataOp);
8940 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8941
8942 // Widen index.
8943 EVT IndexVT = Index.getValueType();
8944 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8945 VT: IndexVT.getVectorElementType(), EC: WideEC);
8946 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8947
8948 // The mask should be widened as well.
8949 EVT MaskVT = Mask.getValueType();
8950 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8951 VT: MaskVT.getVectorElementType(), EC: WideEC);
8952 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8953
8954 // Widen the MemoryType
8955 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8956 VT: MSC->getMemoryVT().getScalarType(), EC: WideEC);
8957 } else if (OpNo == 4) {
8958 // Just widen the index. It's allowed to have extra elements.
8959 Index = GetWidenedVector(Op: Index);
8960 } else
8961 llvm_unreachable("Can't widen this operand of mscatter");
8962
8963 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8964 Scale};
8965 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8966 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8967 IsTruncating: MSC->isTruncatingStore());
8968}
8969
8970SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8971 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8972 SDValue DataOp = VPSC->getValue();
8973 SDValue Mask = VPSC->getMask();
8974 SDValue Index = VPSC->getIndex();
8975 SDValue Scale = VPSC->getScale();
8976 EVT WideMemVT = VPSC->getMemoryVT();
8977
8978 if (OpNo == 1) {
8979 DataOp = GetWidenedVector(Op: DataOp);
8980 Index = GetWidenedVector(Op: Index);
8981 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8982 Mask = GetWidenedMask(Mask, EC: WideEC);
8983 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8984 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8985 } else if (OpNo == 3) {
8986 // Just widen the index. It's allowed to have extra elements.
8987 Index = GetWidenedVector(Op: Index);
8988 } else
8989 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8990
8991 SDValue Ops[] = {
8992 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8993 VPSC->getVectorLength()};
8994 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8995 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8996}
8997
8998SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8999 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
9000 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
9001 SDLoc dl(N);
9002 EVT VT = N->getValueType(ResNo: 0);
9003
9004 // WARNING: In this code we widen the compare instruction with garbage.
9005 // This garbage may contain denormal floats which may be slow. Is this a real
9006 // concern ? Should we zero the unused lanes if this is a float compare ?
9007
9008 // Get a new SETCC node to compare the newly widened operands.
9009 // Only some of the compared elements are legal.
9010 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
9011 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
9012 if (VT.getScalarType() == MVT::i1)
9013 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
9014 EC: SVT.getVectorElementCount());
9015
9016 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
9017 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
9018
9019 // Extract the needed results from the result vector.
9020 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
9021 VT: SVT.getVectorElementType(),
9022 EC: VT.getVectorElementCount());
9023 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
9024
9025 EVT OpVT = N->getOperand(Num: 0).getValueType();
9026 ISD::NodeType ExtendCode =
9027 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
9028 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
9029}
9030
9031SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
9032 SDValue Chain = N->getOperand(Num: 0);
9033 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
9034 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
9035 SDValue CC = N->getOperand(Num: 3);
9036 SDLoc dl(N);
9037
9038 EVT VT = N->getValueType(ResNo: 0);
9039 EVT EltVT = VT.getVectorElementType();
9040 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
9041 unsigned NumElts = VT.getVectorNumElements();
9042
9043 // Unroll into a build vector.
9044 SmallVector<SDValue, 8> Scalars(NumElts);
9045 SmallVector<SDValue, 8> Chains(NumElts);
9046
9047 for (unsigned i = 0; i != NumElts; ++i) {
9048 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
9049 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
9050
9051 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
9052 Ops: {Chain, LHSElem, RHSElem, CC});
9053 Chains[i] = Scalars[i].getValue(R: 1);
9054 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
9055 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
9056 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
9057 }
9058
9059 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
9060 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
9061
9062 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
9063}
9064
9065static unsigned getExtendForIntVecReduction(unsigned Opc) {
9066 switch (Opc) {
9067 default:
9068 llvm_unreachable("Expected integer vector reduction");
9069 case ISD::VECREDUCE_ADD:
9070 case ISD::VECREDUCE_MUL:
9071 case ISD::VECREDUCE_AND:
9072 case ISD::VECREDUCE_OR:
9073 case ISD::VECREDUCE_XOR:
9074 return ISD::ANY_EXTEND;
9075 case ISD::VECREDUCE_SMAX:
9076 case ISD::VECREDUCE_SMIN:
9077 return ISD::SIGN_EXTEND;
9078 case ISD::VECREDUCE_UMAX:
9079 case ISD::VECREDUCE_UMIN:
9080 return ISD::ZERO_EXTEND;
9081 }
9082}
9083
9084SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
9085 SDLoc dl(N);
9086 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
9087 EVT VT = N->getValueType(ResNo: 0);
9088 EVT OrigVT = N->getOperand(Num: 0).getValueType();
9089 EVT WideVT = Op.getValueType();
9090 EVT ElemVT = OrigVT.getVectorElementType();
9091 SDNodeFlags Flags = N->getFlags();
9092
9093 unsigned Opc = N->getOpcode();
9094 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
9095 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
9096 assert(NeutralElem && "Neutral element must exist");
9097
9098 // Pad the vector with the neutral element.
9099 unsigned OrigElts = OrigVT.getVectorMinNumElements();
9100 unsigned WideElts = WideVT.getVectorMinNumElements();
9101
9102 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
9103 // needing to pad the source vector, because the inactive lanes can simply be
9104 // disabled and not contribute to the result.
9105 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
9106 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
9107 SDValue Start = NeutralElem;
9108 if (VT.isInteger())
9109 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
9110 assert(Start.getValueType() == VT);
9111 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
9112 EC: WideVT.getVectorElementCount());
9113 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
9114 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
9115 EC: OrigVT.getVectorElementCount());
9116 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
9117 }
9118
9119 if (WideVT.isScalableVector()) {
9120 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
9121 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
9122 EC: ElementCount::getScalable(MinVal: GCD));
9123 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
9124 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
9125 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
9126 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
9127 }
9128
9129 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
9130 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
9131
9132 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
9133}
9134
9135SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
9136 SDLoc dl(N);
9137 SDValue AccOp = N->getOperand(Num: 0);
9138 SDValue VecOp = N->getOperand(Num: 1);
9139 SDValue Op = GetWidenedVector(Op: VecOp);
9140
9141 EVT VT = N->getValueType(ResNo: 0);
9142 EVT OrigVT = VecOp.getValueType();
9143 EVT WideVT = Op.getValueType();
9144 EVT ElemVT = OrigVT.getVectorElementType();
9145 SDNodeFlags Flags = N->getFlags();
9146
9147 unsigned Opc = N->getOpcode();
9148 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
9149 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
9150
9151 // Pad the vector with the neutral element.
9152 unsigned OrigElts = OrigVT.getVectorMinNumElements();
9153 unsigned WideElts = WideVT.getVectorMinNumElements();
9154
9155 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
9156 // needing to pad the source vector, because the inactive lanes can simply be
9157 // disabled and not contribute to the result.
9158 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
9159 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
9160 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
9161 EC: WideVT.getVectorElementCount());
9162 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
9163 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
9164 EC: OrigVT.getVectorElementCount());
9165 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
9166 }
9167
9168 if (WideVT.isScalableVector()) {
9169 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
9170 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
9171 EC: ElementCount::getScalable(MinVal: GCD));
9172 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
9173 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
9174 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
9175 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
9176 }
9177
9178 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
9179 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
9180
9181 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
9182}
9183
9184SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
9185 assert(N->isVPOpcode() && "Expected VP opcode");
9186
9187 SDLoc dl(N);
9188 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
9189 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
9190 EC: Op.getValueType().getVectorElementCount());
9191
9192 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
9193 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
9194 Flags: N->getFlags());
9195}
9196
9197SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
9198 // This only gets called in the case that the left and right inputs and
9199 // result are of a legal odd vector type, and the condition is illegal i1 of
9200 // the same odd width that needs widening.
9201 EVT VT = N->getValueType(ResNo: 0);
9202 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
9203
9204 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
9205 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
9206 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
9207 SDLoc DL(N);
9208
9209 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
9210 N2: LeftIn, N3: RightIn);
9211 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
9212}
9213
9214SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
9215 SDLoc DL(N);
9216 SDValue Source = N->getOperand(Num: 0);
9217 EVT SourceVT = Source.getValueType();
9218 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
9219
9220 SDValue WideSource;
9221 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
9222 WideSource = GetWidenedVector(Op: Source);
9223 } else {
9224 // Pad the widened portion with all-ones so the extra lanes appear as
9225 // active (non-zero) elements and do not contribute trailing zeros.
9226 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: WideVT);
9227 if (WideVT.isFixedLengthVector() &&
9228 getTypeAction(VT: WideVT) == TargetLowering::TypeSplitVector) {
9229 WideSource = GetWidenedVector(Op: Source);
9230 unsigned WideElts = WideVT.getVectorNumElements();
9231 SmallVector<int> Mask(WideElts);
9232 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
9233 for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
9234 Mask[I] += WideElts;
9235 WideSource = DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: WideSource, N2: AllOnes, Mask);
9236 } else {
9237 WideSource = DAG.getInsertSubvector(DL, Vec: AllOnes, SubVec: Source, Idx: 0);
9238 }
9239 }
9240
9241 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0), Operand: WideSource,
9242 Flags: N->getFlags());
9243}
9244
9245SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
9246 SDLoc DL(N);
9247 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
9248 EVT SrcVT = Source.getValueType();
9249 SDValue Mask =
9250 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
9251
9252 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
9253 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
9254}
9255
9256SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
9257 SDLoc DL(N);
9258 SDValue Mask = N->getOperand(Num: 0);
9259 EVT OrigMaskVT = Mask.getValueType();
9260 SDValue WideMask = GetWidenedVector(Op: Mask);
9261 EVT WideMaskVT = WideMask.getValueType();
9262
9263 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
9264 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
9265 unsigned WideElts = WideMaskVT.getVectorNumElements();
9266 if (OrigElts != WideElts) {
9267 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
9268 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
9269 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9270 }
9271
9272 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
9273 Operand: WideMask);
9274}
9275
9276SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_SHUFFLE_VAR(SDNode *N,
9277 unsigned OpNo) {
9278 // The only possibility for an illegal operand is the mask, since result type
9279 // legalization would have handled this node already otherwise.
9280 assert(OpNo == 1 && "Illegal operand must be mask");
9281 // TODO: Legalize the mask type instead once a target lowers the node.
9282 return TLI.expandVECTOR_SHUFFLE_VAR(Node: N, DAG);
9283}
9284
9285SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
9286 if (OpNo == 0) {
9287 SDLoc DL(N);
9288 EVT ResVT = N->getValueType(ResNo: 0);
9289 EVT SourceVT = N->getOperand(Num: 0).getValueType();
9290 EVT WideSourceVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
9291 EVT WidenVT =
9292 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
9293 EC: WideSourceVT.getVectorElementCount());
9294
9295 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
9296 SubVec: N->getOperand(Num: 0), Idx: 0);
9297 SDValue WideMask = DAG.getInsertSubvector(
9298 DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT), SubVec: N->getOperand(Num: 2), Idx: 0);
9299 SDValue WideMatch = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
9300 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
9301 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideMatch, Idx: 0);
9302 }
9303
9304 // Note: The Mask (OpNo == 2) should be widened with the result.
9305 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
9306
9307 SDLoc DL(N);
9308 SDValue Needle = N->getOperand(Num: 1);
9309 EVT NeedleVT = Needle.getValueType();
9310 if (NeedleVT.getVectorNumElements() == 1)
9311 return TLI.expandVectorMatch(N, DAG);
9312
9313 EVT WidenNeedleVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: NeedleVT);
9314
9315 SDValue Fill =
9316 DAG.getExtractVectorElt(DL, VT: NeedleVT.getVectorElementType(), Vec: Needle, Idx: 0);
9317 SDValue WideNeedle = DAG.getSplatVector(VT: WidenNeedleVT, DL, Op: Fill);
9318 WideNeedle = DAG.getInsertSubvector(DL, Vec: WideNeedle, SubVec: Needle, Idx: 0);
9319
9320 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0),
9321 N1: N->getOperand(Num: 0), N2: WideNeedle, N3: N->getOperand(Num: 2),
9322 Flags: N->getFlags());
9323}
9324
9325SDValue DAGTypeLegalizer::WidenVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
9326 // The accumulator is legal and only the inputs need widening. Expand, as
9327 // when widening the result; the extended inputs are widened from there.
9328 return TLI.expandPartialReduceMLA(Node: N, DAG);
9329}
9330
9331//===----------------------------------------------------------------------===//
9332// Vector Widening Utilities
9333//===----------------------------------------------------------------------===//
9334
9335// Utility function to find the type to chop up a widen vector for load/store
9336// TLI: Target lowering used to determine legal types.
9337// Width: Width left need to load/store.
9338// WidenVT: The widen vector type to load to/store from
9339// Align: If 0, don't allow use of a wider type
9340// WidenEx: If Align is not 0, the amount additional we can load/store from.
9341
9342static std::optional<EVT> findMemType(SelectionDAG &DAG,
9343 const TargetLowering &TLI, unsigned Width,
9344 EVT WidenVT, unsigned Align = 0,
9345 unsigned WidenEx = 0) {
9346 EVT WidenEltVT = WidenVT.getVectorElementType();
9347 const bool Scalable = WidenVT.isScalableVector();
9348 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
9349 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
9350 unsigned AlignInBits = Align*8;
9351
9352 EVT RetVT = WidenEltVT;
9353 // Don't bother looking for an integer type if the vector is scalable, skip
9354 // to vector types.
9355 if (!Scalable) {
9356 // If we have one element to load/store, return it.
9357 if (Width == WidenEltWidth)
9358 return RetVT;
9359
9360 // See if there is larger legal integer than the element type to load/store.
9361 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
9362 unsigned MemVTWidth = MemVT.getSizeInBits();
9363 if (MemVT.getSizeInBits() <= WidenEltWidth)
9364 break;
9365 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
9366 if ((Action == TargetLowering::TypeLegal ||
9367 Action == TargetLowering::TypePromoteInteger) &&
9368 (WidenWidth % MemVTWidth) == 0 &&
9369 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
9370 (MemVTWidth <= Width ||
9371 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9372 if (MemVTWidth == WidenWidth)
9373 return MemVT;
9374 RetVT = MemVT;
9375 break;
9376 }
9377 }
9378 }
9379
9380 // See if there is a larger vector type to load/store that has the same vector
9381 // element type and is evenly divisible with the WidenVT.
9382 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
9383 // Skip vector MVTs which don't match the scalable property of WidenVT.
9384 if (Scalable != MemVT.isScalableVector())
9385 continue;
9386 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
9387 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
9388 if ((Action == TargetLowering::TypeLegal ||
9389 Action == TargetLowering::TypePromoteInteger) &&
9390 WidenEltVT == MemVT.getVectorElementType() &&
9391 (WidenWidth % MemVTWidth) == 0 &&
9392 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
9393 (MemVTWidth <= Width ||
9394 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9395 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
9396 return MemVT;
9397 }
9398 }
9399
9400 // Using element-wise loads and stores for widening operations is not
9401 // supported for scalable vectors
9402 if (Scalable)
9403 return std::nullopt;
9404
9405 return RetVT;
9406}
9407
9408// Builds a vector type from scalar loads
9409// VecTy: Resulting Vector type
9410// LDOps: Load operators to build a vector type
9411// [Start,End) the list of loads to use.
9412static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
9413 SmallVectorImpl<SDValue> &LdOps,
9414 unsigned Start, unsigned End) {
9415 SDLoc dl(LdOps[Start]);
9416 EVT LdTy = LdOps[Start].getValueType();
9417 unsigned Width = VecTy.getSizeInBits();
9418 unsigned NumElts = Width / LdTy.getSizeInBits();
9419 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
9420
9421 unsigned Idx = 1;
9422 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
9423
9424 for (unsigned i = Start + 1; i != End; ++i) {
9425 EVT NewLdTy = LdOps[i].getValueType();
9426 if (NewLdTy != LdTy) {
9427 NumElts = Width / NewLdTy.getSizeInBits();
9428 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
9429 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
9430 // Readjust position and vector position based on new load type.
9431 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
9432 LdTy = NewLdTy;
9433 }
9434 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
9435 }
9436 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
9437}
9438
9439SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
9440 LoadSDNode *LD) {
9441 // The strategy assumes that we can efficiently load power-of-two widths.
9442 // The routine chops the vector into the largest vector loads with the same
9443 // element type or scalar loads and then recombines it to the widen vector
9444 // type.
9445 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9446 EVT LdVT = LD->getMemoryVT();
9447 SDLoc dl(LD);
9448 assert(LdVT.isVector() && WidenVT.isVector());
9449 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9450 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
9451
9452 // Load information
9453 SDValue Chain = LD->getChain();
9454 SDValue BasePtr = LD->getBasePtr();
9455 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9456 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9457
9458 TypeSize LdWidth = LdVT.getSizeInBits();
9459 TypeSize WidenWidth = WidenVT.getSizeInBits();
9460 TypeSize WidthDiff = WidenWidth - LdWidth;
9461 // Allow wider loads if they are sufficiently aligned to avoid memory faults
9462 // and if the original load is simple.
9463 unsigned LdAlign =
9464 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
9465
9466 // Find the vector type that can load from.
9467 std::optional<EVT> FirstVT =
9468 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
9469 WidenEx: WidthDiff.getKnownMinValue());
9470
9471 if (!FirstVT)
9472 return SDValue();
9473
9474 SmallVector<EVT, 8> MemVTs;
9475 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
9476
9477 // Unless we're able to load in one instruction we must work out how to load
9478 // the remainder.
9479 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
9480 std::optional<EVT> NewVT = FirstVT;
9481 TypeSize RemainingWidth = LdWidth;
9482 TypeSize NewVTWidth = FirstVTWidth;
9483 do {
9484 RemainingWidth -= NewVTWidth;
9485 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
9486 // The current type we are using is too large. Find a better size.
9487 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
9488 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
9489 if (!NewVT)
9490 return SDValue();
9491 NewVTWidth = NewVT->getSizeInBits();
9492 }
9493 MemVTs.push_back(Elt: *NewVT);
9494 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
9495 }
9496
9497 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9498 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9499 LdChain.push_back(Elt: LdOp.getValue(R: 1));
9500
9501 // Check if we can load the element with one instruction.
9502 if (MemVTs.empty())
9503 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
9504 DAG);
9505
9506 // Load vector by using multiple loads from largest vector to scalar.
9507 SmallVector<SDValue, 16> LdOps;
9508 LdOps.push_back(Elt: LdOp);
9509
9510 uint64_t ScaledOffset = 0;
9511 MachinePointerInfo MPI = LD->getPointerInfo();
9512
9513 // First incremement past the first load.
9514 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
9515 ScaledOffset: &ScaledOffset);
9516
9517 for (EVT MemVT : MemVTs) {
9518 Align NewAlign = ScaledOffset == 0
9519 ? LD->getBaseAlign()
9520 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
9521 SDValue L = DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags,
9522 Metadata);
9523
9524 LdOps.push_back(Elt: L);
9525 LdChain.push_back(Elt: L.getValue(R: 1));
9526 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
9527 }
9528
9529 // Build the vector from the load operations.
9530 unsigned End = LdOps.size();
9531 if (!LdOps[0].getValueType().isVector())
9532 // All the loads are scalar loads.
9533 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
9534
9535 // If the load contains vectors, build the vector using concat vector.
9536 // All of the vectors used to load are power-of-2, and the scalar loads can be
9537 // combined to make a power-of-2 vector.
9538 SmallVector<SDValue, 16> ConcatOps(End);
9539 int i = End - 1;
9540 int Idx = End;
9541 EVT LdTy = LdOps[i].getValueType();
9542 // First, combine the scalar loads to a vector.
9543 if (!LdTy.isVector()) {
9544 for (--i; i >= 0; --i) {
9545 LdTy = LdOps[i].getValueType();
9546 if (LdTy.isVector())
9547 break;
9548 }
9549 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
9550 }
9551
9552 ConcatOps[--Idx] = LdOps[i];
9553 for (--i; i >= 0; --i) {
9554 EVT NewLdTy = LdOps[i].getValueType();
9555 if (NewLdTy != LdTy) {
9556 // Create a larger vector.
9557 TypeSize LdTySize = LdTy.getSizeInBits();
9558 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
9559 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
9560 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
9561 unsigned NumOps =
9562 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
9563 SmallVector<SDValue, 16> WidenOps(NumOps);
9564 unsigned j = 0;
9565 for (; j != End-Idx; ++j)
9566 WidenOps[j] = ConcatOps[Idx+j];
9567 for (; j != NumOps; ++j)
9568 WidenOps[j] = DAG.getPOISON(VT: LdTy);
9569
9570 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
9571 Ops: WidenOps);
9572 Idx = End - 1;
9573 LdTy = NewLdTy;
9574 }
9575 ConcatOps[--Idx] = LdOps[i];
9576 }
9577
9578 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9579 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
9580 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
9581
9582 // We need to fill the rest with undefs to build the vector.
9583 unsigned NumOps =
9584 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9585 SmallVector<SDValue, 16> WidenOps(NumOps);
9586 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
9587 {
9588 unsigned i = 0;
9589 for (; i != End-Idx; ++i)
9590 WidenOps[i] = ConcatOps[Idx+i];
9591 for (; i != NumOps; ++i)
9592 WidenOps[i] = UndefVal;
9593 }
9594 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
9595}
9596
9597SDValue
9598DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9599 LoadSDNode *LD,
9600 ISD::LoadExtType ExtType) {
9601 // For extension loads, it may not be more efficient to chop up the vector
9602 // and then extend it. Instead, we unroll the load and build a new vector.
9603 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9604 EVT LdVT = LD->getMemoryVT();
9605 SDLoc dl(LD);
9606 assert(LdVT.isVector() && WidenVT.isVector());
9607 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9608
9609 // Load information
9610 SDValue Chain = LD->getChain();
9611 SDValue BasePtr = LD->getBasePtr();
9612 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9613 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9614
9615 if (LdVT.isScalableVector())
9616 return SDValue();
9617
9618 EVT EltVT = WidenVT.getVectorElementType();
9619 EVT LdEltVT = LdVT.getVectorElementType();
9620 unsigned NumElts = LdVT.getVectorNumElements();
9621
9622 // Load each element and widen.
9623 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9624 SmallVector<SDValue, 16> Ops(WidenNumElts);
9625 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9626 Ops[0] =
9627 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9628 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9629 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
9630 unsigned i = 0, Offset = Increment;
9631 for (i=1; i < NumElts; ++i, Offset += Increment) {
9632 SDValue NewBasePtr =
9633 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
9634 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
9635 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
9636 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9637 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
9638 }
9639
9640 // Fill the rest with undefs.
9641 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9642 for (; i != WidenNumElts; ++i)
9643 Ops[i] = UndefVal;
9644
9645 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
9646}
9647
9648bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9649 StoreSDNode *ST) {
9650 // The strategy assumes that we can efficiently store power-of-two widths.
9651 // The routine chops the vector into the largest vector stores with the same
9652 // element type or scalar stores.
9653 SDValue Chain = ST->getChain();
9654 SDValue BasePtr = ST->getBasePtr();
9655 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9656 MMOMetadata Metadata = ST->getMMOMetadataForSubAccess();
9657 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
9658 SDLoc dl(ST);
9659
9660 EVT StVT = ST->getMemoryVT();
9661 TypeSize StWidth = StVT.getSizeInBits();
9662 EVT ValVT = ValOp.getValueType();
9663 TypeSize ValWidth = ValVT.getSizeInBits();
9664 EVT ValEltVT = ValVT.getVectorElementType();
9665 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9666 assert(StVT.getVectorElementType() == ValEltVT);
9667 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9668 "Mismatch between store and value types");
9669
9670 int Idx = 0; // current index to store
9671
9672 MachinePointerInfo MPI = ST->getPointerInfo();
9673 uint64_t ScaledOffset = 0;
9674
9675 // A breakdown of how to widen this vector store. Each element of the vector
9676 // is a memory VT combined with the number of times it is to be stored to,
9677 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9678 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
9679
9680 while (StWidth.isNonZero()) {
9681 // Find the largest vector type we can store with.
9682 std::optional<EVT> NewVT =
9683 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
9684 if (!NewVT)
9685 return false;
9686 MemVTs.push_back(Elt: {*NewVT, 0});
9687 TypeSize NewVTWidth = NewVT->getSizeInBits();
9688
9689 do {
9690 StWidth -= NewVTWidth;
9691 MemVTs.back().second++;
9692 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
9693 }
9694
9695 for (const auto &Pair : MemVTs) {
9696 EVT NewVT = Pair.first;
9697 unsigned Count = Pair.second;
9698 TypeSize NewVTWidth = NewVT.getSizeInBits();
9699
9700 if (NewVT.isVector()) {
9701 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9702 do {
9703 Align NewAlign = ScaledOffset == 0
9704 ? ST->getBaseAlign()
9705 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
9706 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
9707 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
9708 MMOFlags, Metadata);
9709 StChain.push_back(Elt: PartStore);
9710
9711 Idx += NumVTElts;
9712 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
9713 ScaledOffset: &ScaledOffset);
9714 } while (--Count);
9715 } else {
9716 // Cast the vector to the scalar type we can store.
9717 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9718 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
9719 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
9720 // Readjust index position based on new vector type.
9721 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9722 do {
9723 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
9724 SDValue PartStore =
9725 DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: ST->getBaseAlign(),
9726 MMOFlags, Metadata);
9727 StChain.push_back(Elt: PartStore);
9728
9729 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
9730 } while (--Count);
9731 // Restore index back to be relative to the original widen element type.
9732 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9733 }
9734 }
9735
9736 return true;
9737}
9738
9739/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9740/// input vector must have the same element type as NVT.
9741/// FillWithZeroes specifies that the vector should be widened with zeroes.
9742SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9743 bool FillWithZeroes) {
9744 // Note that InOp might have been widened so it might already have
9745 // the right width or it might need be narrowed.
9746 EVT InVT = InOp.getValueType();
9747 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
9748 "input and widen element type must match");
9749 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9750 "cannot modify scalable vectors in this way");
9751 SDLoc dl(InOp);
9752
9753 // Check if InOp already has the right width.
9754 if (InVT == NVT)
9755 return InOp;
9756
9757 ElementCount InEC = InVT.getVectorElementCount();
9758 ElementCount WidenEC = NVT.getVectorElementCount();
9759 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
9760 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
9761 SmallVector<SDValue, 16> Ops(NumConcat);
9762 SDValue FillVal =
9763 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
9764 Ops[0] = InOp;
9765 for (unsigned i = 1; i != NumConcat; ++i)
9766 Ops[i] = FillVal;
9767
9768 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9769 }
9770
9771 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
9772 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
9773
9774 if (NVT.isScalableVector() && InVT.isScalableVector()) {
9775 // Split the input into the largest equal-sized scalable subvectors.
9776 unsigned InNumElts = InVT.getVectorMinNumElements();
9777 unsigned NewNumElts = NVT.getVectorMinNumElements();
9778 unsigned CommonFactor = std::gcd(m: InNumElts, n: NewNumElts);
9779 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(),
9780 EC: ElementCount::getScalable(MinVal: CommonFactor));
9781
9782 SmallVector<SDValue, 16> Ops;
9783 unsigned NumCopiedParts = std::min(a: InNumElts, b: NewNumElts) / CommonFactor;
9784 for (unsigned I = 0; I != NumCopiedParts; ++I)
9785 Ops.push_back(
9786 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: I * CommonFactor));
9787
9788 unsigned NumResultParts = NewNumElts / CommonFactor;
9789 if (NumResultParts > NumCopiedParts) {
9790 SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: PartVT)
9791 : DAG.getPOISON(VT: PartVT);
9792 Ops.append(NumInputs: NumResultParts - NumCopiedParts, Elt: FillVal);
9793 }
9794
9795 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9796 }
9797
9798 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9799 "Scalable vectors should have been handled already.");
9800
9801 unsigned InNumElts = InEC.getFixedValue();
9802 unsigned WidenNumElts = WidenEC.getFixedValue();
9803
9804 // Fall back to extract and build (+ mask, if padding with zeros).
9805 SmallVector<SDValue, 16> Ops(WidenNumElts);
9806 EVT EltVT = NVT.getVectorElementType();
9807 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
9808 unsigned Idx;
9809 for (Idx = 0; Idx < MinNumElts; ++Idx)
9810 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
9811
9812 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9813 for (; Idx < WidenNumElts; ++Idx)
9814 Ops[Idx] = UndefVal;
9815
9816 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
9817 if (!FillWithZeroes)
9818 return Widened;
9819
9820 assert(NVT.isInteger() &&
9821 "We expect to never want to FillWithZeroes for non-integral types.");
9822
9823 SmallVector<SDValue, 16> MaskOps;
9824 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
9825 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
9826
9827 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
9828 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
9829}
9830