1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
48 return;
49
50 switch (N->getOpcode()) {
51 default:
52#ifndef NDEBUG
53 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
54 N->dump(&DAG);
55 dbgs() << "\n";
56#endif
57 report_fatal_error(reason: "Do not know how to scalarize the result of this "
58 "operator!\n");
59
60 case ISD::LOOP_DEPENDENCE_WAR_MASK:
61 case ISD::LOOP_DEPENDENCE_RAW_MASK:
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
63 break;
64 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
65 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
66 case ISD::SPLAT_VECTOR:
67 case ISD::BUILD_VECTOR:
68 R = ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(N);
69 break;
70 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
71 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
72 case ISD::CONVERT_FROM_ARBITRARY_FP:
73 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
74 break;
75 case ISD::CONVERT_TO_ARBITRARY_FP:
76 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
77 break;
78 case ISD::AssertZext:
79 case ISD::AssertSext:
80 case ISD::FPOWI:
81 case ISD::AssertNoFPClass:
82 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
83 break;
84 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
85 case ISD::ATOMIC_LOAD:
86 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
87 break;
88 case ISD::ATOMIC_SWAP:
89 case ISD::ATOMIC_LOAD_ADD:
90 case ISD::ATOMIC_LOAD_SUB:
91 case ISD::ATOMIC_LOAD_AND:
92 case ISD::ATOMIC_LOAD_OR:
93 case ISD::ATOMIC_LOAD_XOR:
94 case ISD::ATOMIC_LOAD_NAND:
95 case ISD::ATOMIC_LOAD_MIN:
96 case ISD::ATOMIC_LOAD_MAX:
97 case ISD::ATOMIC_LOAD_UMIN:
98 case ISD::ATOMIC_LOAD_UMAX:
99 case ISD::ATOMIC_LOAD_FADD:
100 case ISD::ATOMIC_LOAD_FSUB:
101 case ISD::ATOMIC_LOAD_FMIN:
102 case ISD::ATOMIC_LOAD_FMAX:
103 case ISD::ATOMIC_LOAD_FMINIMUM:
104 case ISD::ATOMIC_LOAD_FMAXIMUM:
105 case ISD::ATOMIC_LOAD_FMINIMUMNUM:
106 case ISD::ATOMIC_LOAD_FMAXIMUMNUM:
107 case ISD::ATOMIC_LOAD_UINC_WRAP:
108 case ISD::ATOMIC_LOAD_UDEC_WRAP:
109 case ISD::ATOMIC_LOAD_USUB_COND:
110 case ISD::ATOMIC_LOAD_USUB_SAT:
111 R = ScalarizeVecRes_ATOMIC_RMW(N: cast<AtomicSDNode>(Val: N));
112 break;
113 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
114 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
115 case ISD::VECTOR_DEINTERLEAVE:
116 case ISD::VECTOR_INTERLEAVE:
117 R = ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
118 break;
119 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
120 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
121 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
122 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
123 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
124 case ISD::VECTOR_MATCH:
125 R = ScalarizeVecRes_VECTOR_MATCH(N);
126 break;
127 case ISD::POISON:
128 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
129 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
130 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
131 case ISD::ANY_EXTEND_VECTOR_INREG:
132 case ISD::SIGN_EXTEND_VECTOR_INREG:
133 case ISD::ZERO_EXTEND_VECTOR_INREG:
134 R = ScalarizeVecRes_VecInregOp(N);
135 break;
136 case ISD::ABS:
137 case ISD::ABS_MIN_POISON:
138 case ISD::ANY_EXTEND:
139 case ISD::BITREVERSE:
140 case ISD::BSWAP:
141 case ISD::CTLZ:
142 case ISD::CTLZ_ZERO_POISON:
143 case ISD::CTPOP:
144 case ISD::CTTZ:
145 case ISD::CTTZ_ZERO_POISON:
146 case ISD::FABS:
147 case ISD::FACOS:
148 case ISD::FASIN:
149 case ISD::FATAN:
150 case ISD::FCEIL:
151 case ISD::FCOS:
152 case ISD::FCOSH:
153 case ISD::FEXP:
154 case ISD::FEXP2:
155 case ISD::FEXP10:
156 case ISD::FFLOOR:
157 case ISD::FLOG:
158 case ISD::FLOG10:
159 case ISD::FLOG2:
160 case ISD::FNEARBYINT:
161 case ISD::FNEG:
162 case ISD::FREEZE:
163 case ISD::ARITH_FENCE:
164 case ISD::FP_EXTEND:
165 case ISD::FP_TO_SINT:
166 case ISD::FP_TO_UINT:
167 case ISD::FRINT:
168 case ISD::LRINT:
169 case ISD::LLRINT:
170 case ISD::FROUND:
171 case ISD::FROUNDEVEN:
172 case ISD::LROUND:
173 case ISD::LLROUND:
174 case ISD::FSIN:
175 case ISD::FSINH:
176 case ISD::FSQRT:
177 case ISD::FTAN:
178 case ISD::FTANH:
179 case ISD::FTRUNC:
180 case ISD::SIGN_EXTEND:
181 case ISD::SINT_TO_FP:
182 case ISD::TRUNCATE:
183 case ISD::UINT_TO_FP:
184 case ISD::ZERO_EXTEND:
185 case ISD::FCANONICALIZE:
186 R = ScalarizeVecRes_UnaryOp(N);
187 break;
188 case ISD::ADDRSPACECAST:
189 R = ScalarizeVecRes_ADDRSPACECAST(N);
190 break;
191 case ISD::FMODF:
192 case ISD::FFREXP:
193 case ISD::FSINCOS:
194 case ISD::FSINCOSPI:
195 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
196 break;
197 case ISD::ADD:
198 case ISD::AND:
199 case ISD::AVGCEILS:
200 case ISD::AVGCEILU:
201 case ISD::AVGFLOORS:
202 case ISD::AVGFLOORU:
203 case ISD::FADD:
204 case ISD::FCOPYSIGN:
205 case ISD::FDIV:
206 case ISD::FMUL:
207 case ISD::FMINNUM:
208 case ISD::FMAXNUM:
209 case ISD::FMINNUM_IEEE:
210 case ISD::FMAXNUM_IEEE:
211 case ISD::FMINIMUM:
212 case ISD::FMAXIMUM:
213 case ISD::FMINIMUMNUM:
214 case ISD::FMAXIMUMNUM:
215 case ISD::ABDS:
216 case ISD::ABDU:
217 case ISD::SMIN:
218 case ISD::SMAX:
219 case ISD::UMIN:
220 case ISD::UMAX:
221
222 case ISD::SADDSAT:
223 case ISD::UADDSAT:
224 case ISD::SSUBSAT:
225 case ISD::USUBSAT:
226 case ISD::SSHLSAT:
227 case ISD::USHLSAT:
228
229 case ISD::FPOW:
230 case ISD::FATAN2:
231 case ISD::FREM:
232 case ISD::FSUB:
233 case ISD::MUL:
234 case ISD::MULHS:
235 case ISD::MULHU:
236 case ISD::OR:
237 case ISD::SDIV:
238 case ISD::SREM:
239 case ISD::SUB:
240 case ISD::UDIV:
241 case ISD::UREM:
242 case ISD::XOR:
243 case ISD::SHL:
244 case ISD::SRA:
245 case ISD::SRL:
246 case ISD::ROTL:
247 case ISD::ROTR:
248 case ISD::CLMUL:
249 case ISD::CLMULR:
250 case ISD::CLMULH:
251 case ISD::PEXT:
252 case ISD::PDEP:
253 R = ScalarizeVecRes_BinOp(N);
254 break;
255
256 case ISD::MASKED_UDIV:
257 case ISD::MASKED_SDIV:
258 case ISD::MASKED_UREM:
259 case ISD::MASKED_SREM:
260 R = ScalarizeVecRes_MaskedBinOp(N);
261 break;
262
263 case ISD::FLDEXP:
264 R = ScalarizeVecRes_FPOp_MultiType(N);
265 break;
266
267 case ISD::SCMP:
268 case ISD::UCMP:
269 R = ScalarizeVecRes_CMP(N);
270 break;
271
272 case ISD::FMA:
273 case ISD::FSHL:
274 case ISD::FSHR:
275 R = ScalarizeVecRes_TernaryOp(N);
276 break;
277
278#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
279 case ISD::STRICT_##DAGN:
280#include "llvm/IR/ConstrainedOps.def"
281 R = ScalarizeVecRes_StrictFPOp(N);
282 break;
283
284 case ISD::FP_TO_UINT_SAT:
285 case ISD::FP_TO_SINT_SAT:
286 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
287 break;
288
289 case ISD::UADDO:
290 case ISD::SADDO:
291 case ISD::USUBO:
292 case ISD::SSUBO:
293 case ISD::UMULO:
294 case ISD::SMULO:
295 R = ScalarizeVecRes_OverflowOp(N, ResNo);
296 break;
297 case ISD::SMULFIX:
298 case ISD::SMULFIXSAT:
299 case ISD::UMULFIX:
300 case ISD::UMULFIXSAT:
301 case ISD::SDIVFIX:
302 case ISD::SDIVFIXSAT:
303 case ISD::UDIVFIX:
304 case ISD::UDIVFIXSAT:
305 R = ScalarizeVecRes_FIX(N);
306 break;
307 }
308
309 // If R is null, the sub-method took care of registering the result.
310 if (R.getNode())
311 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
312}
313
314SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
315 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
316 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
317 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
318 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
319}
320
321SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
322 SDLoc DL(N);
323 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
324 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
325 SDValue Mask = N->getOperand(Num: 2);
326 EVT MaskVT = Mask.getValueType();
327 // The vselect result and input vectors need scalarizing, but it's
328 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
329 // See the similar logic in ScalarizeVecRes_SETCC.
330 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeScalarizeVector)
331 Mask = GetScalarizedVector(Op: Mask);
332 else
333 Mask = DAG.getExtractVectorElt(DL, VT: MaskVT.getVectorElementType(), Vec: Mask, Idx: 0);
334 // Vectors may have a different boolean contents to scalars, so truncate to i1
335 // and let type legalization promote appropriately.
336 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
337 // Masked binary ops don't have UB on disabled lanes but produce poison, so
338 // use 1 as the divisor to avoid division by zero and overflow.
339 SDValue Divisor = DAG.getSelect(DL, VT: LHS.getValueType(), Cond: Mask, LHS: RHS,
340 RHS: DAG.getConstant(Val: 1, DL, VT: LHS.getValueType()));
341 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL,
342 VT: LHS.getValueType(), N1: LHS, N2: Divisor);
343}
344
345SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
346 SDLoc DL(N);
347
348 SDValue LHS = N->getOperand(Num: 0);
349 SDValue RHS = N->getOperand(Num: 1);
350 if (getTypeAction(VT: LHS.getValueType()) ==
351 TargetLowering::TypeScalarizeVector) {
352 LHS = GetScalarizedVector(Op: LHS);
353 RHS = GetScalarizedVector(Op: RHS);
354 } else {
355 EVT VT = LHS.getValueType().getVectorElementType();
356 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
357 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
358 }
359
360 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
361 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
362}
363
364SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
365 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
366 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
367 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
368 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
369 N3: Op2, Flags: N->getFlags());
370}
371
372SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
373 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
374 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
375 SDValue Op2 = N->getOperand(Num: 2);
376 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
377 N3: Op2, Flags: N->getFlags());
378}
379
380SDValue
381DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
382 unsigned ResNo) {
383 assert(N->getValueType(0).getVectorNumElements() == 1 &&
384 "Unexpected vector type!");
385 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
386
387 EVT VT0 = N->getValueType(ResNo: 0);
388 EVT VT1 = N->getValueType(ResNo: 1);
389 SDLoc dl(N);
390
391 SDNode *ScalarNode =
392 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
393 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
394 .getNode();
395
396 // Replace the other vector result not being explicitly scalarized here.
397 unsigned OtherNo = 1 - ResNo;
398 EVT OtherVT = N->getValueType(ResNo: OtherNo);
399 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
400 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
401 } else {
402 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
403 Operand: SDValue(ScalarNode, OtherNo));
404 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
405 }
406
407 return SDValue(ScalarNode, ResNo);
408}
409
410SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
411 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
412 unsigned NumOpers = N->getNumOperands();
413 SDValue Chain = N->getOperand(Num: 0);
414 EVT ValueVTs[] = {VT, MVT::Other};
415 SDLoc dl(N);
416
417 SmallVector<SDValue, 4> Opers(NumOpers);
418
419 // The Chain is the first operand.
420 Opers[0] = Chain;
421
422 // Now process the remaining operands.
423 for (unsigned i = 1; i < NumOpers; ++i) {
424 SDValue Oper = N->getOperand(Num: i);
425 EVT OperVT = Oper.getValueType();
426
427 if (OperVT.isVector()) {
428 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
429 Oper = GetScalarizedVector(Op: Oper);
430 else
431 Oper =
432 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
433 }
434
435 Opers[i] = Oper;
436 }
437
438 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
439 Ops: Opers, Flags: N->getFlags());
440
441 // Legalize the chain result - switch anything that used the old chain to
442 // use the new one.
443 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
444 return Result;
445}
446
447SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
448 unsigned ResNo) {
449 SDLoc DL(N);
450 EVT ResVT = N->getValueType(ResNo: 0);
451 EVT OvVT = N->getValueType(ResNo: 1);
452
453 SDValue ScalarLHS, ScalarRHS;
454 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
455 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
456 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
457 } else {
458 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
459 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
460 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
461 ScalarLHS = ElemsLHS[0];
462 ScalarRHS = ElemsRHS[0];
463 }
464
465 SDVTList ScalarVTs = DAG.getVTList(
466 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
467 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
468 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
469 .getNode();
470
471 // Replace the other vector result not being explicitly scalarized here.
472 unsigned OtherNo = 1 - ResNo;
473 EVT OtherVT = N->getValueType(ResNo: OtherNo);
474 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
475 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
476 } else {
477 SDValue OtherVal = DAG.getNode(
478 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
479 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
480 }
481
482 return SDValue(ScalarNode, ResNo);
483}
484
485SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
486 unsigned ResNo) {
487 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
488 return GetScalarizedVector(Op);
489}
490
491SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
492 SDLoc DL(N);
493 // Reuse the expansion (which should scalarize).
494 SDValue Mask = TLI.expandLoopDependenceMask(N, DAG);
495 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
496 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
497 N2: DAG.getVectorIdxConstant(Val: 0, DL));
498}
499
500SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
501 SDValue Op = N->getOperand(Num: 0);
502 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
503 Op = GetScalarizedVector(Op);
504 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
505 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
506 VT: NewVT, Operand: Op);
507}
508
509SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR_OR_SPLAT(SDNode *N) {
510 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
511 SDValue InOp = N->getOperand(Num: 0);
512 // The BUILD_VECTOR / SPLAT operands may be of wider element types and
513 // we may need to truncate them back to the requested return type.
514 if (EltVT.isInteger())
515 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
516 return InOp;
517}
518
519SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
520 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
521 VT: N->getValueType(ResNo: 0).getVectorElementType(),
522 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
523}
524
525SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
526 SDLoc DL(N);
527 SDValue Op = N->getOperand(Num: 0);
528 EVT OpVT = Op.getValueType();
529 // The result needs scalarizing, but it's not a given that the source does.
530 // See similar logic in ScalarizeVecRes_UnaryOp.
531 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
532 Op = GetScalarizedVector(Op);
533 } else {
534 EVT VT = OpVT.getVectorElementType();
535 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
536 }
537 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
538 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
539 N2: N->getOperand(Num: 1));
540}
541
542SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
543 SDLoc DL(N);
544 SDValue Op = N->getOperand(Num: 0);
545 EVT OpVT = Op.getValueType();
546 // The result needs scalarizing, but it's not a given that the source does.
547 // See similar logic in ScalarizeVecRes_UnaryOp.
548 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
549 Op = GetScalarizedVector(Op);
550 } else {
551 EVT VT = OpVT.getVectorElementType();
552 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
553 }
554 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
555 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
556 N2: N->getOperand(Num: 1));
557}
558
559SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
560 SDLoc DL(N);
561 SDValue Op = N->getOperand(Num: 0);
562 EVT OpVT = Op.getValueType();
563 // The result needs scalarizing, but it's not a given that the source does.
564 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
565 Op = GetScalarizedVector(Op);
566 } else {
567 EVT VT = OpVT.getVectorElementType();
568 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
569 }
570 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL,
571 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
572 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
573}
574
575SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
576 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
577 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
578 N2: N->getOperand(Num: 1));
579}
580
581SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOp_MultiType(SDNode *N) {
582 SDLoc DL(N);
583 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
584 SDValue RHS = N->getOperand(Num: 1);
585 EVT RHSVT = RHS.getValueType();
586 // The exponent has its own type action and may not have been scalarized:
587 // v1i1 is legal on AVX-512, v1i32 is widened on AArch64.
588 if (RHSVT.isVector()) {
589 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeScalarizeVector)
590 RHS = GetScalarizedVector(Op: RHS);
591 else
592 RHS = DAG.getExtractVectorElt(DL, VT: RHSVT.getVectorElementType(), Vec: RHS, Idx: 0);
593 }
594 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHS.getValueType(), N1: LHS, N2: RHS,
595 Flags: N->getFlags());
596}
597
598SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
599 // The value to insert may have a wider type than the vector element type,
600 // so be sure to truncate it to the element type if necessary.
601 SDValue Op = N->getOperand(Num: 1);
602 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
603 if (Op.getValueType() != EltVT)
604 // FIXME: Can this happen for floating point types?
605 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
606 return Op;
607}
608
609SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
610 SDValue Result = DAG.getAtomicLoad(
611 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
612 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
613 MMO: N->getMemOperand());
614
615 // Legalize the chain result - switch anything that used the old chain to
616 // use the new one.
617 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
618 return Result;
619}
620
621SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_RMW(AtomicSDNode *N) {
622 SDValue Result = DAG.getAtomic(
623 Opcode: N->getOpcode(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
624 Chain: N->getChain(), Ptr: N->getBasePtr(), Val: GetScalarizedVector(Op: N->getVal()),
625 MMO: N->getMemOperand());
626
627 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
628 return Result;
629}
630
631SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
632 assert(N->isUnindexed() && "Indexed vector load?");
633
634 SDValue Result = DAG.getLoad(
635 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
636 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
637 Ptr: N->getBasePtr(), Offset: DAG.getPOISON(VT: N->getBasePtr().getValueType()),
638 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
639 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(),
640 Metadata: N->getMMOMetadataForSubAccess());
641
642 // Legalize the chain result - switch anything that used the old chain to
643 // use the new one.
644 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
645 return Result;
646}
647
648SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
649 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
650 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
651 SDValue Op = N->getOperand(Num: 0);
652 EVT OpVT = Op.getValueType();
653 SDLoc DL(N);
654 // The result needs scalarizing, but it's not a given that the source does.
655 // This is a workaround for targets where it's impossible to scalarize the
656 // result of a conversion, because the source type is legal.
657 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
658 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
659 // legal and was not scalarized.
660 // See the similar logic in ScalarizeVecRes_SETCC
661 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
662 Op = GetScalarizedVector(Op);
663 } else {
664 EVT VT = OpVT.getVectorElementType();
665 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
666 }
667 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
668}
669
670SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
671 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
672 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
673 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
674 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
675 N1: LHS, N2: DAG.getValueType(ExtVT));
676}
677
678SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
679 SDLoc DL(N);
680 SDValue Op = N->getOperand(Num: 0);
681
682 EVT OpVT = Op.getValueType();
683 EVT OpEltVT = OpVT.getVectorElementType();
684 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
685
686 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
687 Op = GetScalarizedVector(Op);
688 } else {
689 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
690 }
691
692 switch (N->getOpcode()) {
693 case ISD::ANY_EXTEND_VECTOR_INREG:
694 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
695 case ISD::SIGN_EXTEND_VECTOR_INREG:
696 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
697 case ISD::ZERO_EXTEND_VECTOR_INREG:
698 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
699 }
700
701 llvm_unreachable("Illegal extend_vector_inreg opcode");
702}
703
704SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
705 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
706 SDValue Op = N->getOperand(Num: 0);
707 EVT OpVT = Op.getValueType();
708 SDLoc DL(N);
709 // The result needs scalarizing, but it's not a given that the source does.
710 // This is a workaround for targets where it's impossible to scalarize the
711 // result of a conversion, because the source type is legal.
712 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
713 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
714 // legal and was not scalarized.
715 // See the similar logic in ScalarizeVecRes_SETCC
716 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
717 Op = GetScalarizedVector(Op);
718 } else {
719 EVT VT = OpVT.getVectorElementType();
720 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
721 }
722 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
723 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
724 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
725 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS,
726 Flags: AddrSpaceCastN->getFlags());
727}
728
729SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
730 // If the operand is wider than the vector element type then it is implicitly
731 // truncated. Make that explicit here.
732 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
733 SDValue InOp = N->getOperand(Num: 0);
734 if (InOp.getValueType() != EltVT)
735 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
736 return InOp;
737}
738
739SDValue
740DAGTypeLegalizer::ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
741 assert(N->getNumValues() == N->getNumOperands() &&
742 "Expected one result per operand");
743
744 // Interleaving or deinterleaving one-element vectors leaves each result
745 // equal to the corresponding operand.
746 for (unsigned I = 0; I != N->getNumValues(); ++I)
747 SetScalarizedVector(Op: SDValue(N, I), Result: GetScalarizedVector(Op: N->getOperand(Num: I)));
748 return SDValue();
749}
750
751SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
752 SDValue Cond = N->getOperand(Num: 0);
753 EVT OpVT = Cond.getValueType();
754 SDLoc DL(N);
755 // The vselect result and true/value operands needs scalarizing, but it's
756 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
757 // See the similar logic in ScalarizeVecRes_SETCC
758 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
759 Cond = GetScalarizedVector(Op: Cond);
760 } else {
761 EVT VT = OpVT.getVectorElementType();
762 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
763 }
764
765 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
766 TargetLowering::BooleanContent ScalarBool =
767 TLI.getBooleanContents(isVec: false, isFloat: false);
768 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
769
770 // If integer and float booleans have different contents then we can't
771 // reliably optimize in all cases. There is a full explanation for this in
772 // DAGCombiner::visitSELECT() where the same issue affects folding
773 // (select C, 0, 1) to (xor C, 1).
774 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
775 TLI.getBooleanContents(isVec: false, isFloat: true)) {
776 // At least try the common case where the boolean is generated by a
777 // comparison.
778 if (Cond->getOpcode() == ISD::SETCC) {
779 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
780 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
781 VecBool = TLI.getBooleanContents(Type: OpVT);
782 } else
783 ScalarBool = TargetLowering::UndefinedBooleanContent;
784 }
785
786 EVT CondVT = Cond.getValueType();
787 if (ScalarBool != VecBool) {
788 switch (ScalarBool) {
789 case TargetLowering::UndefinedBooleanContent:
790 break;
791 case TargetLowering::ZeroOrOneBooleanContent:
792 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
793 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
794 // Vector read from all ones, scalar expects a single 1 so mask.
795 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
796 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
797 break;
798 case TargetLowering::ZeroOrNegativeOneBooleanContent:
799 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
800 VecBool == TargetLowering::ZeroOrOneBooleanContent);
801 // Vector reads from a one, scalar from all ones so sign extend.
802 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
803 N1: Cond, N2: DAG.getValueType(MVT::i1));
804 break;
805 }
806 }
807
808 // Truncate the condition if needed
809 auto BoolVT = getSetCCResultType(VT: CondVT);
810 if (BoolVT.bitsLT(VT: CondVT))
811 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
812
813 return DAG.getSelect(DL: SDLoc(N), VT: LHS.getValueType(), Cond, LHS,
814 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)), Flags: N->getFlags());
815}
816
817SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
818 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
819 return DAG.getSelect(DL: SDLoc(N),
820 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
821 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
822}
823
824SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
825 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
826 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
827 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
828 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
829 N5: N->getOperand(Num: 4));
830}
831
832SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
833 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
834}
835
836SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
837 // Figure out if the scalar is the LHS or RHS and return it.
838 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
839 if (Arg.isUndef())
840 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
841 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
842 return GetScalarizedVector(Op: N->getOperand(Num: Op));
843}
844
845SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
846 SDValue Src = N->getOperand(Num: 0);
847 EVT SrcVT = Src.getValueType();
848 SDLoc dl(N);
849
850 // Handle case where result is scalarized but operand is not
851 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
852 Src = GetScalarizedVector(Op: Src);
853 else
854 Src = DAG.getNode(
855 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
856 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
857
858 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
859 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
860}
861
862SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
863 assert(N->getValueType(0).isVector() &&
864 N->getOperand(0).getValueType().isVector() &&
865 "Operand types must be vectors");
866 SDValue LHS = N->getOperand(Num: 0);
867 SDValue RHS = N->getOperand(Num: 1);
868 EVT OpVT = LHS.getValueType();
869 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
870 SDLoc DL(N);
871
872 // The result needs scalarizing, but it's not a given that the source does.
873 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
874 LHS = GetScalarizedVector(Op: LHS);
875 RHS = GetScalarizedVector(Op: RHS);
876 } else {
877 EVT VT = OpVT.getVectorElementType();
878 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
879 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
880 }
881
882 // Turn it into a scalar SETCC.
883 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
884 N3: N->getOperand(Num: 2));
885 // Vectors may have a different boolean contents to scalars. Promote the
886 // value appropriately.
887 ISD::NodeType ExtendCode =
888 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
889 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
890}
891
892SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
893 SDLoc DL(N);
894 SDValue Arg = N->getOperand(Num: 0);
895 SDValue Test = N->getOperand(Num: 1);
896 EVT ArgVT = Arg.getValueType();
897 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
898
899 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
900 Arg = GetScalarizedVector(Op: Arg);
901 } else {
902 EVT VT = ArgVT.getVectorElementType();
903 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
904 }
905
906 SDValue Res =
907 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
908 // Vectors may have a different boolean contents to scalars. Promote the
909 // value appropriately.
910 ISD::NodeType ExtendCode =
911 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
912 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
913}
914
915//===----------------------------------------------------------------------===//
916// Operand Vector Scalarization <1 x ty> -> ty.
917//===----------------------------------------------------------------------===//
918
919bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
920 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
921 N->dump(&DAG));
922 SDValue Res = SDValue();
923
924 // See if the target wants to custom scalarize this node.
925 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
926 return false;
927
928 switch (N->getOpcode()) {
929 default:
930#ifndef NDEBUG
931 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
932 N->dump(&DAG);
933 dbgs() << "\n";
934#endif
935 report_fatal_error(reason: "Do not know how to scalarize this operator's "
936 "operand!\n");
937 case ISD::BITCAST:
938 Res = ScalarizeVecOp_BITCAST(N);
939 break;
940 case ISD::FAKE_USE:
941 Res = ScalarizeVecOp_FAKE_USE(N);
942 break;
943 case ISD::ANY_EXTEND:
944 case ISD::ZERO_EXTEND:
945 case ISD::SIGN_EXTEND:
946 case ISD::TRUNCATE:
947 case ISD::FP_TO_SINT:
948 case ISD::FP_TO_UINT:
949 case ISD::SINT_TO_FP:
950 case ISD::UINT_TO_FP:
951 case ISD::LROUND:
952 case ISD::LLROUND:
953 case ISD::LRINT:
954 case ISD::LLRINT:
955 Res = ScalarizeVecOp_UnaryOp(N);
956 break;
957 case ISD::FP_TO_SINT_SAT:
958 case ISD::FP_TO_UINT_SAT:
959 case ISD::CONVERT_FROM_ARBITRARY_FP:
960 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
961 break;
962 case ISD::CONVERT_TO_ARBITRARY_FP: {
963 assert(N->getValueType(0).getVectorNumElements() == 1 &&
964 "Unexpected vector type!");
965 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
966 SDValue Op = DAG.getNode(
967 Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(), N1: Elt,
968 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
969 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
970 break;
971 }
972 case ISD::STRICT_SINT_TO_FP:
973 case ISD::STRICT_UINT_TO_FP:
974 case ISD::STRICT_FP_TO_SINT:
975 case ISD::STRICT_FP_TO_UINT:
976 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
977 break;
978 case ISD::CONCAT_VECTORS:
979 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
980 break;
981 case ISD::INSERT_SUBVECTOR:
982 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
983 break;
984 case ISD::EXTRACT_VECTOR_ELT:
985 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
986 break;
987 case ISD::VSELECT:
988 Res = ScalarizeVecOp_VSELECT(N);
989 break;
990 case ISD::SETCC:
991 Res = ScalarizeVecOp_VSETCC(N);
992 break;
993 case ISD::STRICT_FSETCC:
994 case ISD::STRICT_FSETCCS:
995 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
996 break;
997 case ISD::STORE:
998 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
999 break;
1000 case ISD::ATOMIC_STORE:
1001 Res = ScalarizeVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
1002 break;
1003 case ISD::STRICT_FP_ROUND:
1004 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
1005 break;
1006 case ISD::FP_ROUND:
1007 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
1008 break;
1009 case ISD::STRICT_FP_EXTEND:
1010 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
1011 break;
1012 case ISD::FP_EXTEND:
1013 Res = ScalarizeVecOp_FP_EXTEND(N);
1014 break;
1015 case ISD::VECREDUCE_FADD:
1016 case ISD::VECREDUCE_FMUL:
1017 case ISD::VECREDUCE_ADD:
1018 case ISD::VECREDUCE_MUL:
1019 case ISD::VECREDUCE_AND:
1020 case ISD::VECREDUCE_OR:
1021 case ISD::VECREDUCE_XOR:
1022 case ISD::VECREDUCE_SMAX:
1023 case ISD::VECREDUCE_SMIN:
1024 case ISD::VECREDUCE_UMAX:
1025 case ISD::VECREDUCE_UMIN:
1026 case ISD::VECREDUCE_FMAX:
1027 case ISD::VECREDUCE_FMIN:
1028 case ISD::VECREDUCE_FMAXIMUM:
1029 case ISD::VECREDUCE_FMINIMUM:
1030 case ISD::VECREDUCE_FMAXIMUMNUM:
1031 case ISD::VECREDUCE_FMINIMUMNUM:
1032 Res = ScalarizeVecOp_VECREDUCE(N);
1033 break;
1034 case ISD::VECREDUCE_SEQ_FADD:
1035 case ISD::VECREDUCE_SEQ_FMUL:
1036 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
1037 break;
1038 case ISD::SCMP:
1039 case ISD::UCMP:
1040 Res = ScalarizeVecOp_CMP(N);
1041 break;
1042 case ISD::VECTOR_FIND_LAST_ACTIVE:
1043 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
1044 break;
1045 case ISD::CTTZ_ELTS:
1046 case ISD::CTTZ_ELTS_ZERO_POISON:
1047 Res = ScalarizeVecOp_CTTZ_ELTS(N);
1048 break;
1049 case ISD::VECTOR_MATCH:
1050 Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
1051 break;
1052 case ISD::MASKED_UDIV:
1053 case ISD::MASKED_SDIV:
1054 case ISD::MASKED_UREM:
1055 case ISD::MASKED_SREM:
1056 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
1057 break;
1058 }
1059
1060 // If the result is null, the sub-method took care of registering results etc.
1061 if (!Res.getNode()) return false;
1062
1063 // If the result is N, the sub-method updated N in place. Tell the legalizer
1064 // core about this.
1065 if (Res.getNode() == N)
1066 return true;
1067
1068 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1069 "Invalid operand expansion");
1070
1071 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1072 return false;
1073}
1074
1075/// If the value to convert is a vector that needs to be scalarized, it must be
1076/// <1 x ty>. Convert the element instead.
1077SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
1078 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1079 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
1080 VT: N->getValueType(ResNo: 0), Operand: Elt);
1081}
1082
1083// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1084SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1085 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1086 "Fake Use: Unexpected vector type!");
1087 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1088 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
1089}
1090
1091/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1092/// Do the operation on the element instead.
1093SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1094 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1095 "Unexpected vector type!");
1096 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1097 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1098 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
1099 // Revectorize the result so the types line up with what the uses of this
1100 // expression expect.
1101 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1102}
1103
1104/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1105/// typesize).
1106SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1107 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1108 "Unexpected vector type!");
1109 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1110 SDValue Op =
1111 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
1112 N1: Elt, N2: N->getOperand(Num: 1));
1113 // Revectorize the result so the types line up with what the uses of this
1114 // expression expect.
1115 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1116}
1117
1118/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1119/// Do the strict FP operation on the element instead.
1120SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1121 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1122 "Unexpected vector type!");
1123 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1124 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1125 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
1126 Ops: { N->getOperand(Num: 0), Elt });
1127 // Legalize the chain result - switch anything that used the old chain to
1128 // use the new one.
1129 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1130 // Revectorize the result so the types line up with what the uses of this
1131 // expression expect.
1132 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1133
1134 // Do our own replacement and return SDValue() to tell the caller that we
1135 // handled all replacements since caller can only handle a single result.
1136 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1137 return SDValue();
1138}
1139
1140/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1141SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1142 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1143 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1144 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1145 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1146}
1147
1148/// The inserted subvector is to be scalarized - use insert vector element
1149/// instead.
1150SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1151 unsigned OpNo) {
1152 // We should not be attempting to scalarize the containing vector
1153 assert(OpNo == 1);
1154 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1155 SDValue ContainingVec = N->getOperand(Num: 0);
1156 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1157 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1158 N3: N->getOperand(Num: 2));
1159}
1160
1161/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1162/// so just return the element, ignoring the index.
1163SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1164 EVT VT = N->getValueType(ResNo: 0);
1165 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1166 if (Res.getValueType() != VT)
1167 Res = VT.isFloatingPoint()
1168 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1169 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1170 return Res;
1171}
1172
1173/// If the input condition is a vector that needs to be scalarized, it must be
1174/// <1 x i1>, so just convert to a normal ISD::SELECT
1175/// (still with vector output type since that was acceptable if we got here).
1176SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1177 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1178 EVT VT = N->getValueType(ResNo: 0);
1179
1180 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1181 N3: N->getOperand(Num: 2));
1182}
1183
1184/// If the operand is a vector that needs to be scalarized then the
1185/// result must be a single-element vector, so just convert to a scalar
1186/// SETCC and wrap with a scalar_to_vector since the res type is legal
1187/// if we got here
1188SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1189 assert(N->getValueType(0).isVector() &&
1190 N->getOperand(0).getValueType().isVector() &&
1191 "Operand types must be vectors");
1192 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1193 "Expected single-element vector type");
1194
1195 EVT VT = N->getValueType(ResNo: 0);
1196 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1197 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1198
1199 EVT OpVT = N->getOperand(Num: 0).getValueType();
1200 EVT NVT = VT.getVectorElementType();
1201 SDLoc DL(N);
1202 // Turn it into a scalar SETCC.
1203 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1204 N3: N->getOperand(Num: 2));
1205
1206 // Vectors may have a different boolean contents to scalars. Promote the
1207 // value appropriately.
1208 ISD::NodeType ExtendCode =
1209 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1210
1211 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1212
1213 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1214}
1215
1216// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1217SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1218 unsigned OpNo) {
1219 assert(OpNo == 1 && "Wrong operand for scalarization!");
1220 assert(N->getValueType(0).isVector() &&
1221 N->getOperand(1).getValueType().isVector() &&
1222 "Operand types must be vectors");
1223 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1224 "Expected single-element vector type");
1225
1226 EVT VT = N->getValueType(ResNo: 0);
1227 SDValue Ch = N->getOperand(Num: 0);
1228 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1229 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1230 SDValue CC = N->getOperand(Num: 3);
1231
1232 EVT OpVT = N->getOperand(Num: 1).getValueType();
1233 EVT NVT = VT.getVectorElementType();
1234 SDLoc DL(N);
1235 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1236 Ops: {Ch, LHS, RHS, CC});
1237
1238 // Legalize the chain result - switch anything that used the old chain to
1239 // use the new one.
1240 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1241
1242 ISD::NodeType ExtendCode =
1243 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1244
1245 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1246 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1247
1248 // Do our own replacement and return SDValue() to tell the caller that we
1249 // handled all replacements since caller can only handle a single result.
1250 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1251 return SDValue();
1252}
1253
1254/// If the value to store is a vector that needs to be scalarized, it must be
1255/// <1 x ty>. Just store the element.
1256SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1257 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1258 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1259 SDLoc dl(N);
1260
1261 if (N->isTruncatingStore())
1262 return DAG.getTruncStore(
1263 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1264 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1265 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1266 MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getMMOMetadataForSubAccess());
1267
1268 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1269 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1270 MMOFlags: N->getMemOperand()->getFlags(),
1271 Metadata: N->getMMOMetadataForSubAccess());
1272}
1273
1274/// If the value to store is a vector that needs to be scalarized, it must be
1275/// <1 x ty>. Just store the element.
1276SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1277 SDValue ScalarVal = GetScalarizedVector(Op: N->getVal());
1278 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SDLoc(N),
1279 MemVT: N->getMemoryVT().getVectorElementType(), Chain: N->getChain(),
1280 Ptr: ScalarVal, Val: N->getBasePtr(), MMO: N->getMemOperand());
1281}
1282
1283/// If the value to round is a vector that needs to be scalarized, it must be
1284/// <1 x ty>. Convert the element instead.
1285SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1286 assert(OpNo == 0 && "Wrong operand for scalarization!");
1287 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1288 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1289 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1290 N2: N->getOperand(Num: 1));
1291 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1292}
1293
1294SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1295 unsigned OpNo) {
1296 assert(OpNo == 1 && "Wrong operand for scalarization!");
1297 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1298 SDValue Res =
1299 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1300 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1301 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1302 // Legalize the chain result - switch anything that used the old chain to
1303 // use the new one.
1304 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1305
1306 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1307
1308 // Do our own replacement and return SDValue() to tell the caller that we
1309 // handled all replacements since caller can only handle a single result.
1310 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1311 return SDValue();
1312}
1313
1314/// If the value to extend is a vector that needs to be scalarized, it must be
1315/// <1 x ty>. Convert the element instead.
1316SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1317 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1318 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1319 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1320 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1321}
1322
1323/// If the value to extend is a vector that needs to be scalarized, it must be
1324/// <1 x ty>. Convert the element instead.
1325SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1326 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1327 SDValue Res =
1328 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1329 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1330 Ops: {N->getOperand(Num: 0), Elt});
1331 // Legalize the chain result - switch anything that used the old chain to
1332 // use the new one.
1333 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1334
1335 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1336
1337 // Do our own replacement and return SDValue() to tell the caller that we
1338 // handled all replacements since caller can only handle a single result.
1339 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1340 return SDValue();
1341}
1342
1343SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1344 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1345 // Result type may be wider than element type.
1346 if (Res.getValueType() != N->getValueType(ResNo: 0))
1347 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1348 return Res;
1349}
1350
1351SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1352 SDValue AccOp = N->getOperand(Num: 0);
1353 SDValue VecOp = N->getOperand(Num: 1);
1354
1355 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1356
1357 SDValue Op = GetScalarizedVector(Op: VecOp);
1358 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1359 N1: AccOp, N2: Op, Flags: N->getFlags());
1360}
1361
1362SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1363 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1364 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1365
1366 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1367 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1368 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1369}
1370
1371SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1372 // Since there is no "none-active" result, the only valid return for <1 x ty>
1373 // is 0. Note: Since we check the high mask during splitting this is safe.
1374 // As e.g., a <2 x ty> operation would split to:
1375 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1376 // : `last_active(%lo_mask)`
1377 // Which then scalarizes to:
1378 // %mask[1] ? 1 : 0
1379 EVT VT = N->getValueType(ResNo: 0);
1380 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1381}
1382
1383SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1384 // The number of trailing zero elements is 1 if the element is 0, and 0
1385 // otherwise.
1386 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1387 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1388 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1389 SDValue SetCC =
1390 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1391 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1392 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1393}
1394
1395SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
1396 SDLoc DL(N);
1397 // Reuse the expansion (which should scalarize).
1398 SDValue Mask = TLI.expandVectorMatch(N, DAG);
1399 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL,
1400 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
1401 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1402}
1403
1404SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
1405 unsigned OpNo) {
1406 return TLI.expandVectorMatch(N, DAG);
1407}
1408
1409SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1410 assert(OpNo == 2 && "Can only scalarize mask operand");
1411 SDLoc DL(N);
1412 EVT VT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1413 SDValue LHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 0), Idx: 0);
1414 SDValue RHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 1), Idx: 0);
1415 SDValue Mask = GetScalarizedVector(Op: N->getOperand(Num: 2));
1416 // Vectors may have a different boolean contents to scalars, so truncate to i1
1417 // and let type legalization promote appropriately.
1418 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
1419 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1420 // use 1 as the divisor to avoid division by zero and overflow.
1421 SDValue BinOp =
1422 DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL, VT, N1: LHS,
1423 N2: DAG.getSelect(DL, VT, Cond: Mask, LHS: RHS, RHS: DAG.getConstant(Val: 1, DL, VT)));
1424 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: N->getValueType(ResNo: 0), Operand: BinOp);
1425}
1426
1427//===----------------------------------------------------------------------===//
1428// Result Vector Splitting
1429//===----------------------------------------------------------------------===//
1430
1431/// This method is called when the specified result of the specified node is
1432/// found to need vector splitting. At this point, the node may also have
1433/// invalid operands or may have other results that need legalization, we just
1434/// know that (at least) one result needs vector splitting.
1435void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1436 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1437 SDValue Lo, Hi;
1438
1439 // See if the target wants to custom expand this node.
1440 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1441 return;
1442
1443 switch (N->getOpcode()) {
1444 default:
1445#ifndef NDEBUG
1446 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1447 N->dump(&DAG);
1448 dbgs() << "\n";
1449#endif
1450 report_fatal_error(reason: "Do not know how to split the result of this "
1451 "operator!\n");
1452
1453 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1454 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1455 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1456 break;
1457 case ISD::MASK_BEFOREFIRST:
1458 SplitVecRes_MASK_BEFOREFIRST(N, Lo, Hi);
1459 break;
1460 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1461 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1462 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1463 case ISD::VSELECT:
1464 case ISD::SELECT:
1465 case ISD::VP_MERGE: SplitRes_Select(N, Lo, Hi); break;
1466 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1467 case ISD::POISON:
1468 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1469 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1470 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1471 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1472 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1473 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1474 case ISD::FPOWI:
1475 case ISD::FLDEXP:
1476 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1477 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1478 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1479 case ISD::SPLAT_VECTOR:
1480 case ISD::SCALAR_TO_VECTOR:
1481 SplitVecRes_ScalarOp(N, Lo, Hi);
1482 break;
1483 case ISD::STEP_VECTOR:
1484 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1485 break;
1486 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1487 case ISD::ATOMIC_LOAD:
1488 SplitVecRes_ATOMIC_LOAD(LD: cast<AtomicSDNode>(Val: N), Lo, Hi);
1489 break;
1490 case ISD::LOAD:
1491 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1492 break;
1493 case ISD::VP_LOAD:
1494 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1495 break;
1496 case ISD::VP_LOAD_FF:
1497 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1498 break;
1499 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1500 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1501 break;
1502 case ISD::MLOAD:
1503 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1504 break;
1505 case ISD::MGATHER:
1506 case ISD::VP_GATHER:
1507 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1508 break;
1509 case ISD::VECTOR_COMPRESS:
1510 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1511 break;
1512 case ISD::SETCC:
1513 SplitVecRes_SETCC(N, Lo, Hi);
1514 break;
1515 case ISD::VECTOR_REPEAT:
1516 SplitVecRes_VECTOR_REPEAT(N, Lo, Hi);
1517 break;
1518 case ISD::VECTOR_REVERSE:
1519 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1520 break;
1521 case ISD::VECTOR_SHUFFLE:
1522 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1523 break;
1524 case ISD::VECTOR_SPLICE_LEFT:
1525 case ISD::VECTOR_SPLICE_RIGHT:
1526 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1527 break;
1528 case ISD::VECTOR_DEINTERLEAVE:
1529 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1530 return;
1531 case ISD::VECTOR_INTERLEAVE:
1532 SplitVecRes_VECTOR_INTERLEAVE(N);
1533 return;
1534 case ISD::VAARG:
1535 SplitVecRes_VAARG(N, Lo, Hi);
1536 break;
1537
1538 case ISD::ANY_EXTEND_VECTOR_INREG:
1539 case ISD::SIGN_EXTEND_VECTOR_INREG:
1540 case ISD::ZERO_EXTEND_VECTOR_INREG:
1541 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1542 break;
1543
1544 case ISD::ABS:
1545 case ISD::ABS_MIN_POISON:
1546 case ISD::BITREVERSE:
1547 case ISD::BSWAP:
1548 case ISD::CTLZ:
1549 case ISD::CTTZ:
1550 case ISD::CTLZ_ZERO_POISON:
1551 case ISD::CTTZ_ZERO_POISON:
1552 case ISD::CTPOP:
1553 case ISD::FABS:
1554 case ISD::FACOS:
1555 case ISD::FASIN:
1556 case ISD::FATAN:
1557 case ISD::FCEIL:
1558 case ISD::FCOS:
1559 case ISD::FCOSH:
1560 case ISD::FEXP:
1561 case ISD::FEXP2:
1562 case ISD::FEXP10:
1563 case ISD::FFLOOR:
1564 case ISD::FLOG:
1565 case ISD::FLOG10:
1566 case ISD::FLOG2:
1567 case ISD::FNEARBYINT:
1568 case ISD::FNEG:
1569 case ISD::FREEZE:
1570 case ISD::ARITH_FENCE:
1571 case ISD::FP_EXTEND:
1572 case ISD::FP_ROUND:
1573 case ISD::FP_TO_SINT:
1574 case ISD::FP_TO_UINT:
1575 case ISD::FRINT:
1576 case ISD::LRINT:
1577 case ISD::LLRINT:
1578 case ISD::FROUND:
1579 case ISD::FROUNDEVEN:
1580 case ISD::LROUND:
1581 case ISD::LLROUND:
1582 case ISD::FSIN:
1583 case ISD::FSINH:
1584 case ISD::FSQRT:
1585 case ISD::FTAN:
1586 case ISD::FTANH:
1587 case ISD::FTRUNC:
1588 case ISD::SINT_TO_FP:
1589 case ISD::TRUNCATE:
1590 case ISD::UINT_TO_FP:
1591 case ISD::FCANONICALIZE:
1592 case ISD::AssertNoFPClass:
1593 case ISD::CONVERT_FROM_ARBITRARY_FP:
1594 case ISD::CONVERT_TO_ARBITRARY_FP:
1595 SplitVecRes_UnaryOp(N, Lo, Hi);
1596 break;
1597 case ISD::ADDRSPACECAST:
1598 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1599 break;
1600 case ISD::FMODF:
1601 case ISD::FFREXP:
1602 case ISD::FSINCOS:
1603 case ISD::FSINCOSPI:
1604 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1605 break;
1606
1607 case ISD::ANY_EXTEND:
1608 case ISD::SIGN_EXTEND:
1609 case ISD::ZERO_EXTEND:
1610 SplitVecRes_ExtendOp(N, Lo, Hi);
1611 break;
1612
1613 case ISD::ADD:
1614 case ISD::SUB:
1615 case ISD::MUL:
1616 case ISD::CLMUL:
1617 case ISD::CLMULR:
1618 case ISD::CLMULH:
1619 case ISD::PEXT:
1620 case ISD::PDEP:
1621 case ISD::MULHS:
1622 case ISD::MULHU:
1623 case ISD::ABDS:
1624 case ISD::ABDU:
1625 case ISD::AVGCEILS:
1626 case ISD::AVGCEILU:
1627 case ISD::AVGFLOORS:
1628 case ISD::AVGFLOORU:
1629 case ISD::FADD:
1630 case ISD::FSUB:
1631 case ISD::FMUL:
1632 case ISD::FMINNUM:
1633 case ISD::FMINNUM_IEEE:
1634 case ISD::FMAXNUM:
1635 case ISD::FMAXNUM_IEEE:
1636 case ISD::FMINIMUM:
1637 case ISD::FMAXIMUM:
1638 case ISD::FMINIMUMNUM:
1639 case ISD::FMAXIMUMNUM:
1640 case ISD::SDIV: case ISD::VP_SDIV:
1641 case ISD::UDIV: case ISD::VP_UDIV:
1642 case ISD::FDIV:
1643 case ISD::FPOW:
1644 case ISD::FATAN2:
1645 case ISD::AND:
1646 case ISD::OR:
1647 case ISD::XOR:
1648 case ISD::SHL:
1649 case ISD::SRA:
1650 case ISD::SRL:
1651 case ISD::UREM: case ISD::VP_UREM:
1652 case ISD::SREM: case ISD::VP_SREM:
1653 case ISD::FREM:
1654 case ISD::SMIN:
1655 case ISD::SMAX:
1656 case ISD::UMIN:
1657 case ISD::UMAX:
1658 case ISD::SADDSAT:
1659 case ISD::UADDSAT:
1660 case ISD::SSUBSAT:
1661 case ISD::USUBSAT:
1662 case ISD::SSHLSAT:
1663 case ISD::USHLSAT:
1664 case ISD::ROTL:
1665 case ISD::ROTR:
1666 SplitVecRes_BinOp(N, Lo, Hi);
1667 break;
1668 case ISD::MASKED_UDIV:
1669 case ISD::MASKED_SDIV:
1670 case ISD::MASKED_UREM:
1671 case ISD::MASKED_SREM:
1672 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1673 break;
1674 case ISD::FMA:
1675 case ISD::FSHL:
1676 case ISD::FSHR:
1677 SplitVecRes_TernaryOp(N, Lo, Hi);
1678 break;
1679
1680 case ISD::SCMP: case ISD::UCMP:
1681 SplitVecRes_CMP(N, Lo, Hi);
1682 break;
1683
1684#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1685 case ISD::STRICT_##DAGN:
1686#include "llvm/IR/ConstrainedOps.def"
1687 SplitVecRes_StrictFPOp(N, Lo, Hi);
1688 break;
1689
1690 case ISD::FP_TO_UINT_SAT:
1691 case ISD::FP_TO_SINT_SAT:
1692 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1693 break;
1694
1695 case ISD::UADDO:
1696 case ISD::SADDO:
1697 case ISD::USUBO:
1698 case ISD::SSUBO:
1699 case ISD::UMULO:
1700 case ISD::SMULO:
1701 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1702 break;
1703 case ISD::SMULFIX:
1704 case ISD::SMULFIXSAT:
1705 case ISD::UMULFIX:
1706 case ISD::UMULFIXSAT:
1707 case ISD::SDIVFIX:
1708 case ISD::SDIVFIXSAT:
1709 case ISD::UDIVFIX:
1710 case ISD::UDIVFIXSAT:
1711 SplitVecRes_FIX(N, Lo, Hi);
1712 break;
1713 case ISD::EXPERIMENTAL_VP_SPLICE:
1714 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1715 break;
1716 case ISD::EXPERIMENTAL_VP_REVERSE:
1717 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1718 break;
1719 case ISD::PARTIAL_REDUCE_UMLA:
1720 case ISD::PARTIAL_REDUCE_SMLA:
1721 case ISD::PARTIAL_REDUCE_SUMLA:
1722 case ISD::PARTIAL_REDUCE_FMLA:
1723 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1724 break;
1725 case ISD::GET_ACTIVE_LANE_MASK:
1726 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1727 break;
1728 case ISD::VECTOR_MATCH:
1729 SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
1730 break;
1731 }
1732
1733 // If Lo/Hi is null, the sub-method took care of registering results etc.
1734 if (Lo.getNode())
1735 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1736}
1737
1738void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1739 MachinePointerInfo &MPI, SDValue &Ptr,
1740 uint64_t *ScaledOffset) {
1741 SDLoc DL(N);
1742 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1743
1744 if (MemVT.isScalableVector()) {
1745 SDValue BytesIncrement = DAG.getVScale(
1746 DL, VT: Ptr.getValueType(),
1747 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1748 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1749 if (ScaledOffset)
1750 *ScaledOffset += IncrementSize;
1751 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1752 Flags: SDNodeFlags::NoUnsignedWrap);
1753 } else {
1754 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1755 // Increment the pointer to the other half.
1756 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1757 }
1758}
1759
1760std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1761 return SplitMask(Mask, DL: SDLoc(Mask));
1762}
1763
1764std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1765 const SDLoc &DL) {
1766 SDValue MaskLo, MaskHi;
1767 EVT MaskVT = Mask.getValueType();
1768 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1769 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1770 else
1771 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1772 return std::make_pair(x&: MaskLo, y&: MaskHi);
1773}
1774
1775void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1776 SDValue LHSLo, LHSHi;
1777 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1778 SDValue RHSLo, RHSHi;
1779 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1780 SDLoc dl(N);
1781
1782 const SDNodeFlags Flags = N->getFlags();
1783 unsigned Opcode = N->getOpcode();
1784 if (N->getNumOperands() == 2) {
1785 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1786 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1787 return;
1788 }
1789
1790 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1791 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
1792 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
1793 "Expected VP opcode");
1794
1795 SDValue MaskLo, MaskHi;
1796 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1797
1798 SDValue EVLLo, EVLHi;
1799 std::tie(args&: EVLLo, args&: EVLHi) =
1800 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1801
1802 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1803 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1804 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1805 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1806}
1807
1808void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1809 SDValue &Hi) {
1810 SDValue LHSLo, LHSHi;
1811 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1812 SDValue RHSLo, RHSHi;
1813 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1814
1815 SDValue MaskLo, MaskHi, Mask = N->getOperand(Num: 2);
1816 if (Mask.getOpcode() == ISD::SETCC)
1817 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
1818 else
1819 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask);
1820
1821 SDLoc dl(N);
1822
1823 const SDNodeFlags Flags = N->getFlags();
1824 unsigned Opcode = N->getOpcode();
1825 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: MaskLo,
1826 Flags);
1827 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: MaskHi,
1828 Flags);
1829}
1830
1831void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1832 SDValue &Hi) {
1833 SDValue Op0Lo, Op0Hi;
1834 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1835 SDValue Op1Lo, Op1Hi;
1836 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1837 SDValue Op2Lo, Op2Hi;
1838 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1839 SDLoc dl(N);
1840
1841 const SDNodeFlags Flags = N->getFlags();
1842 unsigned Opcode = N->getOpcode();
1843 Lo =
1844 DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1845 Hi =
1846 DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1847}
1848
1849void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1850 LLVMContext &Ctxt = *DAG.getContext();
1851 SDLoc dl(N);
1852
1853 SDValue LHS = N->getOperand(Num: 0);
1854 SDValue RHS = N->getOperand(Num: 1);
1855
1856 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1857 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1858 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1859 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1860 } else {
1861 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1862 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1863 }
1864
1865 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1866 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1867 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1868}
1869
1870void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1871 SDValue LHSLo, LHSHi;
1872 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1873 SDValue RHSLo, RHSHi;
1874 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1875 SDLoc dl(N);
1876 SDValue Op2 = N->getOperand(Num: 2);
1877
1878 unsigned Opcode = N->getOpcode();
1879 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1880 Flags: N->getFlags());
1881 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1882 Flags: N->getFlags());
1883}
1884
1885void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1886 SDValue &Hi) {
1887 // We know the result is a vector. The input may be either a vector or a
1888 // scalar value.
1889 EVT LoVT, HiVT;
1890 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1891 SDLoc dl(N);
1892
1893 SDValue InOp = N->getOperand(Num: 0);
1894 EVT InVT = InOp.getValueType();
1895
1896 // Handle some special cases efficiently.
1897 switch (getTypeAction(VT: InVT)) {
1898 case TargetLowering::TypeLegal:
1899 case TargetLowering::TypePromoteInteger:
1900 case TargetLowering::TypeSoftPromoteHalf:
1901 case TargetLowering::TypeSoftenFloat:
1902 case TargetLowering::TypeScalarizeVector:
1903 case TargetLowering::TypeWidenVector:
1904 break;
1905 case TargetLowering::TypeExpandInteger:
1906 case TargetLowering::TypeExpandFloat:
1907 // A scalar to vector conversion, where the scalar needs expansion.
1908 // If the vector is being split in two then we can just convert the
1909 // expanded pieces.
1910 if (LoVT == HiVT) {
1911 GetExpandedOp(Op: InOp, Lo, Hi);
1912 if (DAG.getDataLayout().isBigEndian())
1913 std::swap(a&: Lo, b&: Hi);
1914 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1915 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1916 return;
1917 }
1918 break;
1919 case TargetLowering::TypeSplitVector:
1920 // If the input is a vector that needs to be split, convert each split
1921 // piece of the input now.
1922 GetSplitVector(Op: InOp, Lo, Hi);
1923 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1924 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1925 return;
1926 case TargetLowering::TypeScalarizeScalableVector:
1927 report_fatal_error(reason: "scalarization of scalable vectors is not supported");
1928 }
1929
1930 if (LoVT.isScalableVector()) {
1931 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1932 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1933 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1934 return;
1935 }
1936
1937 // In the general case, convert the input to an integer and split it by hand.
1938 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1939 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1940 if (DAG.getDataLayout().isBigEndian())
1941 std::swap(a&: LoIntVT, b&: HiIntVT);
1942
1943 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1944
1945 if (DAG.getDataLayout().isBigEndian())
1946 std::swap(a&: Lo, b&: Hi);
1947 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1948 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1949}
1950
1951void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1952 SDValue &Hi) {
1953 SDLoc DL(N);
1954 EVT LoVT, HiVT;
1955 SDValue PtrA = N->getOperand(Num: 0);
1956 SDValue PtrB = N->getOperand(Num: 1);
1957 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1958
1959 // The lane offset for the "Lo" half of the mask is unchanged.
1960 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1961 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1962 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1963 // The lane offset for the "Hi" half of the mask is incremented by the number
1964 // of elements in the "Lo" half.
1965 unsigned LaneOffset =
1966 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1967 // Note: The lane offset is implicitly scalable for scalable masks.
1968 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1969 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1970 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1971}
1972
1973void DAGTypeLegalizer::SplitVecRes_MASK_BEFOREFIRST(SDNode *N, SDValue &Lo,
1974 SDValue &Hi) {
1975 SDLoc DL(N);
1976 SDValue InLo, InHi;
1977 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
1978 EVT VT = InLo.getValueType();
1979 Lo = DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL, VT, Operand: InLo);
1980
1981 // hi = AnyLoActive ? all-zeros : (mask_beforefirst hi)
1982 SDValue AnyLoActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: InLo);
1983 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyLoActive, SL: DL,
1984 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
1985 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond, N2: DAG.getConstant(Val: 0, DL, VT),
1986 N3: DAG.getNode(Opcode: ISD::MASK_BEFOREFIRST, DL, VT, Operand: InHi));
1987}
1988
1989void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1990 SDValue &Hi) {
1991 EVT LoVT, HiVT;
1992 SDLoc dl(N);
1993 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1994 unsigned LoNumElts = LoVT.getVectorNumElements();
1995 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1996 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1997
1998 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1999 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
2000}
2001
2002void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
2003 SDValue &Hi) {
2004 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
2005 SDLoc dl(N);
2006 unsigned NumSubvectors = N->getNumOperands() / 2;
2007 if (NumSubvectors == 1) {
2008 Lo = N->getOperand(Num: 0);
2009 Hi = N->getOperand(Num: 1);
2010 return;
2011 }
2012
2013 EVT LoVT, HiVT;
2014 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2015
2016 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
2017 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
2018
2019 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
2020 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
2021}
2022
2023void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
2024 SDValue &Hi) {
2025 SDValue Vec = N->getOperand(Num: 0);
2026 SDValue Idx = N->getOperand(Num: 1);
2027 SDLoc dl(N);
2028
2029 EVT LoVT, HiVT;
2030 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2031
2032 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
2033 uint64_t IdxVal = Idx->getAsZExtVal();
2034 Hi = DAG.getNode(
2035 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
2036 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
2037}
2038
2039void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
2040 SDValue &Hi) {
2041 SDValue Vec = N->getOperand(Num: 0);
2042 SDValue SubVec = N->getOperand(Num: 1);
2043 SDValue Idx = N->getOperand(Num: 2);
2044 SDLoc dl(N);
2045 GetSplitVector(Op: Vec, Lo, Hi);
2046
2047 EVT VecVT = Vec.getValueType();
2048 EVT LoVT = Lo.getValueType();
2049 EVT SubVecVT = SubVec.getValueType();
2050 unsigned VecElems = VecVT.getVectorMinNumElements();
2051 unsigned SubElems = SubVecVT.getVectorMinNumElements();
2052 unsigned LoElems = LoVT.getVectorMinNumElements();
2053
2054 // If we know the index is in the first half, and we know the subvector
2055 // doesn't cross the boundary between the halves, we can avoid spilling the
2056 // vector, and insert into the lower half of the split vector directly.
2057 unsigned IdxVal = Idx->getAsZExtVal();
2058 if (IdxVal + SubElems <= LoElems) {
2059 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
2060 return;
2061 }
2062 // Similarly if the subvector is fully in the high half, but mind that we
2063 // can't tell whether a fixed-length subvector is fully within the high half
2064 // of a scalable vector.
2065 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
2066 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
2067 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
2068 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
2069 return;
2070 }
2071
2072 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
2073 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
2074 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
2075 if (WideSubVec.getValueType() == VecVT) {
2076 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
2077 return;
2078 }
2079 }
2080
2081 // Spill the vector to the stack.
2082 // In cases where the vector is illegal it will be broken down into parts
2083 // and stored in parts - we should use the alignment for the smallest part.
2084 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2085 SDValue StackPtr =
2086 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2087 auto &MF = DAG.getMachineFunction();
2088 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2089 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2090
2091 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2092 Alignment: SmallestAlign);
2093
2094 // Store the new subvector into the specified index.
2095 SDValue SubVecPtr =
2096 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
2097 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
2098 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2099
2100 // Load the Lo part from the stack slot.
2101 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
2102 Alignment: SmallestAlign);
2103
2104 // Increment the pointer to the other part.
2105 auto *Load = cast<LoadSDNode>(Val&: Lo);
2106 MachinePointerInfo MPI = Load->getPointerInfo();
2107 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2108
2109 // Load the Hi part from the stack slot.
2110 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2111}
2112
2113// Handle splitting an FP where the second operand does not match the first
2114// type. The second operand may be a scalar, or a vector that has exactly as
2115// many elements as the first
2116void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2117 SDValue &Hi) {
2118 SDValue LHSLo, LHSHi;
2119 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2120 SDLoc DL(N);
2121
2122 SDValue RHSLo, RHSHi;
2123 SDValue RHS = N->getOperand(Num: 1);
2124 EVT RHSVT = RHS.getValueType();
2125 if (RHSVT.isVector()) {
2126 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
2127 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
2128 else
2129 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
2130
2131 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
2132 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
2133 } else {
2134 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
2135 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
2136 }
2137}
2138
2139void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2140 SDValue &Hi) {
2141 SDLoc DL(N);
2142 SDValue ArgLo, ArgHi;
2143 SDValue Test = N->getOperand(Num: 1);
2144 SDValue FpValue = N->getOperand(Num: 0);
2145 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2146 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
2147 else
2148 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
2149 EVT LoVT, HiVT;
2150 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2151
2152 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
2153 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
2154}
2155
2156void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2157 SDValue &Hi) {
2158 SDValue LHSLo, LHSHi;
2159 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2160 SDLoc dl(N);
2161
2162 EVT LoVT, HiVT;
2163 std::tie(args&: LoVT, args&: HiVT) =
2164 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
2165
2166 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
2167 N2: DAG.getValueType(LoVT));
2168 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
2169 N2: DAG.getValueType(HiVT));
2170}
2171
2172void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2173 SDValue &Hi) {
2174 unsigned Opcode = N->getOpcode();
2175 SDValue N0 = N->getOperand(Num: 0);
2176
2177 SDLoc dl(N);
2178 SDValue InLo, InHi;
2179
2180 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
2181 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
2182 else
2183 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2184
2185 EVT InLoVT = InLo.getValueType();
2186 unsigned InNumElements = InLoVT.getVectorNumElements();
2187
2188 EVT OutLoVT, OutHiVT;
2189 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2190 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2191 assert((2 * OutNumElements) <= InNumElements &&
2192 "Illegal extend vector in reg split");
2193
2194 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2195 // input vector (i.e. we only use InLo):
2196 // OutLo will extend the first OutNumElements from InLo.
2197 // OutHi will extend the next OutNumElements from InLo.
2198
2199 // Shuffle the elements from InLo for OutHi into the bottom elements to
2200 // create a 'fake' InHi.
2201 SmallVector<int, 8> SplitHi(InNumElements, -1);
2202 for (unsigned i = 0; i != OutNumElements; ++i)
2203 SplitHi[i] = i + OutNumElements;
2204 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2205
2206 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2207 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2208}
2209
2210void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2211 SDValue &Hi) {
2212 unsigned NumOps = N->getNumOperands();
2213 SDValue Chain = N->getOperand(Num: 0);
2214 EVT LoVT, HiVT;
2215 SDLoc dl(N);
2216 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2217
2218 SmallVector<SDValue, 4> OpsLo(NumOps);
2219 SmallVector<SDValue, 4> OpsHi(NumOps);
2220
2221 // The Chain is the first operand.
2222 OpsLo[0] = Chain;
2223 OpsHi[0] = Chain;
2224
2225 // Now process the remaining operands.
2226 for (unsigned i = 1; i < NumOps; ++i) {
2227 SDValue Op = N->getOperand(Num: i);
2228 SDValue OpLo = Op;
2229 SDValue OpHi = Op;
2230
2231 EVT InVT = Op.getValueType();
2232 if (InVT.isVector()) {
2233 // If the input also splits, handle it directly for a
2234 // compile time speedup. Otherwise split it by hand.
2235 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2236 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2237 else
2238 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2239 }
2240
2241 OpsLo[i] = OpLo;
2242 OpsHi[i] = OpHi;
2243 }
2244
2245 EVT LoValueVTs[] = {LoVT, MVT::Other};
2246 EVT HiValueVTs[] = {HiVT, MVT::Other};
2247 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2248 Flags: N->getFlags());
2249 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2250 Flags: N->getFlags());
2251
2252 // Build a factor node to remember that this Op is independent of the
2253 // other one.
2254 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2255 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2256
2257 // Legalize the chain result - switch anything that used the old chain to
2258 // use the new one.
2259 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2260}
2261
2262SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2263 SDValue Chain = N->getOperand(Num: 0);
2264 EVT VT = N->getValueType(ResNo: 0);
2265 unsigned NE = VT.getVectorNumElements();
2266 EVT EltVT = VT.getVectorElementType();
2267 SDLoc dl(N);
2268
2269 SmallVector<SDValue, 8> Scalars;
2270 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2271
2272 // If ResNE is 0, fully unroll the vector op.
2273 if (ResNE == 0)
2274 ResNE = NE;
2275 else if (NE > ResNE)
2276 NE = ResNE;
2277
2278 //The results of each unrolled operation, including the chain.
2279 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2280 SmallVector<SDValue, 8> Chains;
2281
2282 unsigned i;
2283 for (i = 0; i != NE; ++i) {
2284 Operands[0] = Chain;
2285 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2286 SDValue Operand = N->getOperand(Num: j);
2287 EVT OperandVT = Operand.getValueType();
2288 if (OperandVT.isVector()) {
2289 EVT OperandEltVT = OperandVT.getVectorElementType();
2290 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2291 } else {
2292 Operands[j] = Operand;
2293 }
2294 }
2295 SDValue Scalar =
2296 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2297
2298 //Add in the scalar as well as its chain value to the
2299 //result vectors.
2300 Scalars.push_back(Elt: Scalar);
2301 Chains.push_back(Elt: Scalar.getValue(R: 1));
2302 }
2303
2304 for (; i < ResNE; ++i)
2305 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2306
2307 // Build a new factor node to connect the chain back together.
2308 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2309 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2310
2311 // Create a new BUILD_VECTOR node
2312 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2313 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2314}
2315
2316void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2317 SDValue &Lo, SDValue &Hi) {
2318 SDLoc dl(N);
2319 EVT ResVT = N->getValueType(ResNo: 0);
2320 EVT OvVT = N->getValueType(ResNo: 1);
2321 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2322 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2323 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2324
2325 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2326 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2327 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2328 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2329 } else {
2330 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2331 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2332 }
2333
2334 unsigned Opcode = N->getOpcode();
2335 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2336 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2337 SDNode *LoNode =
2338 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2339 SDNode *HiNode =
2340 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2341
2342 Lo = SDValue(LoNode, ResNo);
2343 Hi = SDValue(HiNode, ResNo);
2344
2345 // Replace the other vector result not being explicitly split here.
2346 unsigned OtherNo = 1 - ResNo;
2347 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2348 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2349 SetSplitVector(Op: SDValue(N, OtherNo),
2350 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2351 } else {
2352 SDValue OtherVal = DAG.getNode(
2353 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2354 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2355 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2356 }
2357}
2358
2359void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2360 SDValue &Hi) {
2361 SDValue Vec = N->getOperand(Num: 0);
2362 SDValue Elt = N->getOperand(Num: 1);
2363 SDValue Idx = N->getOperand(Num: 2);
2364 SDLoc dl(N);
2365 GetSplitVector(Op: Vec, Lo, Hi);
2366
2367 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2368 unsigned IdxVal = CIdx->getZExtValue();
2369 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2370 if (IdxVal < LoNumElts) {
2371 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2372 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2373 return;
2374 } else if (!Vec.getValueType().isScalableVector()) {
2375 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2376 return;
2377 }
2378 }
2379
2380 // Make the vector elements byte-addressable if they aren't already.
2381 EVT VecVT = Vec.getValueType();
2382 EVT EltVT = VecVT.getVectorElementType();
2383 if (!EltVT.isByteSized()) {
2384 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2385 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2386 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2387 // Extend the element type to match if needed.
2388 if (EltVT.bitsGT(VT: Elt.getValueType()))
2389 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2390 }
2391
2392 // Spill the vector to the stack.
2393 // In cases where the vector is illegal it will be broken down into parts
2394 // and stored in parts - we should use the alignment for the smallest part.
2395 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2396 SDValue StackPtr =
2397 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2398 auto &MF = DAG.getMachineFunction();
2399 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2400 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2401
2402 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2403 Alignment: SmallestAlign);
2404
2405 // Store the new element. This may be larger than the vector element type,
2406 // so use a truncating store.
2407 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2408 Store = DAG.getTruncStore(
2409 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2410 Alignment: commonAlignment(A: SmallestAlign,
2411 Offset: EltVT.getFixedSizeInBits() / 8));
2412
2413 EVT LoVT, HiVT;
2414 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2415
2416 // Load the Lo part from the stack slot.
2417 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2418
2419 // Increment the pointer to the other part.
2420 auto Load = cast<LoadSDNode>(Val&: Lo);
2421 MachinePointerInfo MPI = Load->getPointerInfo();
2422 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2423
2424 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2425
2426 // If we adjusted the original type, we need to truncate the results.
2427 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2428 if (LoVT != Lo.getValueType())
2429 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2430 if (HiVT != Hi.getValueType())
2431 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2432}
2433
2434void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2435 SDValue &Hi) {
2436 EVT LoVT, HiVT;
2437 SDLoc dl(N);
2438 assert(N->getValueType(0).isScalableVector() &&
2439 "Only scalable vectors are supported for STEP_VECTOR");
2440 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2441 SDValue Step = N->getOperand(Num: 0);
2442
2443 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2444
2445 // Hi = Lo + (EltCnt * Step)
2446 EVT EltVT = Step.getValueType();
2447 APInt StepVal = Step->getAsAPIntVal();
2448 SDValue StartOfHi =
2449 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2450 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2451 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2452
2453 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2454 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2455}
2456
2457void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2458 SDValue &Hi) {
2459 EVT LoVT, HiVT;
2460 SDLoc dl(N);
2461 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2462 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2463 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2464 Hi = DAG.getPOISON(VT: HiVT);
2465 } else {
2466 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2467 Hi = Lo;
2468 }
2469}
2470
2471void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2472 SDValue &Hi) {
2473 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2474 "Extended load during type legalization!");
2475 SDLoc dl(LD);
2476 EVT VT = LD->getValueType(ResNo: 0);
2477 EVT LoVT, HiVT;
2478 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2479
2480 SDValue Ch = LD->getChain();
2481 SDValue Ptr = LD->getBasePtr();
2482
2483 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2484 EVT MemIntVT =
2485 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LD->getMemoryVT().getSizeInBits());
2486 SDValue ALD = DAG.getAtomicLoad(ExtType: LD->getExtensionType(), dl, MemVT: MemIntVT, VT: IntVT,
2487 Chain: Ch, Ptr, MMO: LD->getMemOperand());
2488
2489 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2490 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2491 SDValue ExtractLo, ExtractHi;
2492 SplitInteger(Op: ALD, LoVT: LoIntVT, HiVT: HiIntVT, Lo&: ExtractLo, Hi&: ExtractHi);
2493
2494 Lo = DAG.getBitcast(VT: LoVT, V: ExtractLo);
2495 Hi = DAG.getBitcast(VT: HiVT, V: ExtractHi);
2496
2497 // Legalize the chain result - switch anything that used the old chain to
2498 // use the new one.
2499 ReplaceValueWith(From: SDValue(LD, 1), To: ALD.getValue(R: 1));
2500}
2501
2502void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2503 SDValue &Hi) {
2504 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2505 EVT LoVT, HiVT;
2506 SDLoc dl(LD);
2507 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2508
2509 ISD::LoadExtType ExtType = LD->getExtensionType();
2510 SDValue Ch = LD->getChain();
2511 SDValue Ptr = LD->getBasePtr();
2512 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
2513 EVT MemoryVT = LD->getMemoryVT();
2514 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2515 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
2516
2517 EVT LoMemVT, HiMemVT;
2518 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2519
2520 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2521 SDValue Value, NewChain;
2522 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2523 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2524 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2525 return;
2526 }
2527
2528 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2529 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2530 Metadata);
2531
2532 MachinePointerInfo MPI;
2533 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2534
2535 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2536 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
2537
2538 // Build a factor node to remember that this load is independent of the
2539 // other one.
2540 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2541 N2: Hi.getValue(R: 1));
2542
2543 // Legalize the chain result - switch anything that used the old chain to
2544 // use the new one.
2545 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2546}
2547
2548void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2549 SDValue &Hi) {
2550 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2551 EVT LoVT, HiVT;
2552 SDLoc dl(LD);
2553 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2554
2555 ISD::LoadExtType ExtType = LD->getExtensionType();
2556 SDValue Ch = LD->getChain();
2557 SDValue Ptr = LD->getBasePtr();
2558 SDValue Offset = LD->getOffset();
2559 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2560 Align Alignment = LD->getBaseAlign();
2561 SDValue Mask = LD->getMask();
2562 SDValue EVL = LD->getVectorLength();
2563 EVT MemoryVT = LD->getMemoryVT();
2564
2565 EVT LoMemVT, HiMemVT;
2566 bool HiIsEmpty = false;
2567 std::tie(args&: LoMemVT, args&: HiMemVT) =
2568 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2569
2570 // Split Mask operand
2571 SDValue MaskLo, MaskHi;
2572 if (Mask.getOpcode() == ISD::SETCC) {
2573 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2574 } else {
2575 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2576 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2577 else
2578 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2579 }
2580
2581 // Split EVL operand
2582 SDValue EVLLo, EVLHi;
2583 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2584
2585 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2586 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2587 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2588 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2589
2590 Lo =
2591 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2592 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2593
2594 if (HiIsEmpty) {
2595 // The hi vp_load has zero storage size. We therefore simply set it to
2596 // the low vp_load and rely on subsequent removal from the chain.
2597 Hi = Lo;
2598 } else {
2599 // Generate hi vp_load.
2600 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2601 IsCompressedMemory: LD->isExpandingLoad());
2602
2603 MachinePointerInfo MPI;
2604 if (LoMemVT.isScalableVector())
2605 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2606 else
2607 MPI = LD->getPointerInfo().getWithOffset(
2608 O: LoMemVT.getStoreSize().getFixedValue());
2609
2610 MMO = DAG.getMachineFunction().getMachineMemOperand(
2611 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2612 BaseAlignment: Alignment, Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2613
2614 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2615 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2616 IsExpanding: LD->isExpandingLoad());
2617 }
2618
2619 // Build a factor node to remember that this load is independent of the
2620 // other one.
2621 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2622 N2: Hi.getValue(R: 1));
2623
2624 // Legalize the chain result - switch anything that used the old chain to
2625 // use the new one.
2626 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2627}
2628
2629void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2630 SDValue &Hi) {
2631 SDLoc dl(LD);
2632 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2633
2634 SDValue Ch = LD->getChain();
2635 SDValue Ptr = LD->getBasePtr();
2636 Align Alignment = LD->getBaseAlign();
2637 SDValue Mask = LD->getMask();
2638 SDValue EVL = LD->getVectorLength();
2639
2640 // Split Mask operand
2641 SDValue MaskLo, MaskHi;
2642 if (Mask.getOpcode() == ISD::SETCC) {
2643 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2644 } else {
2645 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2646 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2647 else
2648 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2649 }
2650
2651 // Split EVL operand
2652 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2653
2654 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2655 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2656 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2657 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2658
2659 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2660
2661 // Fill the upper half with poison.
2662 Hi = DAG.getPOISON(VT: HiVT);
2663
2664 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2665 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2666}
2667
2668void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2669 SDValue &Lo, SDValue &Hi) {
2670 assert(SLD->isUnindexed() &&
2671 "Indexed VP strided load during type legalization!");
2672 assert(SLD->getOffset().isUndef() &&
2673 "Unexpected indexed variable-length load offset");
2674
2675 SDLoc DL(SLD);
2676
2677 EVT LoVT, HiVT;
2678 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2679
2680 EVT LoMemVT, HiMemVT;
2681 bool HiIsEmpty = false;
2682 std::tie(args&: LoMemVT, args&: HiMemVT) =
2683 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2684
2685 SDValue Mask = SLD->getMask();
2686 SDValue LoMask, HiMask;
2687 if (Mask.getOpcode() == ISD::SETCC) {
2688 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2689 } else {
2690 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2691 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2692 else
2693 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2694 }
2695
2696 SDValue LoEVL, HiEVL;
2697 std::tie(args&: LoEVL, args&: HiEVL) =
2698 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2699
2700 // Generate the low vp_strided_load
2701 Lo = DAG.getStridedLoadVP(
2702 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2703 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2704 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2705
2706 if (HiIsEmpty) {
2707 // The high vp_strided_load has zero storage size. We therefore simply set
2708 // it to the low vp_strided_load and rely on subsequent removal from the
2709 // chain.
2710 Hi = Lo;
2711 } else {
2712 // Generate the high vp_strided_load.
2713 // To calculate the high base address, we need to sum to the low base
2714 // address stride number of bytes for each element already loaded by low,
2715 // that is: Ptr = Ptr + (LoEVL * Stride)
2716 EVT PtrVT = SLD->getBasePtr().getValueType();
2717 SDValue Increment =
2718 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2719 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2720 SDValue Ptr =
2721 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2722
2723 Align Alignment = SLD->getBaseAlign();
2724 if (LoMemVT.isScalableVector())
2725 Alignment = commonAlignment(
2726 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2727
2728 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2729 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2730 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2731 BaseAlignment: Alignment, Metadata: MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
2732
2733 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2734 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2735 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2736 IsExpanding: SLD->isExpandingLoad());
2737 }
2738
2739 // Build a factor node to remember that this load is independent of the
2740 // other one.
2741 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2742 N2: Hi.getValue(R: 1));
2743
2744 // Legalize the chain result - switch anything that used the old chain to
2745 // use the new one.
2746 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2747}
2748
2749void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2750 SDValue &Lo, SDValue &Hi) {
2751 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2752 EVT LoVT, HiVT;
2753 SDLoc dl(MLD);
2754 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2755
2756 SDValue Ch = MLD->getChain();
2757 SDValue Ptr = MLD->getBasePtr();
2758 SDValue Offset = MLD->getOffset();
2759 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2760 SDValue Mask = MLD->getMask();
2761 SDValue PassThru = MLD->getPassThru();
2762 Align Alignment = MLD->getBaseAlign();
2763 ISD::LoadExtType ExtType = MLD->getExtensionType();
2764 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2765
2766 // Split Mask operand
2767 SDValue MaskLo, MaskHi;
2768 if (Mask.getOpcode() == ISD::SETCC) {
2769 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2770 } else {
2771 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2772 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2773 else
2774 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2775 }
2776
2777 EVT MemoryVT = MLD->getMemoryVT();
2778 EVT LoMemVT, HiMemVT;
2779 bool HiIsEmpty = false;
2780 std::tie(args&: LoMemVT, args&: HiMemVT) =
2781 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2782
2783 SDValue PassThruLo, PassThruHi;
2784 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2785 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2786 else
2787 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2788
2789 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2790 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2791 BaseAlignment: Alignment,
2792 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(), MLD->getMemCacheHint()));
2793
2794 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2795 MMO, AM: MLD->getAddressingMode(), ExtType,
2796 IsExpanding: MLD->isExpandingLoad());
2797
2798 if (HiIsEmpty) {
2799 // The hi masked load has zero storage size. We therefore simply set it to
2800 // the low masked load and rely on subsequent removal from the chain.
2801 Hi = Lo;
2802 } else {
2803 // Generate hi masked load.
2804 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2805 IsCompressedMemory: MLD->isExpandingLoad());
2806
2807 MachinePointerInfo MPI;
2808 if (LoMemVT.isScalableVector())
2809 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2810 else
2811 MPI = MLD->getPointerInfo().getWithOffset(
2812 O: LoMemVT.getStoreSize().getFixedValue());
2813
2814 MMO = DAG.getMachineFunction().getMachineMemOperand(
2815 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2816 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges(),
2817 MLD->getMemCacheHint()));
2818
2819 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2820 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2821 IsExpanding: MLD->isExpandingLoad());
2822 }
2823
2824 // Build a factor node to remember that this load is independent of the
2825 // other one.
2826 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2827 N2: Hi.getValue(R: 1));
2828
2829 // Legalize the chain result - switch anything that used the old chain to
2830 // use the new one.
2831 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2832
2833}
2834
2835void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2836 SDValue &Hi, bool SplitSETCC) {
2837 EVT LoVT, HiVT;
2838 SDLoc dl(N);
2839 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2840
2841 SDValue Ch = N->getChain();
2842 SDValue Ptr = N->getBasePtr();
2843 struct Operands {
2844 SDValue Mask;
2845 SDValue Index;
2846 SDValue Scale;
2847 } Ops = [&]() -> Operands {
2848 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2849 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2850 }
2851 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2852 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2853 }();
2854
2855 EVT MemoryVT = N->getMemoryVT();
2856 Align Alignment = N->getBaseAlign();
2857
2858 // Split Mask operand
2859 SDValue MaskLo, MaskHi;
2860 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2861 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2862 } else {
2863 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2864 }
2865
2866 EVT LoMemVT, HiMemVT;
2867 // Split MemoryVT
2868 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2869
2870 SDValue IndexHi, IndexLo;
2871 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2872 TargetLowering::TypeSplitVector)
2873 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2874 else
2875 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2876
2877 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2878 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2879 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2880 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
2881
2882 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2883 SDValue PassThru = MGT->getPassThru();
2884 SDValue PassThruLo, PassThruHi;
2885 if (getTypeAction(VT: PassThru.getValueType()) ==
2886 TargetLowering::TypeSplitVector)
2887 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2888 else
2889 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2890
2891 ISD::LoadExtType ExtType = MGT->getExtensionType();
2892 ISD::MemIndexType IndexTy = MGT->getIndexType();
2893
2894 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2895 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2896 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2897
2898 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2899 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2900 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2901 } else {
2902 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2903 SDValue EVLLo, EVLHi;
2904 std::tie(args&: EVLLo, args&: EVLHi) =
2905 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2906
2907 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2908 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2909 MMO, IndexType: VPGT->getIndexType());
2910
2911 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2912 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2913 MMO, IndexType: VPGT->getIndexType());
2914 }
2915
2916 // Build a factor node to remember that this load is independent of the
2917 // other one.
2918 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2919 N2: Hi.getValue(R: 1));
2920
2921 // Legalize the chain result - switch anything that used the old chain to
2922 // use the new one.
2923 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2924}
2925
2926void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2927 SDValue &Hi) {
2928 // This is not "trivial", as there is a dependency between the two subvectors.
2929 // Depending on the number of 1s in the mask, the elements from the Hi vector
2930 // need to be moved to the Lo vector. Passthru values make this even harder.
2931 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2932 // smaller types and passthru is undef, as it is most likely faster than the
2933 // fully expand path. Otherwise, just do the full expansion as one "big"
2934 // operation and then extract the Lo and Hi vectors from that. This gets
2935 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2936 SDLoc DL(N);
2937 EVT VecVT = N->getValueType(ResNo: 0);
2938
2939 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2940 bool HasCustomLowering = false;
2941 EVT CheckVT = LoVT;
2942 while (CheckVT.getVectorMinNumElements() > 1) {
2943 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2944 // custom lowering for illegal types. So we do the checks separately.
2945 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2946 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2947 HasCustomLowering = true;
2948 break;
2949 }
2950 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2951 }
2952
2953 SDValue Passthru = N->getOperand(Num: 2);
2954 if (!HasCustomLowering) {
2955 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2956 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2957 return;
2958 }
2959
2960 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2961 SDValue Mask = N->getOperand(Num: 1);
2962 SDValue LoMask, HiMask;
2963 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2964 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2965
2966 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2967 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2968 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2969
2970 SDValue StackPtr = DAG.CreateStackTemporary(
2971 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2972 MachineFunction &MF = DAG.getMachineFunction();
2973 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2974 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2975
2976 EVT LoMaskVT = LoMask.getValueType();
2977 assert(LoMaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2978
2979 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2980 EVT WideLoMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2981 EC: LoMaskVT.getVectorElementCount());
2982 SDValue WideLoMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideLoMaskVT, Operand: LoMask);
2983 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideLoMask);
2984 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2985
2986 SDValue Chain = DAG.getEntryNode();
2987 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2988 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2989 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2990
2991 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2992 if (!Passthru.isUndef()) {
2993 // Compress the input mask so only inactive lanes of the result are replaced
2994 // by their passthrough value.
2995 EVT MaskVT = Mask.getValueType();
2996 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2997 EC: MaskVT.getVectorElementCount());
2998 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: Mask);
2999 SDValue NumActiveElts =
3000 DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
3001
3002 SDValue StepVector = DAG.getStepVector(DL, ResVT: WideMaskVT);
3003 SDValue SplatNumActiveElts = DAG.getSplat(VT: WideMaskVT, DL, Op: NumActiveElts);
3004 SDValue CompressedMask =
3005 DAG.getSetCC(DL, VT: MaskVT, LHS: StepVector, RHS: SplatNumActiveElts, Cond: ISD::SETULT);
3006
3007 Compressed = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: CompressedMask,
3008 N2: Compressed, N3: Passthru);
3009 }
3010 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
3011}
3012
3013void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
3014 assert(N->getValueType(0).isVector() &&
3015 N->getOperand(0).getValueType().isVector() &&
3016 "Operand types must be vectors");
3017
3018 EVT LoVT, HiVT;
3019 SDLoc DL(N);
3020 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3021
3022 // If the input also splits, handle it directly. Otherwise split it by hand.
3023 SDValue LL, LH, RL, RH;
3024 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
3025 TargetLowering::TypeSplitVector)
3026 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
3027 else
3028 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
3029
3030 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
3031 TargetLowering::TypeSplitVector)
3032 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
3033 else
3034 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
3035
3036 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
3037 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
3038}
3039
3040void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
3041 SDValue &Hi) {
3042 // Get the dest types - they may not match the input types, e.g. int_to_fp.
3043 EVT LoVT, HiVT;
3044 SDLoc dl(N);
3045 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3046
3047 // If the input also splits, handle it directly for a compile time speedup.
3048 // Otherwise split it by hand.
3049 EVT InVT = N->getOperand(Num: 0).getValueType();
3050 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3051 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3052 else
3053 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3054
3055 const SDNodeFlags Flags = N->getFlags();
3056 unsigned Opcode = N->getOpcode();
3057 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
3058 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
3059 N4: N->getOperand(Num: 3), Flags);
3060 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
3061 N4: N->getOperand(Num: 3), Flags);
3062 return;
3063 }
3064
3065 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
3066 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
3067 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
3068 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
3069 } else {
3070 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
3071 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
3072 }
3073}
3074
3075void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
3076 SDValue &Hi) {
3077 SDLoc dl(N);
3078 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3079
3080 // If the input also splits, handle it directly for a compile time speedup.
3081 // Otherwise split it by hand.
3082 EVT InVT = N->getOperand(Num: 0).getValueType();
3083 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3084 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3085 else
3086 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3087
3088 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
3089 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
3090 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
3091 SDNodeFlags Flags = AddrSpaceCastN->getFlags();
3092 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS, Flags);
3093 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS, Flags);
3094}
3095
3096void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
3097 unsigned ResNo,
3098 SDValue &Lo,
3099 SDValue &Hi) {
3100 SDLoc dl(N);
3101 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3102 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
3103
3104 // If the input also splits, handle it directly for a compile time speedup.
3105 // Otherwise split it by hand.
3106 EVT InVT = N->getOperand(Num: 0).getValueType();
3107 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3108 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3109 else
3110 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3111
3112 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
3113 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
3114
3115 SDNode *HiNode = Hi.getNode();
3116 SDNode *LoNode = Lo.getNode();
3117
3118 // Replace the other vector result not being explicitly split here.
3119 unsigned OtherNo = 1 - ResNo;
3120 EVT OtherVT = N->getValueType(ResNo: OtherNo);
3121 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
3122 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
3123 Hi: SDValue(HiNode, OtherNo));
3124 } else {
3125 SDValue OtherVal =
3126 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
3127 N2: SDValue(HiNode, OtherNo));
3128 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
3129 }
3130}
3131
3132void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3133 SDValue &Hi) {
3134 SDLoc dl(N);
3135 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3136 EVT DestVT = N->getValueType(ResNo: 0);
3137 EVT LoVT, HiVT;
3138 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
3139
3140 // We can do better than a generic split operation if the extend is doing
3141 // more than just doubling the width of the elements and the following are
3142 // true:
3143 // - The number of vector elements is even,
3144 // - the source type is legal,
3145 // - the type of a split source is illegal,
3146 // - the type of an extended (by doubling element size) source is legal, and
3147 // - the type of that extended source when split is legal.
3148 //
3149 // This won't necessarily completely legalize the operation, but it will
3150 // more effectively move in the right direction and prevent falling down
3151 // to scalarization in many cases due to the input vector being split too
3152 // far.
3153 if (SrcVT.getVectorElementCount().isKnownEven() &&
3154 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3155 LLVMContext &Ctx = *DAG.getContext();
3156 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
3157 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
3158
3159 EVT SplitLoVT, SplitHiVT;
3160 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
3161 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
3162 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
3163 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3164 N->dump(&DAG); dbgs() << "\n");
3165 // Extend the source vector by one step.
3166 SDValue NewSrc =
3167 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
3168 // Get the low and high halves of the new, extended one step, vector.
3169 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3170 // Extend those vector halves the rest of the way.
3171 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
3172 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
3173 return;
3174 }
3175 }
3176 // Fall back to the generic unary operator splitting otherwise.
3177 SplitVecRes_UnaryOp(N, Lo, Hi);
3178}
3179
3180void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3181 SDValue &Lo, SDValue &Hi) {
3182 // The low and high parts of the original input give four input vectors.
3183 SDValue Inputs[4];
3184 SDLoc DL(N);
3185 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
3186 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
3187 EVT NewVT = Inputs[0].getValueType();
3188 unsigned NewElts = NewVT.getVectorNumElements();
3189
3190 auto &&IsConstant = [](const SDValue &N) {
3191 APInt SplatValue;
3192 return N.getResNo() == 0 &&
3193 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
3194 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
3195 };
3196 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3197 SDValue &Input2,
3198 ArrayRef<int> Mask) {
3199 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3200 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3201 "Expected build vector node.");
3202 EVT EltVT = NewVT.getVectorElementType();
3203 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
3204 for (unsigned I = 0; I < NewElts; ++I) {
3205 if (Mask[I] == PoisonMaskElem)
3206 continue;
3207 unsigned Idx = Mask[I];
3208 if (Idx >= NewElts)
3209 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3210 else
3211 Ops[I] = Input1.getOperand(i: Idx);
3212 // Make the type of all elements the same as the element type.
3213 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3214 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3215 }
3216 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3217 };
3218
3219 // If Lo or Hi uses elements from at most two of the four input vectors, then
3220 // express it as a vector shuffle of those two inputs. Otherwise extract the
3221 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3222 SmallVector<int> OrigMask(N->getMask());
3223 // Try to pack incoming shuffles/inputs.
3224 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3225 &DL](SmallVectorImpl<int> &Mask) {
3226 // Check if all inputs are shuffles of the same operands or non-shuffles.
3227 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3228 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3229 SDValue Input = Inputs[Idx];
3230 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3231 if (!Shuffle ||
3232 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3233 continue;
3234 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3235 .push_back(Elt: Idx);
3236 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3237 .push_back(Elt: Idx);
3238 }
3239 for (auto &P : ShufflesIdxs) {
3240 if (P.second.size() < 2)
3241 continue;
3242 // Use shuffles operands instead of shuffles themselves.
3243 // 1. Adjust mask.
3244 for (int &Idx : Mask) {
3245 if (Idx == PoisonMaskElem)
3246 continue;
3247 unsigned SrcRegIdx = Idx / NewElts;
3248 if (Inputs[SrcRegIdx].isUndef()) {
3249 Idx = PoisonMaskElem;
3250 continue;
3251 }
3252 auto *Shuffle =
3253 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3254 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3255 continue;
3256 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3257 if (MaskElt == PoisonMaskElem) {
3258 Idx = PoisonMaskElem;
3259 continue;
3260 }
3261 Idx = MaskElt % NewElts +
3262 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3263 ? 0
3264 : 1] *
3265 NewElts;
3266 }
3267 // 2. Update inputs.
3268 Inputs[P.second[0]] = P.first.first;
3269 Inputs[P.second[1]] = P.first.second;
3270 // Clear the pair data.
3271 P.second.clear();
3272 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3273 }
3274 // Check if any concat_vectors can be simplified.
3275 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3276 for (int &Idx : Mask) {
3277 if (Idx == PoisonMaskElem)
3278 continue;
3279 unsigned SrcRegIdx = Idx / NewElts;
3280 if (Inputs[SrcRegIdx].isUndef()) {
3281 Idx = PoisonMaskElem;
3282 continue;
3283 }
3284 TargetLowering::LegalizeTypeAction TypeAction =
3285 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3286 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3287 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3288 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3289 (TypeAction == TargetLowering::TypeLegal ||
3290 TypeAction == TargetLowering::TypeWidenVector))
3291 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3292 }
3293 if (UsedSubVector.count() > 1) {
3294 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3295 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3296 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3297 continue;
3298 if (Pairs.empty() || Pairs.back().size() == 2)
3299 Pairs.emplace_back();
3300 if (UsedSubVector.test(Idx: 2 * I)) {
3301 Pairs.back().emplace_back(Args&: I, Args: 0);
3302 } else {
3303 assert(UsedSubVector.test(2 * I + 1) &&
3304 "Expected to be used one of the subvectors.");
3305 Pairs.back().emplace_back(Args&: I, Args: 1);
3306 }
3307 }
3308 if (!Pairs.empty() && Pairs.front().size() > 1) {
3309 // Adjust mask.
3310 for (int &Idx : Mask) {
3311 if (Idx == PoisonMaskElem)
3312 continue;
3313 unsigned SrcRegIdx = Idx / NewElts;
3314 auto *It = find_if(
3315 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3316 return Idxs.front().first == SrcRegIdx ||
3317 Idxs.back().first == SrcRegIdx;
3318 });
3319 if (It == Pairs.end())
3320 continue;
3321 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3322 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3323 }
3324 // Adjust inputs.
3325 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3326 Inputs[Idxs.front().first] = DAG.getNode(
3327 Opcode: ISD::CONCAT_VECTORS, DL,
3328 VT: Inputs[Idxs.front().first].getValueType(),
3329 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3330 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3331 }
3332 }
3333 }
3334 bool Changed;
3335 do {
3336 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3337 // reused operands.
3338 Changed = false;
3339 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3340 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3341 if (!Shuffle)
3342 continue;
3343 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3344 continue;
3345 int Op = -1;
3346 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3347 !Shuffle->isSplat()) {
3348 Op = 0;
3349 } else if (!Inputs[I].hasOneUse() &&
3350 !Shuffle->getOperand(Num: 1).isUndef()) {
3351 // Find the only used operand, if possible.
3352 for (int &Idx : Mask) {
3353 if (Idx == PoisonMaskElem)
3354 continue;
3355 unsigned SrcRegIdx = Idx / NewElts;
3356 if (SrcRegIdx != I)
3357 continue;
3358 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3359 if (MaskElt == PoisonMaskElem) {
3360 Idx = PoisonMaskElem;
3361 continue;
3362 }
3363 int OpIdx = MaskElt / NewElts;
3364 if (Op == -1) {
3365 Op = OpIdx;
3366 continue;
3367 }
3368 if (Op != OpIdx) {
3369 Op = -1;
3370 break;
3371 }
3372 }
3373 }
3374 if (Op < 0) {
3375 // Try to check if one of the shuffle operands is used already.
3376 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3377 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3378 continue;
3379 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3380 if (It == std::end(arr&: Inputs))
3381 continue;
3382 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3383 // Found that operand is used already.
3384 // 1. Fix the mask for the reused operand.
3385 for (int &Idx : Mask) {
3386 if (Idx == PoisonMaskElem)
3387 continue;
3388 unsigned SrcRegIdx = Idx / NewElts;
3389 if (SrcRegIdx != I)
3390 continue;
3391 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3392 if (MaskElt == PoisonMaskElem) {
3393 Idx = PoisonMaskElem;
3394 continue;
3395 }
3396 int MaskIdx = MaskElt / NewElts;
3397 if (OpIdx == MaskIdx)
3398 Idx = MaskElt % NewElts + FoundOp * NewElts;
3399 }
3400 // 2. Set Op to the unused OpIdx.
3401 Op = (OpIdx + 1) % 2;
3402 break;
3403 }
3404 }
3405 if (Op >= 0) {
3406 Changed = true;
3407 Inputs[I] = Shuffle->getOperand(Num: Op);
3408 // Adjust mask.
3409 for (int &Idx : Mask) {
3410 if (Idx == PoisonMaskElem)
3411 continue;
3412 unsigned SrcRegIdx = Idx / NewElts;
3413 if (SrcRegIdx != I)
3414 continue;
3415 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3416 int OpIdx = MaskElt / NewElts;
3417 if (OpIdx != Op)
3418 continue;
3419 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3420 }
3421 }
3422 }
3423 } while (Changed);
3424 };
3425 TryPeekThroughShufflesInputs(OrigMask);
3426 // Proces unique inputs.
3427 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3428 NewElts](SmallVectorImpl<int> &Mask) {
3429 SetVector<SDValue> UniqueInputs;
3430 SetVector<SDValue> UniqueConstantInputs;
3431 for (const auto &I : Inputs) {
3432 if (IsConstant(I))
3433 UniqueConstantInputs.insert(X: I);
3434 else if (!I.isUndef())
3435 UniqueInputs.insert(X: I);
3436 }
3437 // Adjust mask in case of reused inputs. Also, need to insert constant
3438 // inputs at first, otherwise it affects the final outcome.
3439 if (UniqueInputs.size() != std::size(Inputs)) {
3440 auto &&UniqueVec = UniqueInputs.takeVector();
3441 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3442 unsigned ConstNum = UniqueConstantVec.size();
3443 for (int &Idx : Mask) {
3444 if (Idx == PoisonMaskElem)
3445 continue;
3446 unsigned SrcRegIdx = Idx / NewElts;
3447 if (Inputs[SrcRegIdx].isUndef()) {
3448 Idx = PoisonMaskElem;
3449 continue;
3450 }
3451 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3452 if (It != UniqueConstantVec.end()) {
3453 Idx = (Idx % NewElts) +
3454 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3455 assert(Idx >= 0 && "Expected defined mask idx.");
3456 continue;
3457 }
3458 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3459 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3460 Idx = (Idx % NewElts) +
3461 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3462 assert(Idx >= 0 && "Expected defined mask idx.");
3463 }
3464 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3465 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3466 }
3467 };
3468 MakeUniqueInputs(OrigMask);
3469 SDValue OrigInputs[4];
3470 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3471 for (unsigned High = 0; High < 2; ++High) {
3472 SDValue &Output = High ? Hi : Lo;
3473
3474 // Build a shuffle mask for the output, discovering on the fly which
3475 // input vectors to use as shuffle operands.
3476 unsigned FirstMaskIdx = High * NewElts;
3477 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3478 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3479 assert(!Output && "Expected default initialized initial value.");
3480 TryPeekThroughShufflesInputs(Mask);
3481 MakeUniqueInputs(Mask);
3482 SDValue TmpInputs[4];
3483 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3484 // Track changes in the output registers.
3485 int UsedIdx = -1;
3486 bool SecondIteration = false;
3487 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3488 if (UsedIdx < 0) {
3489 UsedIdx = Idx;
3490 return false;
3491 }
3492 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3493 SecondIteration = true;
3494 return SecondIteration;
3495 };
3496 processShuffleMasks(
3497 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3498 /*NumOfUsedRegs=*/1,
3499 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3500 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3501 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3502 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3503 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3504 else
3505 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3506 N2: DAG.getPOISON(VT: NewVT), Mask);
3507 Inputs[Idx] = Output;
3508 },
3509 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3510 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3511 unsigned Idx2, bool /*Unused*/) {
3512 if (AccumulateResults(Idx1)) {
3513 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3514 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3515 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3516 else
3517 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3518 N2: Inputs[Idx2], Mask);
3519 } else {
3520 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3521 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3522 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3523 else
3524 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3525 N2: TmpInputs[Idx2], Mask);
3526 }
3527 Inputs[Idx1] = Output;
3528 });
3529 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3530 }
3531}
3532
3533void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3534 EVT OVT = N->getValueType(ResNo: 0);
3535 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3536 SDValue Chain = N->getOperand(Num: 0);
3537 SDValue Ptr = N->getOperand(Num: 1);
3538 SDValue SV = N->getOperand(Num: 2);
3539 SDLoc dl(N);
3540
3541 const Align Alignment =
3542 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3543
3544 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3545 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3546 Chain = Hi.getValue(R: 1);
3547
3548 // Modified the chain - switch anything that used the old chain to use
3549 // the new one.
3550 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3551}
3552
3553void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3554 SDValue &Hi) {
3555 EVT DstVTLo, DstVTHi;
3556 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3557 SDLoc dl(N);
3558
3559 SDValue SrcLo, SrcHi;
3560 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3561 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3562 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3563 else
3564 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3565
3566 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3567 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3568}
3569
3570void DAGTypeLegalizer::SplitVecRes_VECTOR_REPEAT(SDNode *N, SDValue &Lo,
3571 SDValue &Hi) {
3572 EVT VT = N->getValueType(ResNo: 0);
3573 SDValue Src = N->getOperand(Num: 0);
3574 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
3575 assert(LoVT == HiVT && "Expected equal split types");
3576
3577 // Use smaller even/odd source vectors so their broadcasts can be
3578 // reinterleaved in the original lane order for every value of vscale.
3579 SDLoc DL(N);
3580 auto [SrcLo, SrcHi] = DAG.SplitVector(N: Src, DL);
3581 EVT SplitSrcVT = SrcLo.getValueType();
3582 SDValue Deinterleaved =
3583 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL,
3584 VTList: DAG.getVTList(VT1: SplitSrcVT, VT2: SplitSrcVT), N1: SrcLo, N2: SrcHi);
3585 SDValue Even =
3586 DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: LoVT, Operand: Deinterleaved.getValue(R: 0));
3587 SDValue Odd =
3588 DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: LoVT, Operand: Deinterleaved.getValue(R: 1));
3589 SDValue Interleaved = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
3590 VTList: DAG.getVTList(VT1: LoVT, VT2: LoVT), N1: Even, N2: Odd);
3591 Lo = Interleaved.getValue(R: 0);
3592 Hi = Interleaved.getValue(R: 1);
3593}
3594
3595void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3596 SDValue &Hi) {
3597 SDValue InLo, InHi;
3598 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3599 SDLoc DL(N);
3600
3601 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3602 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3603}
3604
3605void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3606 SDValue &Hi) {
3607 SDLoc DL(N);
3608
3609 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3610 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3611}
3612
3613void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3614 SDValue &Hi) {
3615 EVT VT = N->getValueType(ResNo: 0);
3616 SDValue Val = N->getOperand(Num: 0);
3617 SDValue Mask = N->getOperand(Num: 1);
3618 SDValue EVL = N->getOperand(Num: 2);
3619 SDLoc DL(N);
3620
3621 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3622 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3623 // then truncate back.
3624 EVT OrigVT = VT;
3625 if (!VT.getVectorElementType().isByteSized()) {
3626 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3627 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3628 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3629 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Val);
3630 }
3631
3632 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3633 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3634
3635 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3636 EC: VT.getVectorElementCount());
3637 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3638 EVT PtrVT = StackPtr.getValueType();
3639 auto &MF = DAG.getMachineFunction();
3640 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3641 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3642
3643 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3644 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3645 BaseAlignment: Alignment);
3646 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3647 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3648 BaseAlignment: Alignment);
3649
3650 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3651 SDValue NumElemMinus1 =
3652 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3653 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3654 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3655 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3656 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3657 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3658
3659 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3660 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3661 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3662 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3663
3664 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3665
3666 // Truncate back if we widened above.
3667 if (OrigVT != VT)
3668 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3669
3670 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3671}
3672
3673void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3674 SDValue &Hi) {
3675 EVT VT = N->getValueType(ResNo: 0);
3676 SDValue V1 = N->getOperand(Num: 0);
3677 SDValue V2 = N->getOperand(Num: 1);
3678 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3679 SDValue Mask = N->getOperand(Num: 3);
3680 SDValue EVL1 = N->getOperand(Num: 4);
3681 SDValue EVL2 = N->getOperand(Num: 5);
3682 SDLoc DL(N);
3683
3684 // Since EVL2 is considered the real VL it gets promoted during
3685 // SelectionDAGBuilder. Promote EVL1 here if needed.
3686 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3687 EVL1 = ZExtPromotedInteger(Op: EVL1);
3688
3689 // The stack splice addresses elements by byte offset/stride, which breaks for
3690 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3691 // stride is 0. Widen to a byte integer, splice, then truncate back.
3692 EVT OrigVT = VT;
3693 if (!VT.getVectorElementType().isByteSized()) {
3694 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3695 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3696 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3697 V1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V1);
3698 V2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V2);
3699 }
3700
3701 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3702
3703 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3704 EC: VT.getVectorElementCount() * 2);
3705 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3706 EVT PtrVT = StackPtr.getValueType();
3707 auto &MF = DAG.getMachineFunction();
3708 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3709 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3710
3711 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3712 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3713 BaseAlignment: Alignment);
3714 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3715 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3716 BaseAlignment: Alignment);
3717
3718 SDValue EltByteSize =
3719 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
3720 SDValue EVL1Ptr = DAG.getZExtOrTrunc(Op: EVL1, DL, VT: PtrVT);
3721 SDValue EVL1Bytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: EVL1Ptr, N2: EltByteSize);
3722 // Clip EVL1Bytes to make sure we stay within the stack object.
3723 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
3724 EVL1Bytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: EVL1Bytes, N2: VTBytes);
3725 SDValue StackPtr2 = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: EVL1Bytes, DL);
3726 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3727
3728 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3729 SDValue StoreV1 =
3730 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3731 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3732
3733 SDValue StoreV2 =
3734 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3735 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3736
3737 SDValue Load;
3738 if (Imm >= 0) {
3739 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3740 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3741 } else {
3742 uint64_t TrailingElts = -Imm;
3743 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3744 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3745
3746 // Make sure TrailingBytes doesn't exceed the size of vec1.
3747 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3748 TrailingBytes =
3749 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3750
3751 // Calculate the start address of the spliced result.
3752 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3753 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3754 }
3755
3756 // Truncate back if we widened above.
3757 if (OrigVT != VT)
3758 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3759
3760 EVT LoVT, HiVT;
3761 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: OrigVT);
3762 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3763 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3764 Hi =
3765 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3766 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3767}
3768
3769void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3770 SDValue &Hi) {
3771 SDLoc DL(N);
3772 SDValue Acc = N->getOperand(Num: 0);
3773 SDValue Input1 = N->getOperand(Num: 1);
3774 SDValue Input2 = N->getOperand(Num: 2);
3775
3776 SDValue AccLo, AccHi;
3777 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3778 unsigned Opcode = N->getOpcode();
3779
3780 // If the input types don't need splitting, just accumulate into the
3781 // low part of the accumulator.
3782 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3783 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3784 Hi = AccHi;
3785 return;
3786 }
3787
3788 SDValue Input1Lo, Input1Hi;
3789 SDValue Input2Lo, Input2Hi;
3790 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3791 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3792 EVT ResultVT = AccLo.getValueType();
3793
3794 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3795 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3796}
3797
3798void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3799 SDValue &Hi) {
3800 SDLoc DL(N);
3801 SDValue Op0 = N->getOperand(Num: 0);
3802 SDValue Op1 = N->getOperand(Num: 1);
3803 EVT OpVT = Op0.getValueType();
3804
3805 EVT LoVT, HiVT;
3806 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3807
3808 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3809 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3810 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3811 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3812}
3813
3814void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
3815 SDValue &Hi) {
3816 SDValue SourceLo, SourceHi;
3817 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SourceLo, Hi&: SourceHi);
3818 SDValue MaskLo, MaskHi;
3819 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
3820 SDLoc DL(N);
3821
3822 Lo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskLo.getValueType(), N1: SourceLo,
3823 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
3824 Hi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskHi.getValueType(), N1: SourceHi,
3825 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
3826}
3827
3828void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3829 unsigned Factor = N->getNumOperands();
3830
3831 SmallVector<SDValue, 8> Ops(Factor * 2);
3832 for (unsigned i = 0; i != Factor; ++i) {
3833 SDValue OpLo, OpHi;
3834 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3835 Ops[i * 2] = OpLo;
3836 Ops[i * 2 + 1] = OpHi;
3837 }
3838
3839 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3840
3841 SDLoc DL(N);
3842 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3843 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3844 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3845 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3846
3847 for (unsigned i = 0; i != Factor; ++i)
3848 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3849}
3850
3851void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3852 unsigned Factor = N->getNumOperands();
3853
3854 SmallVector<SDValue, 8> Ops(Factor * 2);
3855 for (unsigned i = 0; i != Factor; ++i) {
3856 SDValue OpLo, OpHi;
3857 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3858 Ops[i] = OpLo;
3859 Ops[i + Factor] = OpHi;
3860 }
3861
3862 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3863
3864 SDLoc DL(N);
3865 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3866 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3867 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3868 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3869
3870 for (unsigned i = 0; i != Factor; ++i) {
3871 unsigned IdxLo = 2 * i;
3872 unsigned IdxHi = 2 * i + 1;
3873 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3874 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3875 }
3876}
3877
3878//===----------------------------------------------------------------------===//
3879// Operand Vector Splitting
3880//===----------------------------------------------------------------------===//
3881
3882/// This method is called when the specified operand of the specified node is
3883/// found to need vector splitting. At this point, all of the result types of
3884/// the node are known to be legal, but other operands of the node may need
3885/// legalization as well as the specified one.
3886bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3887 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3888 SDValue Res = SDValue();
3889
3890 // See if the target wants to custom split this node.
3891 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3892 return false;
3893
3894 switch (N->getOpcode()) {
3895 default:
3896#ifndef NDEBUG
3897 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3898 N->dump(&DAG);
3899 dbgs() << "\n";
3900#endif
3901 report_fatal_error(reason: "Do not know how to split this operator's "
3902 "operand!\n");
3903
3904 case ISD::STRICT_FSETCC:
3905 case ISD::STRICT_FSETCCS:
3906 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3907 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3908 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3909 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3910 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3911 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3912 case ISD::VECTOR_FIND_LAST_ACTIVE:
3913 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3914 break;
3915 case ISD::TRUNCATE:
3916 Res = SplitVecOp_TruncateHelper(N);
3917 break;
3918 case ISD::STRICT_FP_ROUND:
3919 case ISD::FP_ROUND:
3920 case ISD::CONVERT_FROM_ARBITRARY_FP:
3921 case ISD::CONVERT_TO_ARBITRARY_FP:
3922 Res = SplitVecOp_FP_ROUND(N);
3923 break;
3924 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3925 case ISD::STORE:
3926 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3927 break;
3928 case ISD::ATOMIC_STORE:
3929 Res = SplitVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
3930 break;
3931 case ISD::VP_STORE:
3932 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3933 break;
3934 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3935 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3936 break;
3937 case ISD::MSTORE:
3938 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3939 break;
3940 case ISD::MSCATTER:
3941 case ISD::VP_SCATTER:
3942 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3943 break;
3944 case ISD::MGATHER:
3945 case ISD::VP_GATHER:
3946 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3947 break;
3948 case ISD::VSELECT:
3949 Res = SplitVecOp_VSELECT(N, OpNo);
3950 break;
3951 case ISD::MASKED_UDIV:
3952 case ISD::MASKED_SDIV:
3953 case ISD::MASKED_UREM:
3954 case ISD::MASKED_SREM:
3955 Res = SplitVecOp_MaskedBinOp(N, OpNo);
3956 break;
3957 case ISD::VECTOR_COMPRESS:
3958 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3959 break;
3960 case ISD::STRICT_SINT_TO_FP:
3961 case ISD::STRICT_UINT_TO_FP:
3962 case ISD::SINT_TO_FP:
3963 case ISD::UINT_TO_FP:
3964 if (N->getValueType(ResNo: 0).bitsLT(
3965 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3966 Res = SplitVecOp_TruncateHelper(N);
3967 else
3968 Res = SplitVecOp_UnaryOp(N);
3969 break;
3970 case ISD::FP_TO_SINT_SAT:
3971 case ISD::FP_TO_UINT_SAT:
3972 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3973 break;
3974 case ISD::FP_TO_SINT:
3975 case ISD::FP_TO_UINT:
3976 case ISD::STRICT_FP_TO_SINT:
3977 case ISD::STRICT_FP_TO_UINT:
3978 case ISD::STRICT_FP_EXTEND:
3979 case ISD::FP_EXTEND:
3980 case ISD::SIGN_EXTEND:
3981 case ISD::ZERO_EXTEND:
3982 case ISD::ANY_EXTEND:
3983 case ISD::FTRUNC:
3984 case ISD::LROUND:
3985 case ISD::LLROUND:
3986 case ISD::LRINT:
3987 case ISD::LLRINT:
3988 Res = SplitVecOp_UnaryOp(N);
3989 break;
3990 case ISD::FLDEXP:
3991 Res = SplitVecOp_FPOpDifferentTypes(N);
3992 break;
3993
3994 case ISD::SCMP:
3995 case ISD::UCMP:
3996 Res = SplitVecOp_CMP(N);
3997 break;
3998
3999 case ISD::FAKE_USE:
4000 Res = SplitVecOp_FAKE_USE(N);
4001 break;
4002 case ISD::ANY_EXTEND_VECTOR_INREG:
4003 case ISD::SIGN_EXTEND_VECTOR_INREG:
4004 case ISD::ZERO_EXTEND_VECTOR_INREG:
4005 Res = SplitVecOp_ExtVecInRegOp(N);
4006 break;
4007
4008 case ISD::VECREDUCE_FADD:
4009 case ISD::VECREDUCE_FMUL:
4010 case ISD::VECREDUCE_ADD:
4011 case ISD::VECREDUCE_MUL:
4012 case ISD::VECREDUCE_AND:
4013 case ISD::VECREDUCE_OR:
4014 case ISD::VECREDUCE_XOR:
4015 case ISD::VECREDUCE_SMAX:
4016 case ISD::VECREDUCE_SMIN:
4017 case ISD::VECREDUCE_UMAX:
4018 case ISD::VECREDUCE_UMIN:
4019 case ISD::VECREDUCE_FMAX:
4020 case ISD::VECREDUCE_FMIN:
4021 case ISD::VECREDUCE_FMAXIMUM:
4022 case ISD::VECREDUCE_FMINIMUM:
4023 case ISD::VECREDUCE_FMAXIMUMNUM:
4024 case ISD::VECREDUCE_FMINIMUMNUM:
4025 Res = SplitVecOp_VECREDUCE(N, OpNo);
4026 break;
4027 case ISD::VECREDUCE_SEQ_FADD:
4028 case ISD::VECREDUCE_SEQ_FMUL:
4029 Res = SplitVecOp_VECREDUCE_SEQ(N);
4030 break;
4031 case ISD::VP_REDUCE_FADD:
4032 case ISD::VP_REDUCE_SEQ_FADD:
4033 case ISD::VP_REDUCE_FMUL:
4034 case ISD::VP_REDUCE_SEQ_FMUL:
4035 case ISD::VP_REDUCE_ADD:
4036 case ISD::VP_REDUCE_MUL:
4037 case ISD::VP_REDUCE_AND:
4038 case ISD::VP_REDUCE_OR:
4039 case ISD::VP_REDUCE_XOR:
4040 case ISD::VP_REDUCE_SMAX:
4041 case ISD::VP_REDUCE_SMIN:
4042 case ISD::VP_REDUCE_UMAX:
4043 case ISD::VP_REDUCE_UMIN:
4044 case ISD::VP_REDUCE_FMAX:
4045 case ISD::VP_REDUCE_FMIN:
4046 case ISD::VP_REDUCE_FMAXIMUM:
4047 case ISD::VP_REDUCE_FMINIMUM:
4048 Res = SplitVecOp_VP_REDUCE(N, OpNo);
4049 break;
4050 case ISD::CTTZ_ELTS:
4051 case ISD::CTTZ_ELTS_ZERO_POISON:
4052 Res = SplitVecOp_CttzElts(N);
4053 break;
4054 case ISD::VP_CTTZ_ELTS:
4055 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4056 Res = SplitVecOp_VP_CttzElements(N);
4057 break;
4058 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
4059 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
4060 break;
4061 case ISD::PARTIAL_REDUCE_UMLA:
4062 case ISD::PARTIAL_REDUCE_SMLA:
4063 case ISD::PARTIAL_REDUCE_SUMLA:
4064 case ISD::PARTIAL_REDUCE_FMLA:
4065 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
4066 break;
4067 case ISD::VECTOR_MATCH:
4068 Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
4069 break;
4070 }
4071
4072 // If the result is null, the sub-method took care of registering results etc.
4073 if (!Res.getNode()) return false;
4074
4075 // If the result is N, the sub-method updated N in place. Tell the legalizer
4076 // core about this.
4077 if (Res.getNode() == N)
4078 return true;
4079
4080 if (N->isStrictFPOpcode())
4081 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4082 "Invalid operand expansion");
4083 else
4084 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4085 "Invalid operand expansion");
4086
4087 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4088 return false;
4089}
4090
4091SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
4092 SDLoc DL(N);
4093
4094 SDValue LoMask, HiMask;
4095 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
4096
4097 EVT VT = N->getValueType(ResNo: 0);
4098 EVT SplitVT = LoMask.getValueType();
4099 ElementCount SplitEC = SplitVT.getVectorElementCount();
4100
4101 // Find the last active in both the low and the high masks.
4102 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
4103 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
4104
4105 // Check if any lane is active in the high mask.
4106 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
4107 // sentinel value for "none active".
4108 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
4109 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
4110 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
4111
4112 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
4113 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
4114 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
4115 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
4116 N3: LoFind);
4117}
4118
4119SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
4120 // The only possibility for an illegal operand is the mask, since result type
4121 // legalization would have handled this node already otherwise.
4122 assert(OpNo == 0 && "Illegal operand must be mask");
4123
4124 SDValue Mask = N->getOperand(Num: 0);
4125 SDValue Src0 = N->getOperand(Num: 1);
4126 SDValue Src1 = N->getOperand(Num: 2);
4127 EVT Src0VT = Src0.getValueType();
4128 SDLoc DL(N);
4129 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4130
4131 SDValue Lo, Hi;
4132 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4133 assert(Lo.getValueType() == Hi.getValueType() &&
4134 "Lo and Hi have differing types");
4135
4136 EVT LoOpVT, HiOpVT;
4137 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
4138 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4139
4140 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4141 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
4142 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
4143 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4144
4145 SDValue LoSelect =
4146 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
4147 SDValue HiSelect =
4148 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
4149
4150 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
4151}
4152
4153SDValue DAGTypeLegalizer::SplitVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
4154 assert(OpNo == 2 && "Illegal operand must be mask");
4155
4156 SDLoc DL(N);
4157 auto [LHSLo, LHSHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
4158 auto [RHSLo, RHSHi] = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4159 SDValue MaskLo, MaskHi;
4160 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
4161
4162 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo,
4163 N2: RHSLo, N3: MaskLo, Flags: N->getFlags());
4164 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi,
4165 N2: RHSHi, N3: MaskHi, Flags: N->getFlags());
4166 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4167}
4168
4169SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4170 // The only possibility for an illegal operand is the mask, since result type
4171 // legalization would have handled this node already otherwise.
4172 assert(OpNo == 1 && "Illegal operand must be mask");
4173
4174 // To split the mask, we need to split the result type too, so we can just
4175 // reuse that logic here.
4176 SDValue Lo, Hi;
4177 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4178
4179 EVT VecVT = N->getValueType(ResNo: 0);
4180 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
4181}
4182
4183SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4184 EVT ResVT = N->getValueType(ResNo: 0);
4185 SDValue Lo, Hi;
4186 SDLoc dl(N);
4187
4188 SDValue VecOp = N->getOperand(Num: OpNo);
4189 EVT VecVT = VecOp.getValueType();
4190 assert(VecVT.isVector() && "Can only split reduce vector operand");
4191 GetSplitVector(Op: VecOp, Lo, Hi);
4192 EVT LoOpVT, HiOpVT;
4193 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4194
4195 // Use the appropriate scalar instruction on the split subvectors before
4196 // reducing the now partially reduced smaller vector.
4197 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
4198 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
4199 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
4200}
4201
4202SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4203 EVT ResVT = N->getValueType(ResNo: 0);
4204 SDValue Lo, Hi;
4205 SDLoc dl(N);
4206
4207 SDValue AccOp = N->getOperand(Num: 0);
4208 SDValue VecOp = N->getOperand(Num: 1);
4209 SDNodeFlags Flags = N->getFlags();
4210
4211 EVT VecVT = VecOp.getValueType();
4212 assert(VecVT.isVector() && "Can only split reduce vector operand");
4213 GetSplitVector(Op: VecOp, Lo, Hi);
4214 EVT LoOpVT, HiOpVT;
4215 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4216
4217 // Reduce low half.
4218 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
4219
4220 // Reduce high half, using low half result as initial value.
4221 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
4222}
4223
4224SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4225 assert(N->isVPOpcode() && "Expected VP opcode");
4226 assert(OpNo == 1 && "Can only split reduce vector operand");
4227
4228 unsigned Opc = N->getOpcode();
4229 EVT ResVT = N->getValueType(ResNo: 0);
4230 SDValue Lo, Hi;
4231 SDLoc dl(N);
4232
4233 SDValue VecOp = N->getOperand(Num: OpNo);
4234 EVT VecVT = VecOp.getValueType();
4235 assert(VecVT.isVector() && "Can only split reduce vector operand");
4236 GetSplitVector(Op: VecOp, Lo, Hi);
4237
4238 SDValue MaskLo, MaskHi;
4239 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
4240
4241 SDValue EVLLo, EVLHi;
4242 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
4243
4244 const SDNodeFlags Flags = N->getFlags();
4245
4246 SDValue ResLo =
4247 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
4248 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
4249}
4250
4251SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4252 // The result has a legal vector type, but the input needs splitting.
4253 EVT ResVT = N->getValueType(ResNo: 0);
4254 SDValue Lo, Hi;
4255 SDLoc dl(N);
4256 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4257 EVT InVT = Lo.getValueType();
4258
4259 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4260 EC: InVT.getVectorElementCount());
4261
4262 if (N->isStrictFPOpcode()) {
4263 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4264 Ops: {N->getOperand(Num: 0), Lo});
4265 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4266 Ops: {N->getOperand(Num: 0), Hi});
4267
4268 // Build a factor node to remember that this operation is independent
4269 // of the other one.
4270 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4271 N2: Hi.getValue(R: 1));
4272
4273 // Legalize the chain result - switch anything that used the old chain to
4274 // use the new one.
4275 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4276 } else {
4277 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
4278 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
4279 }
4280
4281 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4282}
4283
4284// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4285SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4286 SDValue Lo, Hi;
4287 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4288 SDValue Chain =
4289 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
4290 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
4291}
4292
4293SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4294 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4295 // end up being split all the way down to individual components. Convert the
4296 // split pieces into integers and reassemble.
4297 EVT ResVT = N->getValueType(ResNo: 0);
4298 SDValue Lo, Hi;
4299 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4300 SDLoc dl(N);
4301
4302 if (ResVT.isScalableVector()) {
4303 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4304 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4305 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4306 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4307 }
4308
4309 Lo = BitConvertToInteger(Op: Lo);
4310 Hi = BitConvertToInteger(Op: Hi);
4311
4312 if (DAG.getDataLayout().isBigEndian())
4313 std::swap(a&: Lo, b&: Hi);
4314
4315 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4316}
4317
4318SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4319 unsigned OpNo) {
4320 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4321 // We know that the result type is legal.
4322 EVT ResVT = N->getValueType(ResNo: 0);
4323
4324 SDValue Vec = N->getOperand(Num: 0);
4325 SDValue SubVec = N->getOperand(Num: 1);
4326 SDValue Idx = N->getOperand(Num: 2);
4327 SDLoc dl(N);
4328
4329 SDValue Lo, Hi;
4330 GetSplitVector(Op: SubVec, Lo, Hi);
4331
4332 uint64_t IdxVal = Idx->getAsZExtVal();
4333 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4334
4335 SDValue FirstInsertion =
4336 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4337 SDValue SecondInsertion =
4338 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4339 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4340
4341 return SecondInsertion;
4342}
4343
4344SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4345 // We know that the extracted result type is legal.
4346 EVT SubVT = N->getValueType(ResNo: 0);
4347 SDValue Idx = N->getOperand(Num: 1);
4348 SDLoc dl(N);
4349 SDValue Lo, Hi;
4350
4351 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4352
4353 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4354 // Note: For scalable vectors, the index is scaled by vscale.
4355 ElementCount IdxVal =
4356 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4357 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4358
4359 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4360 ElementCount NumResultElts = SubVT.getVectorElementCount();
4361
4362 // If the extracted elements are all in the low half, do a simple extract.
4363 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4364 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4365
4366 unsigned LoEltsMin = LoElts.getKnownMinValue();
4367 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4368 SrcVT.isFixedLengthVector()) {
4369 // Extracted subvector crosses vector split, so we need to blend the two
4370 // halves.
4371 // TODO: May be able to emit partial extract_subvector.
4372 SmallVector<SDValue, 8> Elts;
4373 Elts.reserve(N: NumResultElts.getFixedValue());
4374
4375 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4376 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4377 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4378 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4379 /*Count=*/LoEltsMin - IdxValMin);
4380 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4381 /*Count=*/SubVT.getVectorNumElements() -
4382 Elts.size());
4383 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4384 }
4385
4386 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4387 ElementCount ExtractIdx = IdxVal - LoElts;
4388 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4389 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4390 Idx: ExtractIdx.getKnownMinValue());
4391
4392 EVT HiVT = Hi.getValueType();
4393 assert(HiVT.isFixedLengthVector() &&
4394 "Only fixed-vector extracts are supported in this case");
4395
4396 // We cannot create an extract_subvector that isn't a multiple of the
4397 // result size, which may go out of bounds for the last elements. Shuffle
4398 // the desired elements down to 0 and do a simple 0 extract.
4399 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4400 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4401 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4402
4403 SDValue Shuffle =
4404 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4405 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4406 }
4407
4408 // After this point the DAG node only permits extracting fixed-width
4409 // subvectors from scalable vectors.
4410 assert(SubVT.isFixedLengthVector() &&
4411 "Extracting scalable subvector from fixed-width unsupported");
4412
4413 // If the element type is i1 and we're not promoting the result, then we may
4414 // end up loading the wrong data since the bits are packed tightly into
4415 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4416 // type at index 4, then we will load a byte starting at index 0.
4417 if (SubVT.getScalarType() == MVT::i1)
4418 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4419 "subvector from a scalable predicate vector");
4420
4421 // Spill the vector to the stack. We should use the alignment for
4422 // the smallest part.
4423 SDValue Vec = N->getOperand(Num: 0);
4424 EVT VecVT = Vec.getValueType();
4425 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4426 SDValue StackPtr =
4427 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4428 auto &MF = DAG.getMachineFunction();
4429 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4430 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4431
4432 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4433 Alignment: SmallestAlign);
4434
4435 // Extract the subvector by loading the correct part.
4436 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4437
4438 return DAG.getLoad(
4439 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4440 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4441}
4442
4443SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4444 SDValue Vec = N->getOperand(Num: 0);
4445 SDValue Idx = N->getOperand(Num: 1);
4446 EVT VecVT = Vec.getValueType();
4447
4448 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4449 uint64_t IdxVal = Index->getZExtValue();
4450
4451 SDValue Lo, Hi;
4452 GetSplitVector(Op: Vec, Lo, Hi);
4453
4454 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4455
4456 if (IdxVal < LoElts)
4457 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4458 else if (!Vec.getValueType().isScalableVector())
4459 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4460 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4461 VT: Idx.getValueType())), 0);
4462 }
4463
4464 // See if the target wants to custom expand this node.
4465 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4466 return SDValue();
4467
4468 // Make the vector elements byte-addressable if they aren't already.
4469 SDLoc dl(N);
4470 EVT EltVT = VecVT.getVectorElementType();
4471 if (!EltVT.isByteSized()) {
4472 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4473 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4474 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4475 SDValue NewExtract =
4476 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4477 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4478 }
4479
4480 // Store the vector to the stack.
4481 // In cases where the vector is illegal it will be broken down into parts
4482 // and stored in parts - we should use the alignment for the smallest part.
4483 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4484 SDValue StackPtr =
4485 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4486 auto &MF = DAG.getMachineFunction();
4487 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4488 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4489 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4490 Alignment: SmallestAlign);
4491
4492 // Load back the required element.
4493 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4494
4495 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4496 // type, leaving the high bits undefined. But it can't truncate.
4497 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4498
4499 return DAG.getExtLoad(
4500 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4501 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4502 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4503}
4504
4505SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4506 SDValue Lo, Hi;
4507
4508 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4509 // splitting the result has the same effect as splitting the input operand.
4510 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4511
4512 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4513}
4514
4515SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4516 (void)OpNo;
4517 SDValue Lo, Hi;
4518 SplitVecRes_Gather(N, Lo, Hi);
4519
4520 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4521 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4522 return SDValue();
4523}
4524
4525SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4526 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4527 SDValue Ch = N->getChain();
4528 SDValue Ptr = N->getBasePtr();
4529 SDValue Offset = N->getOffset();
4530 assert(Offset.isUndef() && "Unexpected VP store offset");
4531 SDValue Mask = N->getMask();
4532 SDValue EVL = N->getVectorLength();
4533 SDValue Data = N->getValue();
4534 Align Alignment = N->getBaseAlign();
4535 SDLoc DL(N);
4536
4537 SDValue DataLo, DataHi;
4538 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4539 // Split Data operand
4540 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4541 else
4542 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4543
4544 // Split Mask operand
4545 SDValue MaskLo, MaskHi;
4546 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4547 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4548 } else {
4549 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4550 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4551 else
4552 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4553 }
4554
4555 EVT MemoryVT = N->getMemoryVT();
4556 EVT LoMemVT, HiMemVT;
4557 bool HiIsEmpty = false;
4558 std::tie(args&: LoMemVT, args&: HiMemVT) =
4559 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4560
4561 // Split EVL
4562 SDValue EVLLo, EVLHi;
4563 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4564
4565 SDValue Lo, Hi;
4566 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4567 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4568 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4569 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4570
4571 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4572 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4573 IsCompressing: N->isCompressingStore());
4574
4575 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4576 if (HiIsEmpty)
4577 return Lo;
4578
4579 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4580 IsCompressedMemory: N->isCompressingStore());
4581
4582 MachinePointerInfo MPI;
4583 if (LoMemVT.isScalableVector()) {
4584 Alignment = commonAlignment(A: Alignment,
4585 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4586 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4587 } else
4588 MPI = N->getPointerInfo().getWithOffset(
4589 O: LoMemVT.getStoreSize().getFixedValue());
4590
4591 MMO = DAG.getMachineFunction().getMachineMemOperand(
4592 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4593 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4594
4595 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4596 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4597 IsCompressing: N->isCompressingStore());
4598
4599 // Build a factor node to remember that this store is independent of the
4600 // other one.
4601 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4602}
4603
4604SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4605 unsigned OpNo) {
4606 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4607 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4608
4609 SDLoc DL(N);
4610
4611 SDValue Data = N->getValue();
4612 SDValue LoData, HiData;
4613 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4614 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4615 else
4616 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4617
4618 EVT LoMemVT, HiMemVT;
4619 bool HiIsEmpty = false;
4620 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4621 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4622
4623 SDValue Mask = N->getMask();
4624 SDValue LoMask, HiMask;
4625 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4626 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4627 else if (getTypeAction(VT: Mask.getValueType()) ==
4628 TargetLowering::TypeSplitVector)
4629 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4630 else
4631 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4632
4633 SDValue LoEVL, HiEVL;
4634 std::tie(args&: LoEVL, args&: HiEVL) =
4635 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4636
4637 // Generate the low vp_strided_store
4638 SDValue Lo = DAG.getStridedStoreVP(
4639 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4640 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4641 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4642
4643 // If the high vp_strided_store has zero storage size, only the low
4644 // vp_strided_store is needed.
4645 if (HiIsEmpty)
4646 return Lo;
4647
4648 // Generate the high vp_strided_store.
4649 // To calculate the high base address, we need to sum to the low base
4650 // address stride number of bytes for each element already stored by low,
4651 // that is: Ptr = Ptr + (LoEVL * Stride)
4652 EVT PtrVT = N->getBasePtr().getValueType();
4653 SDValue Increment =
4654 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4655 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4656 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4657
4658 Align Alignment = N->getBaseAlign();
4659 if (LoMemVT.isScalableVector())
4660 Alignment = commonAlignment(A: Alignment,
4661 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4662
4663 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4664 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4665 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4666 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4667
4668 SDValue Hi = DAG.getStridedStoreVP(
4669 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4670 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4671 IsCompressing: N->isCompressingStore());
4672
4673 // Build a factor node to remember that this store is independent of the
4674 // other one.
4675 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4676}
4677
4678SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4679 unsigned OpNo) {
4680 assert(N->isUnindexed() && "Indexed masked store of vector?");
4681 SDValue Ch = N->getChain();
4682 SDValue Ptr = N->getBasePtr();
4683 SDValue Offset = N->getOffset();
4684 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4685 SDValue Mask = N->getMask();
4686 SDValue Data = N->getValue();
4687 Align Alignment = N->getBaseAlign();
4688 SDLoc DL(N);
4689
4690 SDValue DataLo, DataHi;
4691 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4692 // Split Data operand
4693 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4694 else
4695 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4696
4697 // Split Mask operand
4698 SDValue MaskLo, MaskHi;
4699 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4700 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4701 } else {
4702 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4703 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4704 else
4705 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4706 }
4707
4708 EVT MemoryVT = N->getMemoryVT();
4709 EVT LoMemVT, HiMemVT;
4710 bool HiIsEmpty = false;
4711 std::tie(args&: LoMemVT, args&: HiMemVT) =
4712 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4713
4714 SDValue Lo, Hi, Res;
4715 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4716 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4717 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4718 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4719
4720 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4721 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4722 IsCompressing: N->isCompressingStore());
4723
4724 if (HiIsEmpty) {
4725 // The hi masked store has zero storage size.
4726 // Only the lo masked store is needed.
4727 Res = Lo;
4728 } else {
4729
4730 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4731 IsCompressedMemory: N->isCompressingStore());
4732
4733 MachinePointerInfo MPI;
4734 if (LoMemVT.isScalableVector()) {
4735 Alignment = commonAlignment(
4736 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4737 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4738 } else
4739 MPI = N->getPointerInfo().getWithOffset(
4740 O: LoMemVT.getStoreSize().getFixedValue());
4741
4742 MMO = DAG.getMachineFunction().getMachineMemOperand(
4743 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4744 BaseAlignment: Alignment,
4745 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges(), N->getMemCacheHint()));
4746
4747 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4748 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4749 IsCompressing: N->isCompressingStore());
4750
4751 // Build a factor node to remember that this store is independent of the
4752 // other one.
4753 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4754 }
4755
4756 return Res;
4757}
4758
4759SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4760 SDValue Ch = N->getChain();
4761 SDValue Ptr = N->getBasePtr();
4762 EVT MemoryVT = N->getMemoryVT();
4763 Align Alignment = N->getBaseAlign();
4764 SDLoc DL(N);
4765 struct Operands {
4766 SDValue Mask;
4767 SDValue Index;
4768 SDValue Scale;
4769 SDValue Data;
4770 } Ops = [&]() -> Operands {
4771 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4772 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4773 .Data: MSC->getValue()};
4774 }
4775 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4776 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4777 .Data: VPSC->getValue()};
4778 }();
4779 // Split all operands
4780
4781 EVT LoMemVT, HiMemVT;
4782 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4783
4784 SDValue DataLo, DataHi;
4785 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4786 // Split Data operand
4787 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4788 else
4789 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4790
4791 // Split Mask operand
4792 SDValue MaskLo, MaskHi;
4793 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4794 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4795 } else {
4796 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4797 }
4798
4799 SDValue IndexHi, IndexLo;
4800 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4801 TargetLowering::TypeSplitVector)
4802 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4803 else
4804 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4805
4806 SDValue Lo;
4807 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4808 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4809 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4810 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4811
4812 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4813 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4814 Lo =
4815 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4816 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4817
4818 // The order of the Scatter operation after split is well defined. The "Hi"
4819 // part comes after the "Lo". So these two operations should be chained one
4820 // after another.
4821 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4822 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4823 MMO, IndexType: MSC->getIndexType(),
4824 IsTruncating: MSC->isTruncatingStore());
4825 }
4826 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4827 SDValue EVLLo, EVLHi;
4828 std::tie(args&: EVLLo, args&: EVLHi) =
4829 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4830
4831 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4832 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4833 IndexType: VPSC->getIndexType());
4834
4835 // The order of the Scatter operation after split is well defined. The "Hi"
4836 // part comes after the "Lo". So these two operations should be chained one
4837 // after another.
4838 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4839 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4840 IndexType: VPSC->getIndexType());
4841}
4842
4843SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4844 assert(N->isUnindexed() && "Indexed store of vector?");
4845 assert(OpNo == 1 && "Can only split the stored value");
4846 SDLoc DL(N);
4847
4848 bool isTruncating = N->isTruncatingStore();
4849 SDValue Ch = N->getChain();
4850 SDValue Ptr = N->getBasePtr();
4851 EVT MemoryVT = N->getMemoryVT();
4852 Align Alignment = N->getBaseAlign();
4853 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4854 MMOMetadata Metadata = N->getMMOMetadataForSubAccess();
4855 SDValue Lo, Hi;
4856 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4857
4858 EVT LoMemVT, HiMemVT;
4859 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4860
4861 // Scalarize if the split halves are not byte-sized.
4862 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4863 return TLI.scalarizeVectorStore(ST: N, DAG);
4864
4865 if (isTruncating)
4866 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4867 Alignment, MMOFlags, Metadata);
4868 else
4869 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4870 Metadata);
4871
4872 MachinePointerInfo MPI;
4873 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4874
4875 if (isTruncating)
4876 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, SVT: HiMemVT, Alignment, MMOFlags,
4877 Metadata);
4878 else
4879 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, Metadata);
4880
4881 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4882}
4883
4884SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4885 SDLoc DL(N);
4886 LLVMContext &Ctx = *DAG.getContext();
4887 SDValue StVal = N->getVal();
4888 EVT VT = StVal.getValueType();
4889 EVT MemIntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: N->getMemoryVT().getSizeInBits());
4890
4891 // The store needs a single value spanning the full memory width. If the
4892 // value can be held in a legal vector register, keep it there and extract
4893 // the low integer element of the memory width. This lets the store be issued
4894 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4895 // bitcasting the split vector straight to a scalar integer, which would
4896 // reassemble the value element by element in GPRs.
4897 //
4898 // Reinterpret the value as a same-shaped integer vector first: an FP element
4899 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
4900 // integer-of-element-size form does. Ask the target which legal vector type
4901 // it widens to.
4902 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
4903 EVT IntEltVT = IntVecVT.getVectorElementType();
4904 EVT WideVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT: IntVecVT);
4905 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(VT: MemIntVT) &&
4906 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
4907 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
4908 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
4909 SDValue Wide = ModifyToType(InOp: DAG.getBitcast(VT: IntVecVT, V: StVal), NVT: WideVT);
4910 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
4911 EVT MemVecVT = EVT::getVectorVT(Context&: Ctx, VT: MemIntVT, NumElements: NumMemElts);
4912 SDValue Elt = DAG.getExtractVectorElt(DL, VT: MemIntVT,
4913 Vec: DAG.getBitcast(VT: MemVecVT, V: Wide), Idx: 0);
4914 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: Elt,
4915 Val: N->getBasePtr(), MMO: N->getMemOperand());
4916 }
4917
4918 // Otherwise issue a single atomic store of an integer that spans the full
4919 // memory width. Bitcasting the (illegal) vector value to that integer lets
4920 // the type legalizer further legalize the BITCAST input as needed, while the
4921 // ATOMIC_STORE itself uses only the legal integer type.
4922 EVT IntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits());
4923 SDValue AsInt = DAG.getBitcast(VT: IntVT, V: StVal);
4924 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: AsInt,
4925 Val: N->getBasePtr(), MMO: N->getMemOperand());
4926}
4927
4928SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4929 SDLoc DL(N);
4930
4931 // The input operands all must have the same type, and we know the result
4932 // type is valid. Convert this to a buildvector which extracts all the
4933 // input elements.
4934 // TODO: If the input elements are power-two vectors, we could convert this to
4935 // a new CONCAT_VECTORS node with elements that are half-wide.
4936 SmallVector<SDValue, 32> Elts;
4937 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4938 for (const SDValue &Op : N->op_values()) {
4939 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4940 i != e; ++i) {
4941 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4942 }
4943 }
4944
4945 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4946}
4947
4948SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4949 // The result type is legal, but the input type is illegal. If splitting
4950 // ends up with the result type of each half still being legal, just
4951 // do that. If, however, that would result in an illegal result type,
4952 // we can try to get more clever with power-two vectors. Specifically,
4953 // split the input type, but also widen the result element size, then
4954 // concatenate the halves and truncate again. For example, consider a target
4955 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4956 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4957 // %inlo = v4i32 extract_subvector %in, 0
4958 // %inhi = v4i32 extract_subvector %in, 4
4959 // %lo16 = v4i16 trunc v4i32 %inlo
4960 // %hi16 = v4i16 trunc v4i32 %inhi
4961 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4962 // %res = v8i8 trunc v8i16 %in16
4963 //
4964 // Without this transform, the original truncate would end up being
4965 // scalarized, which is pretty much always a last resort.
4966 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4967 SDValue InVec = N->getOperand(Num: OpNo);
4968 EVT InVT = InVec->getValueType(ResNo: 0);
4969 EVT OutVT = N->getValueType(ResNo: 0);
4970 ElementCount NumElements = OutVT.getVectorElementCount();
4971 bool IsFloat = OutVT.isFloatingPoint();
4972
4973 unsigned InElementSize = InVT.getScalarSizeInBits();
4974 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4975
4976 // Determine the split output VT. If its legal we can just split dirctly.
4977 EVT LoOutVT, HiOutVT;
4978 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4979 assert(LoOutVT == HiOutVT && "Unequal split?");
4980
4981 // If the input elements are only 1/2 the width of the result elements,
4982 // just use the normal splitting. Our trick only work if there's room
4983 // to split more than once.
4984 if (isTypeLegal(VT: LoOutVT) || InElementSize <= OutElementSize * 2 ||
4985 (IsFloat && !isPowerOf2_32(Value: InElementSize)))
4986 return SplitVecOp_UnaryOp(N);
4987 SDLoc DL(N);
4988
4989 // Don't touch if this will be scalarized.
4990 EVT FinalVT = InVT;
4991 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4992 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4993
4994 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4995 return SplitVecOp_UnaryOp(N);
4996
4997 // Get the split input vector.
4998 SDValue InLoVec, InHiVec;
4999 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
5000
5001 // Truncate them to 1/2 the element size.
5002 //
5003 // This assumes the number of elements is a power of two; any vector that
5004 // isn't should be widened, not split.
5005 EVT HalfElementVT = IsFloat ?
5006 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
5007 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
5008 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
5009 EC: NumElements.divideCoefficientBy(RHS: 2));
5010
5011 SDValue HalfLo;
5012 SDValue HalfHi;
5013 SDValue Chain;
5014 if (N->isStrictFPOpcode()) {
5015 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
5016 Ops: {N->getOperand(Num: 0), InLoVec});
5017 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
5018 Ops: {N->getOperand(Num: 0), InHiVec});
5019 // Legalize the chain result - switch anything that used the old chain to
5020 // use the new one.
5021 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
5022 N2: HalfHi.getValue(R: 1));
5023 } else {
5024 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
5025 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
5026 }
5027
5028 // Concatenate them to get the full intermediate truncation result.
5029 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
5030 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
5031 N2: HalfHi);
5032 // Now finish up by truncating all the way down to the original result
5033 // type. This should normally be something that ends up being legal directly,
5034 // but in theory if a target has very wide vectors and an annoyingly
5035 // restricted set of legal types, this split can chain to build things up.
5036
5037 if (N->isStrictFPOpcode()) {
5038 SDValue Res = DAG.getNode(
5039 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
5040 Ops: {Chain, InterVec,
5041 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
5042 // Relink the chain
5043 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
5044 return Res;
5045 }
5046
5047 return IsFloat
5048 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
5049 N2: DAG.getTargetConstant(
5050 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
5051 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
5052}
5053
5054SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
5055 unsigned Opc = N->getOpcode();
5056 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
5057 assert(N->getValueType(0).isVector() &&
5058 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
5059 "Operand types must be vectors");
5060 // The result has a legal vector type, but the input needs splitting.
5061 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
5062 SDLoc DL(N);
5063 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
5064 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
5065
5066 EVT VT = N->getValueType(ResNo: 0);
5067 EVT PartResVT = getSetCCResultType(VT: Lo0.getValueType());
5068
5069 if (Opc == ISD::SETCC) {
5070 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
5071 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
5072 } else {
5073 assert(isStrict && "unexpected node");
5074 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
5075 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
5076 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
5077 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
5078 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5079 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
5080 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5081 }
5082
5083 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
5084 SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: LoRes, N2: HiRes);
5085 if (VT == ConcatVT)
5086 return Con;
5087
5088 EVT OpVT = N->getOperand(Num: 0).getValueType();
5089 ISD::NodeType ExtendCode =
5090 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
5091 return DAG.getExtOrTrunc(Op: Con, DL, VT, Opcode: ExtendCode);
5092}
5093
5094
5095SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
5096 // The result has a legal vector type, but the input needs splitting.
5097 EVT ResVT = N->getValueType(ResNo: 0);
5098 SDValue Lo, Hi;
5099 SDLoc DL(N);
5100 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
5101 EVT InVT = Lo.getValueType();
5102
5103 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5104 EC: InVT.getVectorElementCount());
5105
5106 if (N->isStrictFPOpcode()) {
5107 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5108 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
5109 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
5110 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
5111 // Legalize the chain result - switch anything that used the old chain to
5112 // use the new one.
5113 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
5114 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
5115 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5116 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
5117 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1),
5118 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5119 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1),
5120 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
5121 } else {
5122 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
5123 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
5124 }
5125
5126 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
5127}
5128
5129// Split a vector type in an FP binary operation where the second operand has a
5130// different type from the first.
5131//
5132// The result (and the first input) has a legal vector type, but the second
5133// input needs splitting.
5134SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5135 SDLoc DL(N);
5136
5137 EVT LHSLoVT, LHSHiVT;
5138 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5139
5140 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
5141 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
5142
5143 SDValue LHSLo, LHSHi;
5144 std::tie(args&: LHSLo, args&: LHSHi) =
5145 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
5146
5147 SDValue RHSLo, RHSHi;
5148 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
5149
5150 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
5151 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
5152
5153 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
5154}
5155
5156SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5157 LLVMContext &Ctxt = *DAG.getContext();
5158 SDLoc dl(N);
5159
5160 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5161 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
5162 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
5163
5164 EVT ResVT = N->getValueType(ResNo: 0);
5165 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5166 EVT NewResVT =
5167 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
5168
5169 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
5170 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
5171
5172 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5173}
5174
5175SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5176 EVT ResVT = N->getValueType(ResNo: 0);
5177 SDValue Lo, Hi;
5178 SDLoc dl(N);
5179 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
5180 EVT InVT = Lo.getValueType();
5181
5182 EVT NewResVT =
5183 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5184 EC: InVT.getVectorElementCount());
5185
5186 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
5187 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
5188
5189 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5190}
5191
5192SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5193 SDLoc DL(N);
5194 EVT ResVT = N->getValueType(ResNo: 0);
5195
5196 SDValue Lo, Hi;
5197 SDValue VecOp = N->getOperand(Num: 0);
5198 GetSplitVector(Op: VecOp, Lo, Hi);
5199
5200 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5201 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5202 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
5203 SDValue VL =
5204 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
5205 SDValue ResLoNotVL =
5206 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
5207 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
5208 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
5209 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
5210}
5211
5212SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5213 SDLoc DL(N);
5214 EVT ResVT = N->getValueType(ResNo: 0);
5215
5216 SDValue Lo, Hi;
5217 SDValue VecOp = N->getOperand(Num: 0);
5218 GetSplitVector(Op: VecOp, Lo, Hi);
5219
5220 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
5221 auto [EVLLo, EVLHi] =
5222 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
5223 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
5224
5225 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5226 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5227 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5228 SDValue ResLoNotEVL =
5229 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
5230 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5231 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
5232 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
5233}
5234
5235SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5236 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
5237 SDLoc DL(HG);
5238 SDValue Inc = HG->getInc();
5239 SDValue Ptr = HG->getBasePtr();
5240 SDValue Scale = HG->getScale();
5241 SDValue IntID = HG->getIntID();
5242 EVT MemVT = HG->getMemoryVT();
5243 MachineMemOperand *MMO = HG->getMemOperand();
5244 ISD::MemIndexType IndexType = HG->getIndexType();
5245
5246 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5247 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
5248 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
5249 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5250 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
5251 Ops: OpsLo, MMO, IndexType);
5252 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5253 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
5254 MMO, IndexType);
5255}
5256
5257SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
5258 SDLoc DL(N);
5259
5260 if (OpNo == 0) {
5261 EVT LoResVT, HiResVT;
5262 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5263 SDValue SourceLo, SourceHi;
5264 std::tie(args&: SourceLo, args&: SourceHi) = DAG.SplitVectorOperand(N, OpNo: 0);
5265 SDValue MaskLo, MaskHi;
5266 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVectorOperand(N, OpNo: 2);
5267
5268 SDValue MatchLo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: LoResVT, N1: SourceLo,
5269 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
5270 SDValue MatchHi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: HiResVT, N1: SourceHi,
5271 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
5272 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo,
5273 N2: MatchHi);
5274 }
5275
5276 // Note: The Mask (OpNo == 2) should be widened with the result.
5277 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
5278
5279 SDValue NeedleLo, NeedleHi;
5280 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: NeedleLo, Hi&: NeedleHi);
5281
5282 SDValue MatchLo =
5283 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5284 N2: NeedleLo, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5285 SDValue MatchHi =
5286 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5287 N2: NeedleHi, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5288 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo, N2: MatchHi);
5289}
5290
5291SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5292 SDValue Acc = N->getOperand(Num: 0);
5293 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5294 "Accumulator should already be a legal type, and shouldn't need "
5295 "further splitting");
5296
5297 SDLoc DL(N);
5298 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5299 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
5300 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
5301 unsigned Opcode = N->getOpcode();
5302 EVT ResultVT = Acc.getValueType();
5303
5304 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
5305 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
5306}
5307
5308//===----------------------------------------------------------------------===//
5309// Result Vector Widening
5310//===----------------------------------------------------------------------===//
5311
5312void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5313 unsigned WidenResNo) {
5314 unsigned NumResults = N->getNumValues();
5315 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5316 if (ResNo == WidenResNo)
5317 continue;
5318 EVT ResVT = N->getValueType(ResNo);
5319 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
5320 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
5321 } else {
5322 SDLoc DL(N);
5323 SDValue ResVal =
5324 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
5325 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
5326 }
5327 }
5328}
5329
5330void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5331 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5332
5333 // See if the target wants to custom widen this node.
5334 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
5335 return;
5336
5337 SDValue Res = SDValue();
5338
5339 auto unrollExpandedOp = [&]() {
5340 // We're going to widen this vector op to a legal type by padding with undef
5341 // elements. If the wide vector op is eventually going to be expanded to
5342 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5343 // libcalls on the undef elements.
5344 EVT ResVT = N->getValueType(ResNo);
5345 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5346 EVT VT0 = N->getValueType(ResNo: 0);
5347 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
5348 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT0.getScalarType())) {
5349 SDValue Unrolled =
5350 DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
5351 Res = Unrolled.getValue(R: ResNo);
5352 if (N->getNumValues() > 1)
5353 ReplaceOtherWidenResults(N, WidenNode: Unrolled.getNode(), WidenResNo: ResNo);
5354 return true;
5355 }
5356 return false;
5357 };
5358
5359 switch (N->getOpcode()) {
5360 default:
5361#ifndef NDEBUG
5362 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5363 N->dump(&DAG);
5364 dbgs() << "\n";
5365#endif
5366 report_fatal_error(reason: "do not know how to widen the result of this operator!");
5367
5368 case ISD::LOOP_DEPENDENCE_RAW_MASK:
5369 case ISD::LOOP_DEPENDENCE_WAR_MASK:
5370 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5371 break;
5372 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5373 case ISD::ADDRSPACECAST:
5374 Res = WidenVecRes_ADDRSPACECAST(N);
5375 break;
5376 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5377 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5378 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5379 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5380 case ISD::INSERT_SUBVECTOR:
5381 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5382 break;
5383 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5384 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5385 case ISD::ATOMIC_LOAD:
5386 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5387 break;
5388 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5389 case ISD::STEP_VECTOR:
5390 case ISD::SPLAT_VECTOR:
5391 case ISD::SCALAR_TO_VECTOR:
5392 Res = WidenVecRes_ScalarOp(N);
5393 break;
5394 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5395 case ISD::VSELECT:
5396 case ISD::SELECT:
5397 case ISD::VP_MERGE:
5398 Res = WidenVecRes_Select(N);
5399 break;
5400 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5401 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5402 case ISD::POISON:
5403 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5404 case ISD::VECTOR_SHUFFLE:
5405 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5406 break;
5407 case ISD::VP_LOAD:
5408 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5409 break;
5410 case ISD::VP_LOAD_FF:
5411 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5412 break;
5413 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5414 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5415 break;
5416 case ISD::VECTOR_COMPRESS:
5417 Res = WidenVecRes_VECTOR_COMPRESS(N);
5418 break;
5419 case ISD::MLOAD:
5420 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5421 break;
5422 case ISD::MGATHER:
5423 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5424 break;
5425 case ISD::VP_GATHER:
5426 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5427 break;
5428 case ISD::VECTOR_REVERSE:
5429 Res = WidenVecRes_VECTOR_REVERSE(N);
5430 break;
5431 case ISD::GET_ACTIVE_LANE_MASK:
5432 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5433 break;
5434 case ISD::VECTOR_INTERLEAVE:
5435 WidenVecRes_VECTOR_INTERLEAVE(N);
5436 break;
5437 case ISD::VECTOR_MATCH:
5438 Res = WidenVecRes_VECTOR_MATCH(N);
5439 break;
5440 case ISD::VECTOR_DEINTERLEAVE:
5441 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5442 break;
5443
5444 case ISD::ADD:
5445 case ISD::AND:
5446 case ISD::MUL:
5447 case ISD::MULHS:
5448 case ISD::MULHU:
5449 case ISD::ABDS:
5450 case ISD::ABDU:
5451 case ISD::OR:
5452 case ISD::SUB:
5453 case ISD::XOR:
5454 case ISD::SHL:
5455 case ISD::SRA:
5456 case ISD::SRL:
5457 case ISD::CLMUL:
5458 case ISD::CLMULR:
5459 case ISD::CLMULH:
5460 case ISD::PEXT:
5461 case ISD::PDEP:
5462 case ISD::FMINNUM:
5463 case ISD::FMINNUM_IEEE:
5464 case ISD::FMAXNUM:
5465 case ISD::FMAXNUM_IEEE:
5466 case ISD::FMINIMUM:
5467 case ISD::FMAXIMUM:
5468 case ISD::FMINIMUMNUM:
5469 case ISD::FMAXIMUMNUM:
5470 case ISD::SMIN:
5471 case ISD::SMAX:
5472 case ISD::UMIN:
5473 case ISD::UMAX:
5474 case ISD::UADDSAT:
5475 case ISD::SADDSAT:
5476 case ISD::USUBSAT:
5477 case ISD::SSUBSAT:
5478 case ISD::SSHLSAT:
5479 case ISD::USHLSAT:
5480 case ISD::ROTL:
5481 case ISD::ROTR:
5482 case ISD::AVGFLOORS:
5483 case ISD::AVGFLOORU:
5484 case ISD::AVGCEILS:
5485 case ISD::AVGCEILU:
5486 case ISD::AVGSUBFLOORS:
5487 case ISD::AVGSUBFLOORU:
5488 // Vector-predicated binary op widening. Note that -- unlike the
5489 // unpredicated versions -- we don't have to worry about trapping on
5490 // operations like UDIV, FADD, etc., as we pass on the original vector
5491 // length parameter. This means the widened elements containing garbage
5492 // aren't active.
5493 case ISD::VP_SDIV:
5494 case ISD::VP_UDIV:
5495 case ISD::VP_SREM:
5496 case ISD::VP_UREM:
5497 Res = WidenVecRes_Binary(N);
5498 break;
5499
5500 case ISD::MASKED_UDIV:
5501 case ISD::MASKED_SDIV:
5502 case ISD::MASKED_UREM:
5503 case ISD::MASKED_SREM:
5504 Res = WidenVecRes_MaskedBinary(N);
5505 break;
5506
5507 case ISD::SCMP:
5508 case ISD::UCMP:
5509 Res = WidenVecRes_CMP(N);
5510 break;
5511
5512 case ISD::FPOW:
5513 case ISD::FATAN2:
5514 case ISD::FREM:
5515 if (unrollExpandedOp())
5516 break;
5517 // If the target has custom/legal support for the scalar FP intrinsic ops
5518 // (they are probably not destined to become libcalls), then widen those
5519 // like any other binary ops.
5520 [[fallthrough]];
5521
5522 case ISD::FADD:
5523 case ISD::FMUL:
5524 case ISD::FSUB:
5525 case ISD::FDIV:
5526 case ISD::SDIV:
5527 case ISD::UDIV:
5528 case ISD::SREM:
5529 case ISD::UREM:
5530 Res = WidenVecRes_BinaryCanTrap(N);
5531 break;
5532
5533 case ISD::SMULFIX:
5534 case ISD::SMULFIXSAT:
5535 case ISD::UMULFIX:
5536 case ISD::UMULFIXSAT:
5537 // These are binary operations, but with an extra operand that shouldn't
5538 // be widened (the scale).
5539 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5540 break;
5541
5542#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5543 case ISD::STRICT_##DAGN:
5544#include "llvm/IR/ConstrainedOps.def"
5545 Res = WidenVecRes_StrictFP(N);
5546 break;
5547
5548 case ISD::UADDO:
5549 case ISD::SADDO:
5550 case ISD::USUBO:
5551 case ISD::SSUBO:
5552 case ISD::UMULO:
5553 case ISD::SMULO:
5554 Res = WidenVecRes_OverflowOp(N, ResNo);
5555 break;
5556
5557 case ISD::FCOPYSIGN:
5558 Res = WidenVecRes_FCOPYSIGN(N);
5559 break;
5560
5561 case ISD::IS_FPCLASS:
5562 case ISD::FPTRUNC_ROUND:
5563 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5564 break;
5565
5566 case ISD::FLDEXP:
5567 case ISD::FPOWI:
5568 if (!unrollExpandedOp())
5569 Res = WidenVecRes_ExpOp(N);
5570 break;
5571
5572 case ISD::ANY_EXTEND_VECTOR_INREG:
5573 case ISD::SIGN_EXTEND_VECTOR_INREG:
5574 case ISD::ZERO_EXTEND_VECTOR_INREG:
5575 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5576 break;
5577
5578 case ISD::ANY_EXTEND:
5579 case ISD::FP_EXTEND:
5580 case ISD::FP_ROUND:
5581 case ISD::FP_TO_SINT:
5582 case ISD::FP_TO_UINT:
5583 case ISD::SIGN_EXTEND:
5584 case ISD::SINT_TO_FP:
5585 case ISD::TRUNCATE:
5586 case ISD::UINT_TO_FP:
5587 case ISD::ZERO_EXTEND:
5588 case ISD::CONVERT_FROM_ARBITRARY_FP:
5589 case ISD::CONVERT_TO_ARBITRARY_FP:
5590 Res = WidenVecRes_Convert(N);
5591 break;
5592
5593 case ISD::FP_TO_SINT_SAT:
5594 case ISD::FP_TO_UINT_SAT:
5595 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5596 break;
5597
5598 case ISD::LRINT:
5599 case ISD::LLRINT:
5600 case ISD::LROUND:
5601 case ISD::LLROUND:
5602 Res = WidenVecRes_XROUND(N);
5603 break;
5604
5605 case ISD::FACOS:
5606 case ISD::FASIN:
5607 case ISD::FATAN:
5608 case ISD::FCEIL:
5609 case ISD::FCOS:
5610 case ISD::FCOSH:
5611 case ISD::FEXP:
5612 case ISD::FEXP2:
5613 case ISD::FEXP10:
5614 case ISD::FFLOOR:
5615 case ISD::FLOG:
5616 case ISD::FLOG10:
5617 case ISD::FLOG2:
5618 case ISD::FNEARBYINT:
5619 case ISD::FRINT:
5620 case ISD::FROUND:
5621 case ISD::FROUNDEVEN:
5622 case ISD::FSIN:
5623 case ISD::FSINH:
5624 case ISD::FSQRT:
5625 case ISD::FTAN:
5626 case ISD::FTANH:
5627 case ISD::FTRUNC:
5628 if (unrollExpandedOp())
5629 break;
5630 // If the target has custom/legal support for the scalar FP intrinsic ops
5631 // (they are probably not destined to become libcalls), then widen those
5632 // like any other unary ops.
5633 [[fallthrough]];
5634
5635 case ISD::ABS:
5636 case ISD::ABS_MIN_POISON:
5637 case ISD::BITREVERSE:
5638 case ISD::BSWAP:
5639 case ISD::CTLZ:
5640 case ISD::CTLZ_ZERO_POISON:
5641 case ISD::CTPOP:
5642 case ISD::CTTZ:
5643 case ISD::CTTZ_ZERO_POISON:
5644 case ISD::FNEG:
5645 case ISD::FABS:
5646 case ISD::FREEZE:
5647 case ISD::ARITH_FENCE:
5648 case ISD::FCANONICALIZE:
5649 case ISD::AssertNoFPClass:
5650 case ISD::MASK_BEFOREFIRST:
5651 Res = WidenVecRes_Unary(N);
5652 break;
5653 case ISD::FMA:
5654 case ISD::FSHL:
5655 case ISD::FSHR:
5656 Res = WidenVecRes_Ternary(N);
5657 break;
5658 case ISD::FMODF:
5659 case ISD::FFREXP:
5660 case ISD::FSINCOS:
5661 case ISD::FSINCOSPI: {
5662 if (!unrollExpandedOp())
5663 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5664 break;
5665 }
5666 case ISD::PARTIAL_REDUCE_UMLA:
5667 case ISD::PARTIAL_REDUCE_SMLA:
5668 case ISD::PARTIAL_REDUCE_SUMLA:
5669 case ISD::PARTIAL_REDUCE_FMLA:
5670 Res = WidenVecRes_PARTIAL_REDUCE_MLA(N);
5671 break;
5672 }
5673
5674 // If Res is null, the sub-method took care of registering the result.
5675 if (Res.getNode())
5676 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5677}
5678
5679SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5680 // Ternary op widening.
5681 SDLoc dl(N);
5682 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5683 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5684 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5685 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5686 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5687}
5688
5689SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5690 // Binary op widening.
5691 SDLoc dl(N);
5692 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5693 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5694 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5695 if (N->getNumOperands() == 2)
5696 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5697 Flags: N->getFlags());
5698
5699 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5700 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
5701 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
5702 "Expected VP opcode");
5703
5704 SDValue Mask =
5705 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5706 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5707 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5708}
5709
5710SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5711 SDLoc dl(N);
5712 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5713 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5714 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5715 SDValue Mask = N->getOperand(Num: 2);
5716 EVT WideMaskVT = WidenVT.changeVectorElementType(
5717 Context&: *DAG.getContext(), EltVT: Mask.getValueType().getVectorElementType());
5718 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeros=*/FillWithZeroes: true);
5719 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask,
5720 Flags: N->getFlags());
5721}
5722
5723SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5724 LLVMContext &Ctxt = *DAG.getContext();
5725 SDLoc dl(N);
5726
5727 SDValue LHS = N->getOperand(Num: 0);
5728 SDValue RHS = N->getOperand(Num: 1);
5729 EVT OpVT = LHS.getValueType();
5730 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5731 LHS = GetWidenedVector(Op: LHS);
5732 RHS = GetWidenedVector(Op: RHS);
5733 OpVT = LHS.getValueType();
5734 }
5735
5736 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5737 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5738 if (WidenResEC == OpVT.getVectorElementCount()) {
5739 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5740 }
5741
5742 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5743}
5744
5745SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5746 // Binary op widening, but with an extra operand that shouldn't be widened.
5747 SDLoc dl(N);
5748 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5749 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5750 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5751 SDValue InOp3 = N->getOperand(Num: 2);
5752 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5753 Flags: N->getFlags());
5754}
5755
5756// Given a vector of operations that have been broken up to widen, see
5757// if we can collect them together into the next widest legal VT. This
5758// implementation is trap-safe.
5759static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5760 SmallVectorImpl<SDValue> &ConcatOps,
5761 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5762 EVT WidenVT) {
5763 // Check to see if we have a single operation with the widen type.
5764 if (ConcatEnd == 1) {
5765 VT = ConcatOps[0].getValueType();
5766 if (VT == WidenVT)
5767 return ConcatOps[0];
5768 }
5769
5770 SDLoc dl(ConcatOps[0]);
5771 EVT WidenEltVT = WidenVT.getVectorElementType();
5772
5773 // while (Some element of ConcatOps is not of type MaxVT) {
5774 // From the end of ConcatOps, collect elements of the same type and put
5775 // them into an op of the next larger supported type
5776 // }
5777 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5778 int Idx = ConcatEnd - 1;
5779 VT = ConcatOps[Idx--].getValueType();
5780 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5781 Idx--;
5782
5783 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5784 EVT NextVT;
5785 do {
5786 NextSize *= 2;
5787 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5788 } while (!TLI.isTypeLegal(VT: NextVT));
5789
5790 if (!VT.isVector()) {
5791 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5792 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5793 unsigned NumToInsert = ConcatEnd - Idx - 1;
5794 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5795 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5796 ConcatOps[Idx+1] = VecOp;
5797 ConcatEnd = Idx + 2;
5798 } else {
5799 // Vector type, create a CONCAT_VECTORS of type NextVT
5800 SDValue undefVec = DAG.getPOISON(VT);
5801 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5802 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5803 unsigned RealVals = ConcatEnd - Idx - 1;
5804 unsigned SubConcatEnd = 0;
5805 unsigned SubConcatIdx = Idx + 1;
5806 while (SubConcatEnd < RealVals)
5807 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5808 while (SubConcatEnd < OpsToConcat)
5809 SubConcatOps[SubConcatEnd++] = undefVec;
5810 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5811 VT: NextVT, Ops: SubConcatOps);
5812 ConcatEnd = SubConcatIdx + 1;
5813 }
5814 }
5815
5816 // Check to see if we have a single operation with the widen type.
5817 if (ConcatEnd == 1) {
5818 VT = ConcatOps[0].getValueType();
5819 if (VT == WidenVT)
5820 return ConcatOps[0];
5821 }
5822
5823 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5824 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5825 if (NumOps != ConcatEnd ) {
5826 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5827 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5828 ConcatOps[j] = UndefVal;
5829 }
5830 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5831 Ops: ArrayRef(ConcatOps.data(), NumOps));
5832}
5833
5834SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5835 // Binary op widening for operations that can trap.
5836 unsigned Opcode = N->getOpcode();
5837 SDLoc dl(N);
5838 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5839 EVT WidenEltVT = WidenVT.getVectorElementType();
5840 EVT VT = WidenVT;
5841 unsigned NumElts = VT.getVectorMinNumElements();
5842 const SDNodeFlags Flags = N->getFlags();
5843 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5844 NumElts = NumElts / 2;
5845 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5846 }
5847
5848 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5849 // Operation doesn't trap so just widen as normal.
5850 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5851 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5852 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5853 }
5854
5855 // Generate a vp.op if it is custom/legal for the target. This avoids need
5856 // to split and tile the subvectors (below), because the inactive lanes can
5857 // simply be disabled. To avoid possible recursion, only do this if the
5858 // widened mask type is legal.
5859 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5860 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5861 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5862 EC: WidenVT.getVectorElementCount());
5863 TLI.isTypeLegal(VT: WideMaskVT)) {
5864 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5865 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5866 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5867 SDValue EVL =
5868 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5869 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5870 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5871 Flags);
5872 }
5873 }
5874
5875 // FIXME: Improve support for scalable vectors.
5876 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5877
5878 // No legal vector version so unroll the vector operation and then widen.
5879 if (NumElts == 1)
5880 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5881
5882 // Since the operation can trap, apply operation on the original vector.
5883 EVT MaxVT = VT;
5884 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5885 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5886 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5887
5888 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5889 unsigned ConcatEnd = 0; // Current ConcatOps index.
5890 int Idx = 0; // Current Idx into input vectors.
5891
5892 // NumElts := greatest legal vector size (at most WidenVT)
5893 // while (orig. vector has unhandled elements) {
5894 // take munches of size NumElts from the beginning and add to ConcatOps
5895 // NumElts := next smaller supported vector size or 1
5896 // }
5897 while (CurNumElts != 0) {
5898 while (CurNumElts >= NumElts) {
5899 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5900 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5901 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5902 Idx += NumElts;
5903 CurNumElts -= NumElts;
5904 }
5905 do {
5906 NumElts = NumElts / 2;
5907 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5908 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5909
5910 if (NumElts == 1) {
5911 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5912 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5913 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5914 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5915 N1: EOp1, N2: EOp2, Flags);
5916 }
5917 CurNumElts = 0;
5918 }
5919 }
5920
5921 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5922}
5923
5924SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5925 switch (N->getOpcode()) {
5926 case ISD::STRICT_FSETCC:
5927 case ISD::STRICT_FSETCCS:
5928 return WidenVecRes_STRICT_FSETCC(N);
5929 case ISD::STRICT_FP_EXTEND:
5930 case ISD::STRICT_FP_ROUND:
5931 case ISD::STRICT_FP_TO_SINT:
5932 case ISD::STRICT_FP_TO_UINT:
5933 case ISD::STRICT_SINT_TO_FP:
5934 case ISD::STRICT_UINT_TO_FP:
5935 return WidenVecRes_Convert_StrictFP(N);
5936 default:
5937 break;
5938 }
5939
5940 // StrictFP op widening for operations that can trap.
5941 unsigned NumOpers = N->getNumOperands();
5942 unsigned Opcode = N->getOpcode();
5943 SDLoc dl(N);
5944 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5945 EVT WidenEltVT = WidenVT.getVectorElementType();
5946 EVT VT = WidenVT;
5947 unsigned NumElts = VT.getVectorNumElements();
5948 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5949 NumElts = NumElts / 2;
5950 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5951 }
5952
5953 // No legal vector version so unroll the vector operation and then widen.
5954 if (NumElts == 1)
5955 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5956
5957 // Since the operation can trap, apply operation on the original vector.
5958 EVT MaxVT = VT;
5959 SmallVector<SDValue, 4> InOps;
5960 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5961
5962 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5963 SmallVector<SDValue, 16> Chains;
5964 unsigned ConcatEnd = 0; // Current ConcatOps index.
5965 int Idx = 0; // Current Idx into input vectors.
5966
5967 // The Chain is the first operand.
5968 InOps.push_back(Elt: N->getOperand(Num: 0));
5969
5970 // Now process the remaining operands.
5971 for (unsigned i = 1; i < NumOpers; ++i) {
5972 SDValue Oper = N->getOperand(Num: i);
5973
5974 EVT OpVT = Oper.getValueType();
5975 if (OpVT.isVector()) {
5976 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5977 Oper = GetWidenedVector(Op: Oper);
5978 else {
5979 EVT WideOpVT =
5980 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5981 EC: WidenVT.getVectorElementCount());
5982 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5983 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5984 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5985 }
5986 }
5987
5988 InOps.push_back(Elt: Oper);
5989 }
5990
5991 // NumElts := greatest legal vector size (at most WidenVT)
5992 // while (orig. vector has unhandled elements) {
5993 // take munches of size NumElts from the beginning and add to ConcatOps
5994 // NumElts := next smaller supported vector size or 1
5995 // }
5996 while (CurNumElts != 0) {
5997 while (CurNumElts >= NumElts) {
5998 SmallVector<SDValue, 4> EOps;
5999
6000 for (unsigned i = 0; i < NumOpers; ++i) {
6001 SDValue Op = InOps[i];
6002
6003 EVT OpVT = Op.getValueType();
6004 if (OpVT.isVector()) {
6005 EVT OpExtractVT =
6006 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
6007 EC: VT.getVectorElementCount());
6008 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
6009 }
6010
6011 EOps.push_back(Elt: Op);
6012 }
6013
6014 EVT OperVT[] = {VT, MVT::Other};
6015 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
6016 ConcatOps[ConcatEnd++] = Oper;
6017 Chains.push_back(Elt: Oper.getValue(R: 1));
6018 Idx += NumElts;
6019 CurNumElts -= NumElts;
6020 }
6021 do {
6022 NumElts = NumElts / 2;
6023 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
6024 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
6025
6026 if (NumElts == 1) {
6027 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
6028 SmallVector<SDValue, 4> EOps;
6029
6030 for (unsigned i = 0; i < NumOpers; ++i) {
6031 SDValue Op = InOps[i];
6032
6033 EVT OpVT = Op.getValueType();
6034 if (OpVT.isVector())
6035 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
6036 Idx);
6037
6038 EOps.push_back(Elt: Op);
6039 }
6040
6041 EVT WidenVT[] = {WidenEltVT, MVT::Other};
6042 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
6043 ConcatOps[ConcatEnd++] = Oper;
6044 Chains.push_back(Elt: Oper.getValue(R: 1));
6045 }
6046 CurNumElts = 0;
6047 }
6048 }
6049
6050 // Build a factor node to remember all the Ops that have been created.
6051 SDValue NewChain;
6052 if (Chains.size() == 1)
6053 NewChain = Chains[0];
6054 else
6055 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
6056 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6057
6058 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
6059}
6060
6061SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
6062 SDLoc DL(N);
6063 EVT ResVT = N->getValueType(ResNo: 0);
6064 EVT OvVT = N->getValueType(ResNo: 1);
6065 EVT WideResVT, WideOvVT;
6066 SDValue WideLHS, WideRHS;
6067
6068 // TODO: This might result in a widen/split loop.
6069 if (ResNo == 0) {
6070 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
6071 WideOvVT = EVT::getVectorVT(
6072 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
6073 NumElements: WideResVT.getVectorNumElements());
6074
6075 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6076 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
6077 } else {
6078 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
6079 WideResVT = EVT::getVectorVT(
6080 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
6081 NumElements: WideOvVT.getVectorNumElements());
6082
6083 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6084 SDValue Poison = DAG.getPOISON(VT: WideResVT);
6085
6086 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6087 N2: N->getOperand(Num: 0), N3: Zero);
6088 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
6089 N2: N->getOperand(Num: 1), N3: Zero);
6090 }
6091
6092 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
6093 SDNode *WideNode = DAG.getNode(
6094 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
6095
6096 // Replace the other vector result not being explicitly widened here.
6097 unsigned OtherNo = 1 - ResNo;
6098 EVT OtherVT = N->getValueType(ResNo: OtherNo);
6099 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
6100 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
6101 } else {
6102 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
6103 SDValue OtherVal = DAG.getNode(
6104 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
6105 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
6106 }
6107
6108 return SDValue(WideNode, ResNo);
6109}
6110
6111SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
6112 LLVMContext &Ctx = *DAG.getContext();
6113 SDValue InOp = N->getOperand(Num: 0);
6114 SDLoc DL(N);
6115
6116 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
6117 ElementCount WidenEC = WidenVT.getVectorElementCount();
6118
6119 EVT InVT = InOp.getValueType();
6120
6121 unsigned Opcode = N->getOpcode();
6122 const SDNodeFlags Flags = N->getFlags();
6123
6124 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
6125 // not equal that of WidenVT.
6126 if (N->getOpcode() == ISD::ZERO_EXTEND &&
6127 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
6128 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
6129 WidenVT.getScalarSizeInBits()) {
6130 InOp = ZExtPromotedInteger(Op: InOp);
6131 InVT = InOp.getValueType();
6132 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
6133 Opcode = ISD::TRUNCATE;
6134 }
6135
6136 EVT InEltVT = InVT.getVectorElementType();
6137 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
6138 ElementCount InVTEC = InVT.getVectorElementCount();
6139
6140 // Helper to build node with all scalar trailing operands.
6141 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6142 if (N->getNumOperands() == 1)
6143 return DAG.getNode(Opcode, DL, VT, Operand: Op, Flags);
6144 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6145 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
6146 N4: N->getOperand(Num: 3), Flags);
6147 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), Flags);
6148 };
6149
6150 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6151 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6152 InVT = InOp.getValueType();
6153 InVTEC = InVT.getVectorElementCount();
6154 if (InVTEC == WidenEC)
6155 return MakeConvertNode(WidenVT, InOp);
6156 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6157 // If both input and result vector types are of same width, extend
6158 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6159 // accepts fewer elements in the result than in the input.
6160 if (Opcode == ISD::ANY_EXTEND)
6161 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6162 if (Opcode == ISD::SIGN_EXTEND)
6163 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6164 if (Opcode == ISD::ZERO_EXTEND)
6165 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6166 }
6167
6168 // For TRUNCATE, try to widen using the legal EC of the input type instead
6169 // if the legalisation action for that intermediate type is not widening.
6170 // E.g. for trunc nxv1i64 -> nxv1i8 where
6171 // - nxv1i64 input gets widened to nxv2i64
6172 // - nxv1i8 output gets widened to nxv16i8
6173 // Then one can try widening the result to nxv2i8 (instead of going all the
6174 // way to nxv16i8) if this later allows type promotion.
6175 EVT MidResVT =
6176 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
6177 if (N->getOpcode() == ISD::TRUNCATE &&
6178 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
6179 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
6180 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
6181 }
6182 }
6183
6184 if (TLI.isTypeLegal(VT: InWidenVT)) {
6185 // Because the result and the input are different vector types, widening
6186 // the result could create a legal type but widening the input might make
6187 // it an illegal type that might lead to repeatedly splitting the input
6188 // and then widening it. To avoid this, we widen the input only if
6189 // it results in a legal type.
6190 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
6191 // Widen the input and call convert on the widened input vector.
6192 unsigned NumConcat =
6193 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6194 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
6195 Ops[0] = InOp;
6196 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
6197 return MakeConvertNode(WidenVT, InVec);
6198 }
6199
6200 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
6201 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
6202 // Extract the input and convert the shorten input vector.
6203 return MakeConvertNode(WidenVT, InVal);
6204 }
6205 }
6206
6207 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6208 EVT EltVT = WidenVT.getVectorElementType();
6209 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
6210 // Use the original element count so we don't do more scalar opts than
6211 // necessary.
6212 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6213 for (unsigned i=0; i < MinElts; ++i) {
6214 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6215 Ops[i] = MakeConvertNode(EltVT, Val);
6216 }
6217
6218 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6219}
6220
6221SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6222 SDLoc dl(N);
6223 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6224 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6225
6226 SDValue Src = N->getOperand(Num: 0);
6227 EVT SrcVT = Src.getValueType();
6228
6229 // Also widen the input.
6230 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6231 Src = GetWidenedVector(Op: Src);
6232 SrcVT = Src.getValueType();
6233 }
6234
6235 // Input and output not widened to the same size, give up.
6236 if (WidenNumElts != SrcVT.getVectorElementCount())
6237 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6238
6239 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
6240}
6241
6242SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6243 SDLoc dl(N);
6244 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6245 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6246
6247 SDValue Src = N->getOperand(Num: 0);
6248 EVT SrcVT = Src.getValueType();
6249
6250 // Also widen the input.
6251 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6252 Src = GetWidenedVector(Op: Src);
6253 SrcVT = Src.getValueType();
6254 }
6255
6256 // Input and output not widened to the same size, give up.
6257 if (WidenNumElts != SrcVT.getVectorElementCount())
6258 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6259
6260 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
6261}
6262
6263SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6264 SDValue InOp = N->getOperand(Num: 1);
6265 SDLoc DL(N);
6266 SmallVector<SDValue, 4> NewOps(N->ops());
6267
6268 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6269 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6270
6271 EVT InVT = InOp.getValueType();
6272 EVT InEltVT = InVT.getVectorElementType();
6273
6274 unsigned Opcode = N->getOpcode();
6275
6276 // FIXME: Optimizations need to be implemented here.
6277
6278 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6279 EVT EltVT = WidenVT.getVectorElementType();
6280 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
6281 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
6282 SmallVector<SDValue, 32> OpChains;
6283 // Use the original element count so we don't do more scalar opts than
6284 // necessary.
6285 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6286 for (unsigned i=0; i < MinElts; ++i) {
6287 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6288 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
6289 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6290 }
6291 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
6292 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6293
6294 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6295}
6296
6297SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6298 unsigned Opcode = N->getOpcode();
6299 SDValue InOp = N->getOperand(Num: 0);
6300 SDLoc DL(N);
6301
6302 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6303 EVT WidenSVT = WidenVT.getVectorElementType();
6304 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6305
6306 EVT InVT = InOp.getValueType();
6307 EVT InSVT = InVT.getVectorElementType();
6308 unsigned InVTNumElts = InVT.getVectorNumElements();
6309
6310 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6311 InOp = GetWidenedVector(Op: InOp);
6312 InVT = InOp.getValueType();
6313 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6314 switch (Opcode) {
6315 case ISD::ANY_EXTEND_VECTOR_INREG:
6316 case ISD::SIGN_EXTEND_VECTOR_INREG:
6317 case ISD::ZERO_EXTEND_VECTOR_INREG:
6318 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
6319 }
6320 }
6321 }
6322
6323 // Unroll, extend the scalars and rebuild the vector.
6324 SmallVector<SDValue, 16> Ops;
6325 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
6326 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
6327 switch (Opcode) {
6328 case ISD::ANY_EXTEND_VECTOR_INREG:
6329 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
6330 break;
6331 case ISD::SIGN_EXTEND_VECTOR_INREG:
6332 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
6333 break;
6334 case ISD::ZERO_EXTEND_VECTOR_INREG:
6335 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
6336 break;
6337 default:
6338 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6339 }
6340 Ops.push_back(Elt: Val);
6341 }
6342
6343 while (Ops.size() != WidenNumElts)
6344 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
6345
6346 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6347}
6348
6349SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6350 // If this is an FCOPYSIGN with same input types, we can treat it as a
6351 // normal (can trap) binary op.
6352 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
6353 return WidenVecRes_BinaryCanTrap(N);
6354
6355 // If the types are different, fall back to unrolling.
6356 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6357 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6358}
6359
6360/// Result and first source operand are different scalar types, but must have
6361/// the same number of elements. There is an additional control argument which
6362/// should be passed through unchanged.
6363SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6364 SDValue FpValue = N->getOperand(Num: 0);
6365 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6366 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6367 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6368 SDValue Arg = GetWidenedVector(Op: FpValue);
6369 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6370 Flags: N->getFlags());
6371}
6372
6373SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6374 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6375 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6376 SDValue RHS = N->getOperand(Num: 1);
6377 EVT ExpVT = RHS.getValueType();
6378 SDValue ExpOp = RHS;
6379 if (ExpVT.isVector()) {
6380 EVT WideExpVT = WidenVT.changeVectorElementType(
6381 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6382 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6383 }
6384
6385 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6386}
6387
6388SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6389 // Unary op widening.
6390 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6391 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6392 if (N->getNumOperands() == 1)
6393 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6394 assert(N->getOpcode() == ISD::AssertNoFPClass && "unexpected opcode");
6395 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1),
6396 Flags: N->getFlags());
6397}
6398
6399SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6400 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6401 EVT ExtVT = EVT::getVectorVT(
6402 Context&: *DAG.getContext(),
6403 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(),
6404 EC: WidenVT.getVectorElementCount());
6405 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6406 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6407 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6408}
6409
6410SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6411 unsigned ResNo) {
6412 EVT VT0 = N->getValueType(ResNo: 0);
6413 EVT VT1 = N->getValueType(ResNo: 1);
6414
6415 assert(VT0.isVector() && VT1.isVector() &&
6416 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6417 "expected both results to be vectors of matching element count");
6418
6419 LLVMContext &Ctx = *DAG.getContext();
6420 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6421
6422 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6423 ElementCount WidenEC = WidenVT.getVectorElementCount();
6424
6425 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6426 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6427
6428 SDNode *WidenNode =
6429 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6430 .getNode();
6431
6432 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6433 return SDValue(WidenNode, ResNo);
6434}
6435
6436SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6437 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6438 return GetWidenedVector(Op: WidenVec);
6439}
6440
6441SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6442 SDLoc DL(N);
6443 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6444 ElementCount WidenEC = WidenVT.getVectorElementCount();
6445 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6446
6447 // The source has the same number of elements as the result, so widen it to
6448 // match WidenVT. It only lives in the widened-vector map if it is itself
6449 // widened; otherwise pad it up to the widened element count
6450 // when it is illegal.
6451 SDValue InOp = N->getOperand(Num: 0);
6452 EVT InVT = InOp.getValueType();
6453 TargetLowering::LegalizeTypeAction InAction = getTypeAction(VT: InVT);
6454 if (InAction == TargetLowering::TypeWidenVector) {
6455 InOp = GetWidenedVector(Op: InOp);
6456 } else if (InAction != TargetLowering::TypeLegal) {
6457 EVT InWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6458 VT: InVT.getVectorElementType(), EC: WidenEC);
6459 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: InWidenVT), SubVec: InOp, Idx: 0);
6460 }
6461
6462 return DAG.getAddrSpaceCast(
6463 dl: DL, VT: WidenVT, Ptr: InOp, SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6464 DestAS: AddrSpaceCastN->getDestAddressSpace(), Flags: AddrSpaceCastN->getFlags());
6465}
6466
6467SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6468 SDValue InOp = N->getOperand(Num: 0);
6469 EVT InVT = InOp.getValueType();
6470 EVT VT = N->getValueType(ResNo: 0);
6471 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6472 SDLoc dl(N);
6473
6474 switch (getTypeAction(VT: InVT)) {
6475 case TargetLowering::TypeLegal:
6476 break;
6477 case TargetLowering::TypeScalarizeScalableVector:
6478 report_fatal_error(reason: "scalarization of scalable vectors is not supported");
6479 case TargetLowering::TypePromoteInteger: {
6480 // If the incoming type is a vector that is being promoted, then
6481 // we know that the elements are arranged differently and that we
6482 // must perform the conversion using a stack slot.
6483 if (InVT.isVector())
6484 break;
6485
6486 // If the InOp is promoted to the same size, convert it. Otherwise,
6487 // fall out of the switch and widen the promoted input.
6488 SDValue NInOp = GetPromotedInteger(Op: InOp);
6489 EVT NInVT = NInOp.getValueType();
6490 if (WidenVT.bitsEq(VT: NInVT)) {
6491 // For big endian targets we need to shift the input integer or the
6492 // interesting bits will end up at the wrong place.
6493 if (DAG.getDataLayout().isBigEndian()) {
6494 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6495 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6496 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6497 }
6498 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6499 }
6500 InOp = NInOp;
6501 InVT = NInVT;
6502 break;
6503 }
6504 case TargetLowering::TypeSoftenFloat:
6505 case TargetLowering::TypeSoftPromoteHalf:
6506 case TargetLowering::TypeExpandInteger:
6507 case TargetLowering::TypeExpandFloat:
6508 case TargetLowering::TypeScalarizeVector:
6509 case TargetLowering::TypeSplitVector:
6510 break;
6511 case TargetLowering::TypeWidenVector:
6512 // If the InOp is widened to the same size, convert it. Otherwise, fall
6513 // out of the switch and widen the widened input.
6514 InOp = GetWidenedVector(Op: InOp);
6515 InVT = InOp.getValueType();
6516 if (WidenVT.bitsEq(VT: InVT))
6517 // The input widens to the same size. Convert to the widen value.
6518 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6519 break;
6520 }
6521
6522 unsigned WidenSize = WidenVT.getSizeInBits();
6523 unsigned InSize = InVT.getSizeInBits();
6524 unsigned InScalarSize = InVT.getScalarSizeInBits();
6525 // x86mmx is not an acceptable vector element type, so don't try.
6526 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6527 // Determine new input vector type. The new input vector type will use
6528 // the same element type (if its a vector) or use the input type as a
6529 // vector. It is the same size as the type to widen to.
6530 EVT NewInVT;
6531 unsigned NewNumParts = WidenSize / InSize;
6532 if (InVT.isVector()) {
6533 EVT InEltVT = InVT.getVectorElementType();
6534 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6535 NumElements: WidenSize / InEltVT.getSizeInBits());
6536 } else {
6537 // For big endian systems, using the promoted input scalar type
6538 // to produce the scalar_to_vector would put the desired bits into
6539 // the least significant byte(s) of the wider element zero. This
6540 // will mean that the users of the result vector are using incorrect
6541 // bits. Use the original input type instead. Although either input
6542 // type can be used on little endian systems, for consistency we
6543 // use the original type there as well.
6544 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6545 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6546 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6547 }
6548
6549 if (TLI.isTypeLegal(VT: NewInVT)) {
6550 SDValue NewVec;
6551 if (InVT.isVector()) {
6552 // Because the result and the input are different vector types, widening
6553 // the result could create a legal type but widening the input might
6554 // make it an illegal type that might lead to repeatedly splitting the
6555 // input and then widening it. To avoid this, we widen the input only if
6556 // it results in a legal type.
6557 if (WidenSize % InSize == 0) {
6558 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6559 Ops[0] = InOp;
6560
6561 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6562 } else {
6563 SmallVector<SDValue, 16> Ops;
6564 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6565 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6566 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6567
6568 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6569 }
6570 } else {
6571 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6572 }
6573 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6574 }
6575 }
6576
6577 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6578}
6579
6580SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6581 return DAG.getNode(
6582 Opcode: N->getOpcode(), DL: SDLoc(N),
6583 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6584 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6585}
6586
6587SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6588 SDLoc dl(N);
6589 // Build a vector with poison for the new nodes.
6590 EVT VT = N->getValueType(ResNo: 0);
6591
6592 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6593 // type. The POISONs need to have the same type as the existing operands.
6594 EVT EltVT = N->getOperand(Num: 0).getValueType();
6595 unsigned NumElts = VT.getVectorNumElements();
6596
6597 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6598 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6599
6600 SmallVector<SDValue, 16> NewOps(N->ops());
6601 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6602 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6603
6604 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6605}
6606
6607SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6608 EVT InVT = N->getOperand(Num: 0).getValueType();
6609 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6610 SDLoc dl(N);
6611 unsigned NumOperands = N->getNumOperands();
6612
6613 bool InputWidened = false; // Indicates we need to widen the input.
6614 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6615 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6616 unsigned NumInElts = InVT.getVectorMinNumElements();
6617 if (WidenNumElts % NumInElts == 0) {
6618 // Add undef vectors to widen to correct length.
6619 unsigned NumConcat = WidenNumElts / NumInElts;
6620 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6621 SmallVector<SDValue, 16> Ops(NumConcat);
6622 for (unsigned i=0; i < NumOperands; ++i)
6623 Ops[i] = N->getOperand(Num: i);
6624 for (unsigned i = NumOperands; i != NumConcat; ++i)
6625 Ops[i] = UndefVal;
6626 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6627 }
6628 } else {
6629 InputWidened = true;
6630 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6631 // The inputs and the result are widen to the same value.
6632 unsigned i;
6633 for (i=1; i < NumOperands; ++i)
6634 if (!N->getOperand(Num: i).isUndef())
6635 break;
6636
6637 if (i == NumOperands)
6638 // Everything but the first operand is an UNDEF so just return the
6639 // widened first operand.
6640 return GetWidenedVector(Op: N->getOperand(Num: 0));
6641
6642 if (NumOperands == 2) {
6643 assert(!WidenVT.isScalableVector() &&
6644 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6645 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6646 unsigned NumInElts = InVT.getVectorNumElements();
6647
6648 // Replace concat of two operands with a shuffle.
6649 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6650 for (unsigned i = 0; i < NumInElts; ++i) {
6651 MaskOps[i] = i;
6652 MaskOps[i + NumInElts] = i + WidenNumElts;
6653 }
6654 return DAG.getVectorShuffle(VT: WidenVT, dl,
6655 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6656 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6657 Mask: MaskOps);
6658 }
6659 }
6660 }
6661
6662 if (WidenVT.isScalableVector()) {
6663 SDValue WideVec = DAG.getPOISON(VT: WidenVT);
6664 unsigned NumInElts = InVT.getVectorMinNumElements();
6665 for (unsigned I = 0; I < NumOperands; ++I)
6666 WideVec =
6667 DAG.getInsertSubvector(DL: dl, Vec: WideVec, SubVec: N->getOperand(Num: I), Idx: I * NumInElts);
6668 return WideVec;
6669 }
6670
6671 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6672 unsigned NumInElts = InVT.getVectorNumElements();
6673
6674 // Fall back to use extracts and build vector.
6675 EVT EltVT = WidenVT.getVectorElementType();
6676 SmallVector<SDValue, 16> Ops(WidenNumElts);
6677 unsigned Idx = 0;
6678 for (unsigned i=0; i < NumOperands; ++i) {
6679 SDValue InOp = N->getOperand(Num: i);
6680 if (InputWidened)
6681 InOp = GetWidenedVector(Op: InOp);
6682 for (unsigned j = 0; j < NumInElts; ++j)
6683 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6684 }
6685 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6686 for (; Idx < WidenNumElts; ++Idx)
6687 Ops[Idx] = UndefVal;
6688 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6689}
6690
6691SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6692 EVT VT = N->getValueType(ResNo: 0);
6693 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6694 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6695 SDValue InOp2 = N->getOperand(Num: 1);
6696 SDValue Idx = N->getOperand(Num: 2);
6697 SDLoc dl(N);
6698 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6699}
6700
6701SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6702 EVT VT = N->getValueType(ResNo: 0);
6703 EVT EltVT = VT.getVectorElementType();
6704 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6705 SDValue InOp = N->getOperand(Num: 0);
6706 SDValue Idx = N->getOperand(Num: 1);
6707 SDLoc dl(N);
6708
6709 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6710 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6711 InOp = GetWidenedVector(Op: InOp);
6712
6713 EVT InVT = InOp.getValueType();
6714
6715 // Check if we can just return the input vector after widening.
6716 uint64_t IdxVal = Idx->getAsZExtVal();
6717 if (IdxVal == 0 && InVT == WidenVT)
6718 return InOp;
6719
6720 // Check if we can extract from the vector.
6721 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6722 unsigned InNumElts = InVT.getVectorMinNumElements();
6723 unsigned VTNumElts = VT.getVectorMinNumElements();
6724 assert(IdxVal % VTNumElts == 0 &&
6725 "Expected Idx to be a multiple of subvector minimum vector length");
6726 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6727 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6728
6729 if (VT.isScalableVector()) {
6730 // Try to split the operation up into smaller extracts and concat the
6731 // results together, e.g.
6732 // nxv6i64 extract_subvector(nxv12i64, 6)
6733 // <->
6734 // nxv8i64 concat(
6735 // nxv2i64 extract_subvector(nxv16i64, 6)
6736 // nxv2i64 extract_subvector(nxv16i64, 8)
6737 // nxv2i64 extract_subvector(nxv16i64, 10)
6738 // undef)
6739 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6740 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6741 "down type's element count");
6742 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6743 EC: ElementCount::getScalable(MinVal: GCD));
6744 // Avoid recursion around e.g. nxv1i8.
6745 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6746 SmallVector<SDValue> Parts;
6747 unsigned I = 0;
6748 for (; I < VTNumElts / GCD; ++I)
6749 Parts.push_back(
6750 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6751 for (; I < WidenNumElts / GCD; ++I)
6752 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6753
6754 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6755 }
6756
6757 // Fallback to extracting through memory.
6758
6759 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6760 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6761 MachineFunction &MF = DAG.getMachineFunction();
6762 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6763 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6764
6765 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6766 PtrInfo, F: MachineMemOperand::MOStore,
6767 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6768 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6769 PtrInfo, F: MachineMemOperand::MOLoad,
6770 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6771
6772 // Write out the input vector.
6773 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6774
6775 // Build a mask to match the length of the non-widened result.
6776 SDValue Mask =
6777 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6778
6779 // Read back the sub-vector setting the remaining lanes to poison.
6780 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6781 return DAG.getMaskedLoad(
6782 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6783 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6784 }
6785
6786 // We could try widening the input to the right length but for now, extract
6787 // the original elements, fill the rest with undefs and build a vector.
6788 SmallVector<SDValue, 16> Ops(WidenNumElts);
6789 unsigned i;
6790 for (i = 0; i < VTNumElts; ++i)
6791 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6792
6793 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6794 for (; i < WidenNumElts; ++i)
6795 Ops[i] = UndefVal;
6796 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6797}
6798
6799SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6800 SDValue InOp = ModifyToType(
6801 InOp: N->getOperand(Num: 0),
6802 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6803 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6804 N2: N->getOperand(Num: 1));
6805}
6806
6807SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6808 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6809 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6810 VT: InOp.getValueType(), N1: InOp,
6811 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6812}
6813
6814/// Either return the same load or provide appropriate casts
6815/// from the load and return that.
6816static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6817 TypeSize LdWidth, TypeSize FirstVTWidth,
6818 SDLoc dl, SelectionDAG &DAG) {
6819 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6820 "Load width must be less than or equal to first value type width");
6821 TypeSize WidenWidth = WidenVT.getSizeInBits();
6822 if (!FirstVT.isVector()) {
6823 unsigned NumElts =
6824 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6825 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6826 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6827 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6828 }
6829 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6830 return LdOp;
6831}
6832
6833/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6834/// widened value so it can be issued in a single atomic store.
6835static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6836 TypeSize FirstVTWidth, const SDLoc &dl,
6837 SelectionDAG &DAG) {
6838 TypeSize WidenWidth = WidenVT.getSizeInBits();
6839 if (!FirstVT.isVector()) {
6840 unsigned NumElts =
6841 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6842 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6843 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: StVal);
6844 return DAG.getExtractVectorElt(DL: dl, VT: FirstVT, Vec: VecOp, Idx: 0);
6845 }
6846 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6847 return StVal;
6848}
6849
6850static std::optional<EVT> findMemType(SelectionDAG &DAG,
6851 const TargetLowering &TLI, unsigned Width,
6852 EVT WidenVT, unsigned Align,
6853 unsigned WidenEx);
6854
6855SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6856 EVT WidenVT =
6857 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6858 EVT LdVT = LD->getMemoryVT();
6859 SDLoc dl(LD);
6860
6861 // Load information
6862 SDValue Chain = LD->getChain();
6863 SDValue BasePtr = LD->getBasePtr();
6864
6865 TypeSize LdWidth = LdVT.getSizeInBits();
6866 TypeSize WidenWidth = WidenVT.getSizeInBits();
6867 TypeSize WidthDiff = WidenWidth - LdWidth;
6868
6869 // Find the vector type that can load from.
6870 std::optional<EVT> FirstVT =
6871 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6872 WidenEx: WidthDiff.getKnownMinValue());
6873
6874 if (!FirstVT)
6875 return SDValue();
6876
6877 SmallVector<EVT, 8> MemVTs;
6878 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6879
6880 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6881 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6882
6883 // Load the element with one instruction.
6884 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6885 FirstVTWidth, dl, DAG);
6886
6887 // Modified the chain - switch anything that used the old chain to use
6888 // the new one.
6889 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6890 return Result;
6891}
6892
6893SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6894 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6895 ISD::LoadExtType ExtType = LD->getExtensionType();
6896
6897 // A vector must always be stored in memory as-is, i.e. without any padding
6898 // between the elements, since various code depend on it, e.g. in the
6899 // handling of a bitcast of a vector type to int, which may be done with a
6900 // vector store followed by an integer load. A vector that does not have
6901 // elements that are byte-sized must therefore be stored as an integer
6902 // built out of the extracted vector elements.
6903 if (!LD->getMemoryVT().isByteSized()) {
6904 SDValue Value, NewChain;
6905 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6906 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6907 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6908 return SDValue();
6909 }
6910
6911 // Generate a vector-predicated load if it is custom/legal on the target. To
6912 // avoid possible recursion, only do this if the widened mask type is legal.
6913 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6914 // removed from the IR by the ExpandVectorPredication pass but we're
6915 // reintroducing them here.
6916 EVT VT = LD->getValueType(ResNo: 0);
6917 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6918 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6919
6920 if (ExtType == ISD::NON_EXTLOAD &&
6921 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6922 TLI.isTypeLegal(VT: WideMaskVT)) {
6923 SDLoc DL(N);
6924 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6925 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6926 EC: VT.getVectorElementCount());
6927 SDValue NewLoad =
6928 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6929 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6930 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6931
6932 // Modified the chain - switch anything that used the old chain to use
6933 // the new one.
6934 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6935
6936 return NewLoad;
6937 }
6938
6939 SDValue Result;
6940 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6941 if (ExtType != ISD::NON_EXTLOAD)
6942 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6943 else
6944 Result = GenWidenVectorLoads(LdChain, LD);
6945
6946 if (Result) {
6947 // If we generate a single load, we can use that for the chain. Otherwise,
6948 // build a factor node to remember the multiple loads are independent and
6949 // chain to that.
6950 SDValue NewChain;
6951 if (LdChain.size() == 1)
6952 NewChain = LdChain[0];
6953 else
6954 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6955
6956 // Modified the chain - switch anything that used the old chain to use
6957 // the new one.
6958 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6959
6960 return Result;
6961 }
6962
6963 if (VT.isVector()) {
6964 // If all else fails replace the load with a wide masked load.
6965 SDLoc DL(N);
6966 SDValue Mask =
6967 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6968
6969 SDValue NewLoad = DAG.getMaskedLoad(
6970 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6971 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6972 AM: LD->getAddressingMode(), LD->getExtensionType());
6973
6974 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6975 return NewLoad;
6976 }
6977
6978 report_fatal_error(reason: "Unable to widen vector load");
6979}
6980
6981SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6982 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6983 SDValue Mask = N->getMask();
6984 SDValue EVL = N->getVectorLength();
6985 ISD::LoadExtType ExtType = N->getExtensionType();
6986 SDLoc dl(N);
6987
6988 // The mask should be widened as well
6989 assert(getTypeAction(Mask.getValueType()) ==
6990 TargetLowering::TypeWidenVector &&
6991 "Unable to widen binary VP op");
6992 Mask = GetWidenedVector(Op: Mask);
6993 assert(Mask.getValueType().getVectorElementCount() ==
6994 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6995 .getVectorElementCount() &&
6996 "Unable to widen vector load");
6997
6998 SDValue Res =
6999 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
7000 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
7001 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
7002 // Legalize the chain result - switch anything that used the old chain to
7003 // use the new one.
7004 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7005 return Res;
7006}
7007
7008SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
7009 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7010 SDValue Mask = N->getMask();
7011 SDValue EVL = N->getVectorLength();
7012 SDLoc dl(N);
7013
7014 // The mask should be widened as well
7015 assert(getTypeAction(Mask.getValueType()) ==
7016 TargetLowering::TypeWidenVector &&
7017 "Unable to widen binary VP op");
7018 Mask = GetWidenedVector(Op: Mask);
7019 assert(Mask.getValueType().getVectorElementCount() ==
7020 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
7021 .getVectorElementCount() &&
7022 "Unable to widen vector load");
7023
7024 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
7025 Mask, EVL, MMO: N->getMemOperand());
7026 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7027 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
7028 return Res;
7029}
7030
7031SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
7032 SDLoc DL(N);
7033
7034 // The mask should be widened as well
7035 SDValue Mask = N->getMask();
7036 assert(getTypeAction(Mask.getValueType()) ==
7037 TargetLowering::TypeWidenVector &&
7038 "Unable to widen VP strided load");
7039 Mask = GetWidenedVector(Op: Mask);
7040
7041 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7042 assert(Mask.getValueType().getVectorElementCount() ==
7043 WidenVT.getVectorElementCount() &&
7044 "Data and mask vectors should have the same number of elements");
7045
7046 SDValue Res = DAG.getStridedLoadVP(
7047 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
7048 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
7049 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
7050 IsExpanding: N->isExpandingLoad());
7051
7052 // Legalize the chain result - switch anything that used the old chain to
7053 // use the new one.
7054 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7055 return Res;
7056}
7057
7058SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
7059 SDValue Vec = N->getOperand(Num: 0);
7060 SDValue Mask = N->getOperand(Num: 1);
7061 SDValue Passthru = N->getOperand(Num: 2);
7062 EVT WideVecVT =
7063 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
7064 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7065 VT: Mask.getValueType().getVectorElementType(),
7066 EC: WideVecVT.getVectorElementCount());
7067
7068 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
7069 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
7070 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
7071 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
7072 N2: WideMask, N3: WidePassthru);
7073}
7074
7075SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
7076 EVT VT = N->getValueType(ResNo: 0);
7077 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7078 SDValue Mask = N->getMask();
7079 EVT MaskVT = Mask.getValueType();
7080 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7081 ISD::LoadExtType ExtType = N->getExtensionType();
7082 SDLoc dl(N);
7083
7084 EVT WideMaskVT =
7085 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
7086 EC: WidenVT.getVectorElementCount());
7087
7088 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
7089 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
7090 TLI.isTypeLegal(VT: WideMaskVT) &&
7091 // If there is a passthru, we shouldn't use vp.load. However,
7092 // type legalizer will struggle on masked.load with
7093 // scalable vectors, so for scalable vectors, we still use vp.load
7094 // but manually merge the load result with the passthru using vp.select.
7095 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
7096 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
7097 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
7098 EC: VT.getVectorElementCount());
7099 SDValue NewLoad =
7100 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
7101 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
7102 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
7103 SDValue NewVal = NewLoad;
7104
7105 // Manually merge with vselect
7106 if (!N->getPassThru()->isUndef()) {
7107 assert(WidenVT.isScalableVector());
7108 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
7109 // The lanes past EVL are poison.
7110 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
7111 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
7112 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
7113 }
7114
7115 // Modified the chain - switch anything that used the old chain to use
7116 // the new one.
7117 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
7118
7119 return NewVal;
7120 }
7121
7122 // The mask should be widened as well
7123 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7124
7125 SDValue Res = DAG.getMaskedLoad(
7126 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
7127 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
7128 ExtType, IsExpanding: N->isExpandingLoad());
7129 // Legalize the chain result - switch anything that used the old chain to
7130 // use the new one.
7131 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7132 return Res;
7133}
7134
7135SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
7136
7137 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7138 SDValue Mask = N->getMask();
7139 EVT MaskVT = Mask.getValueType();
7140 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7141 SDValue Scale = N->getScale();
7142 ElementCount WideEC = WideVT.getVectorElementCount();
7143 SDLoc dl(N);
7144
7145 // The mask should be widened as well
7146 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7147 VT: MaskVT.getVectorElementType(), EC: WideEC);
7148 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7149
7150 // Widen the Index operand
7151 SDValue Index = N->getIndex();
7152 EVT WideIndexVT = EVT::getVectorVT(
7153 Context&: *DAG.getContext(), VT: Index.getValueType().getScalarType(), EC: WideEC);
7154 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7155 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7156 Scale };
7157
7158 // Widen the MemoryType
7159 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7160 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7161 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
7162 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
7163 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
7164
7165 // Legalize the chain result - switch anything that used the old chain to
7166 // use the new one.
7167 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7168 return Res;
7169}
7170
7171SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7172 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7173 SDValue Mask = N->getMask();
7174 SDValue Scale = N->getScale();
7175 ElementCount WideEC = WideVT.getVectorElementCount();
7176 SDLoc dl(N);
7177
7178 SDValue Index = GetWidenedVector(Op: N->getIndex());
7179 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7180 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7181 Mask = GetWidenedMask(Mask, EC: WideEC);
7182
7183 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7184 Mask, N->getVectorLength()};
7185 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
7186 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
7187
7188 // Legalize the chain result - switch anything that used the old chain to
7189 // use the new one.
7190 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7191 return Res;
7192}
7193
7194SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7195 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7196 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
7197}
7198
7199// Return true is this is a SETCC node or a strict version of it.
7200static inline bool isSETCCOp(unsigned Opcode) {
7201 switch (Opcode) {
7202 case ISD::SETCC:
7203 case ISD::STRICT_FSETCC:
7204 case ISD::STRICT_FSETCCS:
7205 return true;
7206 }
7207 return false;
7208}
7209
7210// Return true if this is a node that could have two SETCCs as operands.
7211static inline bool isLogicalMaskOp(unsigned Opcode) {
7212 switch (Opcode) {
7213 case ISD::AND:
7214 case ISD::OR:
7215 case ISD::XOR:
7216 return true;
7217 }
7218 return false;
7219}
7220
7221// If N is a SETCC or a strict variant of it, return the type
7222// of the compare operands.
7223static inline EVT getSETCCOperandType(SDValue N) {
7224 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7225 return N->getOperand(Num: OpNo).getValueType();
7226}
7227
7228// This is used just for the assert in convertMask(). Check that this either
7229// a SETCC or a previously handled SETCC by convertMask().
7230#ifndef NDEBUG
7231static inline bool isSETCCorConvertedSETCC(SDValue N) {
7232 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7233 N = N.getOperand(0);
7234 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7235 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7236 if (!N->getOperand(i)->isUndef())
7237 return false;
7238 N = N.getOperand(0);
7239 }
7240
7241 if (N.getOpcode() == ISD::TRUNCATE)
7242 N = N.getOperand(0);
7243 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7244 N = N.getOperand(0);
7245
7246 if (isLogicalMaskOp(N.getOpcode()))
7247 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7248 isSETCCorConvertedSETCC(N.getOperand(1));
7249
7250 return (isSETCCOp(N.getOpcode()) ||
7251 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
7252}
7253#endif
7254
7255// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7256// to ToMaskVT if needed with vector extension or truncation.
7257SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7258 EVT ToMaskVT) {
7259 // Called from convertMaskTree for SETCC leaf nodes. Re-creates the SETCC with
7260 // result type MaskVT, then sign-extends/truncates and pads to ToMaskVT.
7261 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7262
7263 // Make a new Mask node, with a legal result VT.
7264 SDValue Mask;
7265 SmallVector<SDValue, 4> Ops;
7266 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7267 Ops.push_back(Elt: InMask->getOperand(Num: i));
7268 if (InMask->isStrictFPOpcode()) {
7269 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
7270 ResultTys: { MaskVT, MVT::Other }, Ops);
7271 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
7272 }
7273 else
7274 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
7275 Flags: InMask->getFlags());
7276
7277 return adjustMaskToType(Mask, ToMaskVT);
7278}
7279
7280// Adjust element width (sign-extend/truncate) and element count
7281// (extract/concat) of Mask to match ToMaskVT.
7282SDValue DAGTypeLegalizer::adjustMaskToType(SDValue Mask, EVT ToMaskVT) {
7283 LLVMContext &Ctx = *DAG.getContext();
7284 EVT MaskVT = Mask.getValueType();
7285 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7286 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7287 if (MaskScalarBits < ToMaskScalBits) {
7288 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7289 NumElements: MaskVT.getVectorNumElements());
7290 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
7291 } else if (MaskScalarBits > ToMaskScalBits) {
7292 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7293 NumElements: MaskVT.getVectorNumElements());
7294 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
7295 }
7296
7297 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7298 ToMaskVT.getScalarSizeInBits() &&
7299 "Mask should have the right element size by now.");
7300
7301 // Adjust Mask to the right number of elements.
7302 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
7303 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7304 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
7305 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7306 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7307 EVT SubVT = Mask->getValueType(ResNo: 0);
7308 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
7309 SubOps[0] = Mask;
7310 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
7311 }
7312
7313 assert((Mask->getValueType(0) == ToMaskVT) &&
7314 "A mask of ToMaskVT should have been produced by now.");
7315
7316 return Mask;
7317}
7318
7319// Adjust both operands to a common intermediate mask type, picking a scalar
7320// width that minimizes extend/truncate overhead given the final target ToVT.
7321EVT DAGTypeLegalizer::unifyMaskTypes(SDValue &Op0, bool IsOpLenient0,
7322 SDValue &Op1, bool IsOpLenient1,
7323 EVT ToVT) {
7324 assert(Op0.getValueType().getVectorNumElements() ==
7325 Op1.getValueType().getVectorNumElements() &&
7326 "unifyMaskTypes only handles scalar width differences");
7327
7328 // If only one of the operands lenient type-wise, we can simply
7329 // adjust its type to the other operand's type assuming that this
7330 // adjustment can be folded away.
7331 //
7332 // NOTE: We essentially rely on the fact that further optimizations
7333 // do spot redundant casts in "lenient" cases. If at some
7334 // point we decide that we want to do better here, we can
7335 // postpone converting lenient sub-trees right away and postpone
7336 // it to the moment when we know the best fitting integer type
7337 // to materialize them, and do it there.
7338 if (IsOpLenient0 != IsOpLenient1) {
7339 SDValue *LenientOp, *NonLenientOp;
7340 if (IsOpLenient0) {
7341 LenientOp = &Op0;
7342 NonLenientOp = &Op1;
7343 } else {
7344 LenientOp = &Op1;
7345 NonLenientOp = &Op0;
7346 }
7347 EVT OpVT = NonLenientOp->getValueType();
7348 *LenientOp = adjustMaskToType(Mask: *LenientOp, ToMaskVT: OpVT);
7349 return OpVT;
7350 }
7351
7352 unsigned Bits0 = Op0.getScalarValueSizeInBits();
7353 unsigned Bits1 = Op1.getScalarValueSizeInBits();
7354 unsigned NarrowBits = std::min(a: Bits0, b: Bits1);
7355 unsigned WideBits = std::max(a: Bits0, b: Bits1);
7356 unsigned ToBits = ToVT.getScalarSizeInBits();
7357 unsigned IntBits = NarrowBits == WideBits ? NarrowBits
7358 : ToBits >= WideBits ? WideBits
7359 : ToBits <= NarrowBits ? NarrowBits
7360 : ToBits;
7361 EVT OpVT = Op0.getValueType().changeVectorElementType(
7362 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntBits));
7363 Op0 = adjustMaskToType(Mask: Op0, ToMaskVT: OpVT);
7364 Op1 = adjustMaskToType(Mask: Op1, ToMaskVT: OpVT);
7365 return OpVT;
7366}
7367
7368std::pair<SDValue, bool>
7369DAGTypeLegalizer::convertMaskTreeImpl(SDValue V, EVT ToVT, unsigned Depth) {
7370 // The main idea is to recursively traverse VSELECT's mask that needs
7371 // widening to see if we can avoid unnecessary casts. The problem usually
7372 // stems from a simple fact that SETCC might naturally produce results
7373 // not in i1 (as we model it in LLVM IR) and we can continue using that
7374 // type until we have to switch it up. Another important aspect is that
7375 // "all ones" and "all zeros" constants can be materialized at any type,
7376 // so we can try to utilize that to keep SETCC results at their natural
7377 // types as much as possible.
7378 //
7379 // The algorithm traverses the mask-producing tree of operations that
7380 // retain "mask-vector"-ness of the input (i.e. it remains a vector of
7381 // -1s and 0s).
7382 //
7383 // SETCC1 SETCC2 CONST1 SETCC3 CONST2 CONST3
7384 // | / | / | /
7385 // | / | / | /
7386 // |_____/ |______/ |______/
7387 // | * choose the | * choose SETCC3 | * keep it as final type
7388 // | most fitting | type | but consider it subject to
7389 // | type | / change
7390 // | | /
7391 // | |______________/
7392 // | | * choose SETCC3 type
7393 // | /
7394 // | /
7395 // |_____________/
7396 // | * choose the most fitting type
7397 // | and then cast to the final desired type
7398 // |
7399 // VSELECT
7400 //
7401 if (Depth >= DAG.MaxRecursionDepth)
7402 return {};
7403
7404 // Bail out when encounter the vector element count mismatch.
7405 // It potentially can be just an assertion, but we deliberately try to
7406 // be overly conservative here.
7407 if (V.getValueType().getVectorNumElements() != ToVT.getVectorNumElements())
7408 return {};
7409
7410 unsigned Opcode = V.getOpcode();
7411
7412 // Base case: SETCC produces the mask at its natural type.
7413 if (isSETCCOp(Opcode)) {
7414 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: V));
7415 return {convertMask(InMask: V, MaskVT, ToMaskVT: MaskVT), /*IsTypeLenient=*/false};
7416 }
7417
7418 SDLoc DL(V);
7419
7420 // Base case: all-zeros or all-ones BUILD_VECTOR. Type-lenient since these are
7421 // invariant under sign-extend/truncate.
7422 if (ISD::isBuildVectorAllZeros(N: V.getNode()))
7423 return {DAG.getConstant(Val: 0, DL, VT: ToVT), /*IsTypeLenient=*/true};
7424 if (ISD::isBuildVectorAllOnes(N: V.getNode()))
7425 return {DAG.getAllOnesConstant(DL, VT: ToVT), /*IsTypeLenient=*/true};
7426
7427 // Logical operations (AND/OR/XOR): try picking the best fitting width out
7428 // of children's element widths.
7429 if (isLogicalMaskOp(Opcode)) {
7430 auto [Op0, IsLenientOp0] =
7431 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7432 if (!Op0)
7433 return {};
7434 auto [Op1, IsLenientOp1] =
7435 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7436 if (!Op1)
7437 return {};
7438 EVT OpVT = unifyMaskTypes(Op0, IsOpLenient0: IsLenientOp0, Op1, IsOpLenient1: IsLenientOp1, ToVT);
7439 return {DAG.getNode(Opcode, DL, VT: OpVT, N1: Op0, N2: Op1),
7440 IsLenientOp0 && IsLenientOp1};
7441 }
7442
7443 // FREEZE: widen the operand and re-wrap.
7444 if (Opcode == ISD::FREEZE) {
7445 auto [Inner, IsTypeLenient] =
7446 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7447 if (!Inner)
7448 return {};
7449 return {DAG.getNode(Opcode: ISD::FREEZE, DL, VT: Inner.getValueType(), Operand: Inner),
7450 IsTypeLenient};
7451 }
7452
7453 // Vector shuffle: try inferring the best fitting width from operands.
7454 if (Opcode == ISD::VECTOR_SHUFFLE) {
7455 auto *Shuf = cast<ShuffleVectorSDNode>(Val&: V);
7456 auto [Op0, IsLenientOp0] =
7457 convertMaskTreeImpl(V: V.getOperand(i: 0), ToVT, Depth: Depth + 1);
7458 if (!Op0)
7459 return {};
7460 if (V.getOperand(i: 1).isUndef()) {
7461 EVT OpVT = Op0.getValueType();
7462 return {DAG.getVectorShuffle(VT: OpVT, dl: DL, N1: Op0, N2: DAG.getUNDEF(VT: OpVT),
7463 Mask: Shuf->getMask()),
7464 IsLenientOp0};
7465 }
7466 auto [Op1, IsLenientOp1] =
7467 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7468 if (!Op1)
7469 return {};
7470 EVT OpVT = unifyMaskTypes(Op0, IsOpLenient0: IsLenientOp0, Op1, IsOpLenient1: IsLenientOp1, ToVT);
7471 return {DAG.getVectorShuffle(VT: OpVT, dl: DL, N1: Op0, N2: Op1, Mask: Shuf->getMask()),
7472 IsLenientOp0 && IsLenientOp1};
7473 }
7474
7475 // SELECT/VSELECT: try inferring the best fitting width from operands.
7476 if (Opcode == ISD::SELECT || Opcode == ISD::VSELECT) {
7477 auto [Op1, IsLenientOp1] =
7478 convertMaskTreeImpl(V: V.getOperand(i: 1), ToVT, Depth: Depth + 1);
7479 if (!Op1)
7480 return {};
7481 auto [Op2, IsLenientOp2] =
7482 convertMaskTreeImpl(V: V.getOperand(i: 2), ToVT, Depth: Depth + 1);
7483 if (!Op2)
7484 return {};
7485 EVT OpVT = unifyMaskTypes(Op0&: Op1, IsOpLenient0: IsLenientOp1, Op1&: Op2, IsOpLenient1: IsLenientOp2, ToVT);
7486
7487 // We deliberately skip traversing/modifying VSELECT's mask because
7488 //
7489 // a. We only change bitwidth of the operands and it shouldn't affect
7490 // condition on its own.
7491 //
7492 // b. This VSELECT's mask can be widened in an independent traversal
7493 // if needed.
7494 SDValue Cond = V.getOperand(i: 0);
7495 return {DAG.getNode(Opcode, DL, VT: OpVT, N1: Cond, N2: Op1, N3: Op2),
7496 IsLenientOp1 && IsLenientOp2};
7497 }
7498
7499 return {};
7500}
7501
7502SDValue DAGTypeLegalizer::convertMaskTree(SDValue V, EVT ToVT) {
7503 // In general, we are converting from <N x i1> into <M x iW>.
7504 // This would mean that during the tree traversal we need to pay
7505 // attention to both bitwidth and element count, which can be error-prone.
7506 //
7507 // Instead, we split the task in two, we first widen the type of the tree
7508 // and then change the element count.
7509 EVT MaskTreeVT = ToVT.changeVectorElementCount(
7510 Context&: *DAG.getContext(), EC: V.getValueType().getVectorElementCount());
7511 auto [Result, _] = convertMaskTreeImpl(V, ToVT: MaskTreeVT);
7512 if (!Result)
7513 return Result;
7514 return adjustMaskToType(Mask: Result, ToMaskVT: ToVT);
7515}
7516
7517// This method tries to handle some special cases for the vselect mask
7518// and if needed adjusting the mask vector type to match that of the VSELECT.
7519// Without it, many cases end up with scalarization of the SETCC, with many
7520// unnecessary instructions.
7521SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7522 LLVMContext &Ctx = *DAG.getContext();
7523 SDValue Cond = N->getOperand(Num: 0);
7524
7525 if (N->getOpcode() != ISD::VSELECT)
7526 return SDValue();
7527
7528 // If this is a splitted VSELECT that was previously already handled, do
7529 // nothing.
7530 EVT CondVT = Cond->getValueType(ResNo: 0);
7531 if (CondVT.getScalarSizeInBits() != 1)
7532 return SDValue();
7533
7534 EVT VSelVT = N->getValueType(ResNo: 0);
7535
7536 // This method can't handle scalable vector types.
7537 // FIXME: This support could be added in the future.
7538 if (VSelVT.isScalableVector())
7539 return SDValue();
7540
7541 // Only handle vector types which are a power of 2.
7542 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
7543 return SDValue();
7544
7545 // Don't touch if this will be scalarized.
7546 EVT FinalVT = VSelVT;
7547 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
7548 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
7549
7550 if (FinalVT.getVectorNumElements() == 1)
7551 return SDValue();
7552
7553 // If there is support for an i1 vector mask, don't touch.
7554 if (isSETCCOp(Opcode: Cond.getOpcode())) {
7555 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
7556 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
7557 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
7558 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
7559 if (SetCCResVT.getScalarSizeInBits() == 1)
7560 return SDValue();
7561 } else if (CondVT.getScalarType() == MVT::i1) {
7562 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7563 // don't touch.
7564 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
7565 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
7566
7567 if (CondVT.getScalarType() == MVT::i1)
7568 return SDValue();
7569 }
7570
7571 // Widen the vselect result type if needed.
7572 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
7573 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
7574
7575 // The mask of the VSELECT should have integer elements.
7576 EVT ToMaskVT = VSelVT;
7577 if (!ToMaskVT.getScalarType().isInteger())
7578 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7579
7580 // Try to recursively widen the mask expression tree to the target type.
7581 return convertMaskTree(V: Cond, ToVT: ToMaskVT);
7582}
7583
7584SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7585 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7586 ElementCount WidenEC = WidenVT.getVectorElementCount();
7587
7588 SDValue Cond1 = N->getOperand(Num: 0);
7589 EVT CondVT = Cond1.getValueType();
7590 unsigned Opcode = N->getOpcode();
7591 if (CondVT.isVector()) {
7592 if (SDValue WideCond = WidenVSELECTMask(N)) {
7593 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7594 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7595 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7596 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7597 }
7598
7599 EVT CondEltVT = CondVT.getVectorElementType();
7600 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7601 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7602 Cond1 = GetWidenedVector(Op: Cond1);
7603
7604 // If we have to split the condition there is no point in widening the
7605 // select. This would result in an cycle of widening the select ->
7606 // widening the condition operand -> splitting the condition operand ->
7607 // splitting the select -> widening the select. Instead split this select
7608 // further and widen the resulting type.
7609 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7610 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7611 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7612 return Res;
7613 }
7614
7615 if (Cond1.getValueType() != CondWidenVT)
7616 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7617 }
7618
7619 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7620 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7621 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7622 if (Opcode == ISD::VP_MERGE)
7623 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7624 N4: N->getOperand(Num: 3));
7625 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7626}
7627
7628SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7629 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7630 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7631 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7632 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7633 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7634}
7635
7636SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7637 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7638 return DAG.getUNDEF(VT: WidenVT);
7639}
7640
7641SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7642 EVT VT = N->getValueType(ResNo: 0);
7643 SDLoc dl(N);
7644
7645 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7646 unsigned NumElts = VT.getVectorNumElements();
7647 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7648
7649 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7650 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7651
7652 // Adjust mask based on new input vector length.
7653 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7654 for (unsigned i = 0; i != NumElts; ++i) {
7655 int Idx = N->getMaskElt(Idx: i);
7656 if (Idx < (int)NumElts)
7657 NewMask[i] = Idx;
7658 else
7659 NewMask[i] = Idx - NumElts + WidenNumElts;
7660 }
7661 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7662}
7663
7664SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7665 EVT VT = N->getValueType(ResNo: 0);
7666 EVT EltVT = VT.getVectorElementType();
7667 SDLoc dl(N);
7668
7669 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7670 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7671 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7672
7673 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7674 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7675 unsigned VTNumElts = VT.getVectorMinNumElements();
7676 unsigned IdxVal = WidenNumElts - VTNumElts;
7677
7678 if (VT.isScalableVector()) {
7679 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7680 // results together, e.g.(nxv6i64 -> nxv8i64)
7681 // nxv8i64 vector_reverse
7682 // <->
7683 // nxv8i64 concat(
7684 // nxv2i64 extract_subvector(nxv8i64, 2)
7685 // nxv2i64 extract_subvector(nxv8i64, 4)
7686 // nxv2i64 extract_subvector(nxv8i64, 6)
7687 // nxv2i64 undef)
7688
7689 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7690 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7691 EC: ElementCount::getScalable(MinVal: GCD));
7692 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7693 "down type's element count");
7694 SmallVector<SDValue> Parts;
7695 unsigned i = 0;
7696 for (; i < VTNumElts / GCD; ++i)
7697 Parts.push_back(
7698 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7699 for (; i < WidenNumElts / GCD; ++i)
7700 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7701
7702 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7703 }
7704
7705 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7706 // fixed-vectors.
7707 SmallVector<int, 16> Mask(WidenNumElts, -1);
7708 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7709
7710 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7711 Mask);
7712}
7713
7714SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7715 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7716 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7717}
7718
7719void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
7720 EVT VT = N->getValueType(ResNo: 0);
7721 EVT EltVT = VT.getVectorElementType();
7722 ElementCount OrigEC = VT.getVectorElementCount();
7723 unsigned Factor = N->getNumOperands();
7724 SDLoc DL(N);
7725
7726 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7727 ElementCount WidenEC = WidenVT.getVectorElementCount();
7728
7729 SmallVector<SDValue, 8> WidenOps(Factor);
7730 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7731 WidenOps[Idx] = GetWidenedVector(Op: N->getOperand(Num: Idx));
7732
7733 SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
7734 SDValue Interleaved =
7735 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: WidenVTs, Ops: WidenOps);
7736
7737 EVT PackedWidenVT =
7738 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7739 SmallVector<SDValue, 8> Slices(Factor);
7740 for (unsigned Idx = 0; Idx != Factor; ++Idx)
7741 Slices[Idx] = Interleaved.getValue(R: Idx);
7742
7743 SDValue Packed = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PackedWidenVT, Ops: Slices);
7744
7745 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7746 SDValue Narrow = DAG.getExtractSubvector(DL, VT, Vec: Packed,
7747 Idx: OrigEC.getKnownMinValue() * Idx);
7748 SDValue Wide =
7749 DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: Narrow, /*Idx=*/0U);
7750 SetWidenedVector(Op: SDValue(N, Idx), Result: Wide);
7751 }
7752}
7753
7754SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
7755 SDLoc DL(N);
7756 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7757 EVT SourceVT = N->getOperand(Num: 0).getValueType();
7758 EVT WideSourceVT =
7759 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SourceVT.getVectorElementType(),
7760 EC: WidenVT.getVectorElementCount());
7761
7762 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
7763 SubVec: N->getOperand(Num: 0), Idx: 0);
7764 SDValue WideMask = DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT),
7765 SubVec: N->getOperand(Num: 2), Idx: 0);
7766 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
7767 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
7768}
7769
7770void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7771 EVT VT = N->getValueType(ResNo: 0);
7772 EVT EltVT = VT.getVectorElementType();
7773 ElementCount OrigEC = VT.getVectorElementCount();
7774 unsigned Factor = N->getNumOperands();
7775 SDLoc DL(N);
7776
7777 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7778 ElementCount WidenEC = WidenVT.getVectorElementCount();
7779 // We cannot just use the widened operands directly: since they might be
7780 // individually widened, using them directly will result in de-interleaving
7781 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7782 // not concat the widened operands but the original ones to effectively
7783 // generate a "packed" concated and widened vector, before extracting new
7784 // operand vectors with the widened type.
7785 EVT PackedWidenVT =
7786 EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: WidenEC * Factor);
7787 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: OrigEC * Factor);
7788 SDValue ConcatOp = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops: N->ops());
7789 SDValue PackedWidenVec = DAG.getInsertSubvector(
7790 DL, Vec: DAG.getUNDEF(VT: PackedWidenVT), SubVec: ConcatOp, /*Idx=*/0U);
7791
7792 // Extract the new widened operand vectors.
7793 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7794 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7795 NewOps[Idx] = DAG.getExtractSubvector(DL, VT: WidenVT, Vec: PackedWidenVec,
7796 Idx: WidenEC.getKnownMinValue() * Idx);
7797 }
7798
7799 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7800 SDValue NewRes = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: NewVTs, Ops: NewOps);
7801 // Set the widened results manually.
7802 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7803 SetWidenedVector(Op: SDValue(N, Idx), Result: NewRes.getValue(R: Idx));
7804}
7805
7806SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7807 assert(N->getValueType(0).isVector() &&
7808 N->getOperand(0).getValueType().isVector() &&
7809 "Operands must be vectors");
7810 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7811 ElementCount WidenEC = WidenVT.getVectorElementCount();
7812
7813 SDValue InOp1 = N->getOperand(Num: 0);
7814 EVT InVT = InOp1.getValueType();
7815 assert(InVT.isVector() && "can not widen non-vector type");
7816 EVT WidenInVT =
7817 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7818
7819 // The input and output types often differ here, and it could be that while
7820 // we'd prefer to widen the result type, the input operands have been split.
7821 // In this case, we also need to split the result of this node as well.
7822 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7823 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7824 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7825 return Res;
7826 }
7827
7828 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7829 SDValue InOp2 = N->getOperand(Num: 1);
7830 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7831 InOp1 = GetWidenedVector(Op: InOp1);
7832 InOp2 = GetWidenedVector(Op: InOp2);
7833 } else {
7834 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7835 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7836 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7837 N2: InOp1, N3: ZeroIdx);
7838 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7839 N2: InOp2, N3: ZeroIdx);
7840 }
7841
7842 // Assume that the input and output will be widen appropriately. If not,
7843 // we will have to unroll it at some point.
7844 assert(InOp1.getValueType() == WidenInVT &&
7845 InOp2.getValueType() == WidenInVT &&
7846 "Input not widened to expected type!");
7847 (void)WidenInVT;
7848 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7849 N3: N->getOperand(Num: 2));
7850}
7851
7852SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7853 assert(N->getValueType(0).isVector() &&
7854 N->getOperand(1).getValueType().isVector() &&
7855 "Operands must be vectors");
7856 EVT VT = N->getValueType(ResNo: 0);
7857 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7858 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7859 unsigned NumElts = VT.getVectorNumElements();
7860 EVT EltVT = VT.getVectorElementType();
7861
7862 SDLoc dl(N);
7863 SDValue Chain = N->getOperand(Num: 0);
7864 SDValue LHS = N->getOperand(Num: 1);
7865 SDValue RHS = N->getOperand(Num: 2);
7866 SDValue CC = N->getOperand(Num: 3);
7867 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7868
7869 // Fully unroll and reassemble.
7870 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7871 SmallVector<SDValue, 8> Chains(NumElts);
7872 for (unsigned i = 0; i != NumElts; ++i) {
7873 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7874 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7875
7876 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7877 Ops: {Chain, LHSElem, RHSElem, CC});
7878 Chains[i] = Scalars[i].getValue(R: 1);
7879 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7880 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7881 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7882 }
7883
7884 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7885 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7886
7887 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7888}
7889
7890SDValue DAGTypeLegalizer::WidenVecRes_PARTIAL_REDUCE_MLA(SDNode *N) {
7891 SDLoc DL(N);
7892 EVT VT = N->getValueType(ResNo: 0);
7893
7894 // Expand, then widen the result.
7895 SDValue Expanded = TLI.expandPartialReduceMLA(Node: N, DAG);
7896 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7897 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WideVT), SubVec: Expanded, Idx: 0);
7898}
7899
7900//===----------------------------------------------------------------------===//
7901// Widen Vector Operand
7902//===----------------------------------------------------------------------===//
7903bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7904 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7905 SDValue Res = SDValue();
7906
7907 // See if the target wants to custom widen this node.
7908 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7909 return false;
7910
7911 switch (N->getOpcode()) {
7912 default:
7913#ifndef NDEBUG
7914 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7915 N->dump(&DAG);
7916 dbgs() << "\n";
7917#endif
7918 report_fatal_error(reason: "do not know how to widen this operator's operand!");
7919
7920 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7921 case ISD::FAKE_USE:
7922 Res = WidenVecOp_FAKE_USE(N);
7923 break;
7924 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7925 case ISD::VECTOR_REPEAT:
7926 Res = WidenVecOp_VECTOR_REPEAT(N);
7927 break;
7928 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7929 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7930 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7931 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7932 case ISD::ATOMIC_STORE:
7933 Res = WidenVecOp_ATOMIC_STORE(ST: cast<AtomicSDNode>(Val: N));
7934 break;
7935 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7936 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7937 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7938 break;
7939 case ISD::ANY_EXTEND_VECTOR_INREG:
7940 case ISD::SIGN_EXTEND_VECTOR_INREG:
7941 case ISD::ZERO_EXTEND_VECTOR_INREG:
7942 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7943 break;
7944 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7945 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7946 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7947 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7948 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7949 case ISD::STRICT_FSETCC:
7950 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7951 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7952 case ISD::FLDEXP:
7953 case ISD::FCOPYSIGN:
7954 case ISD::LROUND:
7955 case ISD::LLROUND:
7956 case ISD::LRINT:
7957 case ISD::LLRINT:
7958 Res = WidenVecOp_UnrollVectorOp(N);
7959 break;
7960 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7961
7962 case ISD::ANY_EXTEND:
7963 case ISD::SIGN_EXTEND:
7964 case ISD::ZERO_EXTEND:
7965 Res = WidenVecOp_EXTEND(N);
7966 break;
7967
7968 case ISD::SCMP:
7969 case ISD::UCMP:
7970 Res = WidenVecOp_CMP(N);
7971 break;
7972
7973 case ISD::FP_EXTEND:
7974 case ISD::STRICT_FP_EXTEND:
7975 case ISD::FP_ROUND:
7976 case ISD::STRICT_FP_ROUND:
7977 case ISD::FP_TO_SINT:
7978 case ISD::STRICT_FP_TO_SINT:
7979 case ISD::FP_TO_UINT:
7980 case ISD::STRICT_FP_TO_UINT:
7981 case ISD::SINT_TO_FP:
7982 case ISD::STRICT_SINT_TO_FP:
7983 case ISD::UINT_TO_FP:
7984 case ISD::STRICT_UINT_TO_FP:
7985 case ISD::TRUNCATE:
7986 case ISD::CONVERT_FROM_ARBITRARY_FP:
7987 case ISD::CONVERT_TO_ARBITRARY_FP:
7988 Res = WidenVecOp_Convert(N);
7989 break;
7990
7991 case ISD::FP_TO_SINT_SAT:
7992 case ISD::FP_TO_UINT_SAT:
7993 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7994 break;
7995
7996 case ISD::VECREDUCE_FADD:
7997 case ISD::VECREDUCE_FMUL:
7998 case ISD::VECREDUCE_ADD:
7999 case ISD::VECREDUCE_MUL:
8000 case ISD::VECREDUCE_AND:
8001 case ISD::VECREDUCE_OR:
8002 case ISD::VECREDUCE_XOR:
8003 case ISD::VECREDUCE_SMAX:
8004 case ISD::VECREDUCE_SMIN:
8005 case ISD::VECREDUCE_UMAX:
8006 case ISD::VECREDUCE_UMIN:
8007 case ISD::VECREDUCE_FMAX:
8008 case ISD::VECREDUCE_FMIN:
8009 case ISD::VECREDUCE_FMAXIMUM:
8010 case ISD::VECREDUCE_FMINIMUM:
8011 case ISD::VECREDUCE_FMAXIMUMNUM:
8012 case ISD::VECREDUCE_FMINIMUMNUM:
8013 Res = WidenVecOp_VECREDUCE(N);
8014 break;
8015 case ISD::VECREDUCE_SEQ_FADD:
8016 case ISD::VECREDUCE_SEQ_FMUL:
8017 Res = WidenVecOp_VECREDUCE_SEQ(N);
8018 break;
8019 case ISD::VP_REDUCE_FADD:
8020 case ISD::VP_REDUCE_SEQ_FADD:
8021 case ISD::VP_REDUCE_FMUL:
8022 case ISD::VP_REDUCE_SEQ_FMUL:
8023 case ISD::VP_REDUCE_ADD:
8024 case ISD::VP_REDUCE_MUL:
8025 case ISD::VP_REDUCE_AND:
8026 case ISD::VP_REDUCE_OR:
8027 case ISD::VP_REDUCE_XOR:
8028 case ISD::VP_REDUCE_SMAX:
8029 case ISD::VP_REDUCE_SMIN:
8030 case ISD::VP_REDUCE_UMAX:
8031 case ISD::VP_REDUCE_UMIN:
8032 case ISD::VP_REDUCE_FMAX:
8033 case ISD::VP_REDUCE_FMIN:
8034 case ISD::VP_REDUCE_FMAXIMUM:
8035 case ISD::VP_REDUCE_FMINIMUM:
8036 Res = WidenVecOp_VP_REDUCE(N);
8037 break;
8038 case ISD::CTTZ_ELTS:
8039 case ISD::CTTZ_ELTS_ZERO_POISON:
8040 Res = WidenVecOp_CttzElements(N);
8041 break;
8042 case ISD::VP_CTTZ_ELTS:
8043 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8044 Res = WidenVecOp_VP_CttzElements(N);
8045 break;
8046 case ISD::VECTOR_FIND_LAST_ACTIVE:
8047 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
8048 break;
8049 case ISD::VECTOR_MATCH:
8050 Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
8051 break;
8052 }
8053
8054 // If Res is null, the sub-method took care of registering the result.
8055 if (!Res.getNode()) return false;
8056
8057 // If the result is N, the sub-method updated N in place. Tell the legalizer
8058 // core about this.
8059 if (Res.getNode() == N)
8060 return true;
8061
8062
8063 if (N->isStrictFPOpcode())
8064 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
8065 "Invalid operand expansion");
8066 else
8067 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
8068 "Invalid operand expansion");
8069
8070 ReplaceValueWith(From: SDValue(N, 0), To: Res);
8071 return false;
8072}
8073
8074SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
8075 SDLoc DL(N);
8076 EVT VT = N->getValueType(ResNo: 0);
8077
8078 SDValue InOp = N->getOperand(Num: 0);
8079 assert(getTypeAction(InOp.getValueType()) ==
8080 TargetLowering::TypeWidenVector &&
8081 "Unexpected type action");
8082 InOp = GetWidenedVector(Op: InOp);
8083 assert(VT.getVectorNumElements() <
8084 InOp.getValueType().getVectorNumElements() &&
8085 "Input wasn't widened!");
8086
8087 // We may need to further widen the operand until it has the same total
8088 // vector size as the result.
8089 EVT InVT = InOp.getValueType();
8090 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
8091 EVT InEltVT = InVT.getVectorElementType();
8092 for (EVT FixedVT : MVT::vector_valuetypes()) {
8093 EVT FixedEltVT = FixedVT.getVectorElementType();
8094 if (TLI.isTypeLegal(VT: FixedVT) &&
8095 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
8096 FixedEltVT == InEltVT) {
8097 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
8098 "Not enough elements in the fixed type for the operand!");
8099 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
8100 "We can't have the same type as we started with!");
8101 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
8102 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
8103 else
8104 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
8105 break;
8106 }
8107 }
8108 InVT = InOp.getValueType();
8109 if (InVT.getSizeInBits() != VT.getSizeInBits())
8110 // We couldn't find a legal vector type that was a widening of the input
8111 // and could be extended in-register to the result type, so we have to
8112 // scalarize.
8113 return WidenVecOp_Convert(N);
8114 }
8115
8116 // Use special DAG nodes to represent the operation of extending the
8117 // low lanes.
8118 switch (N->getOpcode()) {
8119 default:
8120 llvm_unreachable("Extend legalization on extend operation!");
8121 case ISD::ANY_EXTEND:
8122 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8123 case ISD::SIGN_EXTEND:
8124 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8125 case ISD::ZERO_EXTEND:
8126 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
8127 }
8128}
8129
8130SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
8131 SDLoc dl(N);
8132
8133 EVT OpVT = N->getOperand(Num: 0).getValueType();
8134 EVT ResVT = N->getValueType(ResNo: 0);
8135 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
8136 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8137
8138 // 1. EXTRACT_SUBVECTOR
8139 // 2. SIGN_EXTEND/ZERO_EXTEND
8140 // 3. CMP
8141 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
8142 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
8143
8144 // At this point the result type is guaranteed to be valid, so we can use it
8145 // as the operand type by extending it appropriately
8146 ISD::NodeType ExtendOpcode =
8147 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8148 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
8149 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
8150
8151 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
8152}
8153
8154SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
8155 // The result (and first input) is legal, but the second input is illegal.
8156 // We can't do much to fix that, so just unroll and let the extracts off of
8157 // the second input be widened as needed later.
8158 return DAG.UnrollVectorOp(N);
8159}
8160
8161SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
8162 SDLoc DL(N);
8163 EVT ResultVT = N->getValueType(ResNo: 0);
8164 SDValue Test = N->getOperand(Num: 1);
8165 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
8166
8167 // Process this node similarly to SETCC.
8168 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
8169 if (ResultVT.getScalarType() == MVT::i1)
8170 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8171 NumElements: WideResultVT.getVectorNumElements());
8172
8173 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
8174 Ops: {WideArg, Test}, Flags: N->getFlags());
8175
8176 // Extract the needed results from the result vector.
8177 EVT ResVT =
8178 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
8179 NumElements: ResultVT.getVectorNumElements());
8180 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
8181
8182 EVT OpVT = N->getOperand(Num: 0).getValueType();
8183 ISD::NodeType ExtendCode =
8184 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8185 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
8186}
8187
8188SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
8189 // Since the result is legal and the input is illegal.
8190 EVT VT = N->getValueType(ResNo: 0);
8191 EVT EltVT = VT.getVectorElementType();
8192 SDLoc dl(N);
8193 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
8194 assert(getTypeAction(InOp.getValueType()) ==
8195 TargetLowering::TypeWidenVector &&
8196 "Unexpected type action");
8197 InOp = GetWidenedVector(Op: InOp);
8198 EVT InVT = InOp.getValueType();
8199 unsigned Opcode = N->getOpcode();
8200
8201 // Helper to build a convert node with all scalar trailing operands.
8202 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
8203 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
8204 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
8205 N4: N->getOperand(Num: 3));
8206 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
8207 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1));
8208 return DAG.getNode(Opcode, DL: dl, VT, Operand: Op);
8209 };
8210
8211 // See if a widened result type would be legal, if so widen the node.
8212 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
8213 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
8214 EC: InVT.getVectorElementCount());
8215 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
8216 SDValue Res;
8217 if (N->isStrictFPOpcode()) {
8218 if (Opcode == ISD::STRICT_FP_ROUND)
8219 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
8220 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
8221 else
8222 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
8223 Ops: { N->getOperand(Num: 0), InOp });
8224 // Legalize the chain result - switch anything that used the old chain to
8225 // use the new one.
8226 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8227 } else {
8228 Res = MakeConvertNode(WideVT, InOp);
8229 }
8230 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
8231 }
8232
8233 EVT InEltVT = InVT.getVectorElementType();
8234
8235 // Unroll the convert into some scalar code and create a nasty build vector.
8236 unsigned NumElts = VT.getVectorNumElements();
8237 SmallVector<SDValue, 16> Ops(NumElts);
8238 if (N->isStrictFPOpcode()) {
8239 SmallVector<SDValue, 4> NewOps(N->ops());
8240 SmallVector<SDValue, 32> OpChains;
8241 for (unsigned i=0; i < NumElts; ++i) {
8242 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
8243 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
8244 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
8245 }
8246 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
8247 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8248 } else {
8249 for (unsigned i = 0; i < NumElts; ++i) {
8250 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
8251 Ops[i] = MakeConvertNode(EltVT, Elt);
8252 }
8253 }
8254
8255 return DAG.getBuildVector(VT, DL: dl, Ops);
8256}
8257
8258SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
8259 EVT DstVT = N->getValueType(ResNo: 0);
8260 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
8261 EVT SrcVT = Src.getValueType();
8262 ElementCount WideNumElts = SrcVT.getVectorElementCount();
8263 SDLoc dl(N);
8264
8265 // See if a widened result type would be legal, if so widen the node.
8266 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8267 VT: DstVT.getVectorElementType(), EC: WideNumElts);
8268 if (TLI.isTypeLegal(VT: WideDstVT)) {
8269 SDValue Res =
8270 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
8271 return DAG.getNode(
8272 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
8273 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
8274 }
8275
8276 // Give up and unroll.
8277 return DAG.UnrollVectorOp(N);
8278}
8279
8280SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
8281 EVT VT = N->getValueType(ResNo: 0);
8282 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8283 EVT InWidenVT = InOp.getValueType();
8284 SDLoc dl(N);
8285
8286 // Check if we can convert between two legal vector types and extract.
8287 TypeSize InWidenSize = InWidenVT.getSizeInBits();
8288 TypeSize Size = VT.getSizeInBits();
8289 // x86mmx is not an acceptable vector element type, so don't try.
8290 if (!VT.isVector() && VT != MVT::x86mmx &&
8291 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
8292 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
8293 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
8294 if (TLI.isTypeLegal(VT: NewVT)) {
8295 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8296 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
8297 }
8298 }
8299
8300 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
8301 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
8302 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
8303 // having to copy via memory.
8304 if (VT.isVector()) {
8305 EVT EltVT = VT.getVectorElementType();
8306 unsigned EltSize = EltVT.getFixedSizeInBits();
8307 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
8308 ElementCount NewNumElts =
8309 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
8310 .divideCoefficientBy(RHS: EltSize);
8311 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
8312 if (TLI.isTypeLegal(VT: NewVT)) {
8313 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8314 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
8315 }
8316 }
8317 }
8318
8319 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
8320}
8321
8322// Vectors with sizes that are not powers of 2 need to be widened to the
8323// next largest power of 2. For example, we may get a vector of 3 32-bit
8324// integers or of 6 16-bit integers, both of which have to be widened to a
8325// 128-bit vector.
8326SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8327 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
8328 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
8329 N2: WidenedOp);
8330}
8331
8332SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8333 EVT VT = N->getValueType(ResNo: 0);
8334 EVT EltVT = VT.getVectorElementType();
8335 EVT InVT = N->getOperand(Num: 0).getValueType();
8336 SDLoc dl(N);
8337
8338 // If the widen width for this operand is the same as the width of the concat
8339 // and all but the first operand is undef, just use the widened operand.
8340 unsigned NumOperands = N->getNumOperands();
8341 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
8342 unsigned i;
8343 for (i = 1; i < NumOperands; ++i)
8344 if (!N->getOperand(Num: i).isUndef())
8345 break;
8346
8347 if (i == NumOperands)
8348 return GetWidenedVector(Op: N->getOperand(Num: 0));
8349 }
8350
8351 if (VT.isScalableVector()) {
8352 SDValue Result = DAG.getPOISON(VT);
8353 unsigned NumInElts = InVT.getVectorMinNumElements();
8354 for (unsigned i = 0; i < NumOperands; ++i) {
8355 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: i));
8356 if (InOp.getValueType() != InVT)
8357 InOp = DAG.getExtractSubvector(DL: dl, VT: InVT, Vec: InOp, Idx: 0);
8358 Result = DAG.getInsertSubvector(DL: dl, Vec: Result, SubVec: InOp, Idx: i * NumInElts);
8359 }
8360 return Result;
8361 }
8362
8363 // Otherwise, fall back to a nasty build vector.
8364 unsigned NumElts = VT.getVectorNumElements();
8365 SmallVector<SDValue, 16> Ops(NumElts);
8366
8367 unsigned NumInElts = InVT.getVectorNumElements();
8368
8369 unsigned Idx = 0;
8370 for (unsigned i=0; i < NumOperands; ++i) {
8371 SDValue InOp = N->getOperand(Num: i);
8372 assert(getTypeAction(InOp.getValueType()) ==
8373 TargetLowering::TypeWidenVector &&
8374 "Unexpected type action");
8375 InOp = GetWidenedVector(Op: InOp);
8376 for (unsigned j = 0; j < NumInElts; ++j)
8377 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
8378 }
8379 return DAG.getBuildVector(VT, DL: dl, Ops);
8380}
8381
8382SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_REPEAT(SDNode *N) {
8383 SDLoc DL(N);
8384 EVT VT = N->getValueType(ResNo: 0);
8385 SDValue Src = N->getOperand(Num: 0);
8386 EVT SrcVT = Src.getValueType();
8387 EVT WidenedSrcVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SrcVT);
8388
8389 if (!WidenedSrcVT.getVectorElementCount().hasKnownScalarFactor(
8390 RHS: SrcVT.getVectorElementCount()))
8391 report_fatal_error(
8392 reason: "Cannot widen VECTOR_REPEAT operand to an ElementCount that's not "
8393 "a known scalar multiple of the input ElementCount.");
8394
8395 // Repeat the original source because the extra lanes of its widened value
8396 // are unspecified.
8397 unsigned NumConcat =
8398 WidenedSrcVT.getVectorNumElements() / SrcVT.getVectorNumElements();
8399 SmallVector<SDValue, 8> Ops(NumConcat, Src);
8400 SDValue WidenedSrc = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WidenedSrcVT, Ops);
8401 EVT WidenedVT = VT.changeVectorElementCount(
8402 Context&: *DAG.getContext(),
8403 EC: ElementCount::getScalable(MinVal: WidenedSrcVT.getVectorNumElements()));
8404 SDValue Widened = DAG.getNode(Opcode: ISD::VECTOR_REPEAT, DL, VT: WidenedVT, Operand: WidenedSrc);
8405 return DAG.getExtractSubvector(DL, VT, Vec: Widened, Idx: 0);
8406}
8407
8408SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8409 EVT VT = N->getValueType(ResNo: 0);
8410 SDValue SubVec = N->getOperand(Num: 1);
8411 SDValue InVec = N->getOperand(Num: 0);
8412
8413 EVT OrigVT = SubVec.getValueType();
8414 SubVec = GetWidenedVector(Op: SubVec);
8415 EVT SubVT = SubVec.getValueType();
8416
8417 // Whether or not all the elements of the widened SubVec will be inserted into
8418 // valid indices of VT.
8419 bool IndicesValid = false;
8420 // If we statically know that VT can fit SubVT, the indices are valid.
8421 if (VT.knownBitsGE(VT: SubVT))
8422 IndicesValid = true;
8423 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8424 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8425 // we know the minimum vscale we can work out if it's valid ourselves.
8426 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8427 Kind: Attribute::VScaleRange);
8428 if (Attr.isValid()) {
8429 unsigned VScaleMin = Attr.getVScaleRangeMin();
8430 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8431 SubVT.getFixedSizeInBits())
8432 IndicesValid = true;
8433 }
8434 }
8435
8436 if (!IndicesValid)
8437 report_fatal_error(
8438 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
8439
8440 SDLoc DL(N);
8441
8442 // We need to make sure that the indices are still valid, otherwise we might
8443 // widen what was previously well-defined to something undefined.
8444 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
8445 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
8446 N3: N->getOperand(Num: 2));
8447
8448 if (OrigVT.isScalableVector()) {
8449 // When the widened types match, overwriting the start of a vector is
8450 // effectively a merge operation that can be implement as a vselect.
8451 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
8452 SDValue Mask =
8453 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
8454 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
8455 }
8456
8457 // Fallback to inserting through memory.
8458 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8459 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
8460 MachineFunction &MF = DAG.getMachineFunction();
8461 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
8462 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
8463
8464 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8465 PtrInfo, F: MachineMemOperand::MOStore,
8466 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8467 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8468 PtrInfo, F: MachineMemOperand::MOLoad,
8469 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8470
8471 // Write out the vector being inserting into.
8472 SDValue Ch =
8473 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
8474
8475 // Build a mask to match the length of the sub-vector.
8476 SDValue Mask =
8477 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
8478
8479 // Overwrite the sub-vector at the required offset.
8480 SDValue SubVecPtr =
8481 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
8482 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
8483 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
8484 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
8485
8486 // Read back the result.
8487 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
8488 }
8489
8490 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8491 // with a series of INSERT_VECTOR_ELT
8492 unsigned Idx = N->getConstantOperandVal(Num: 2);
8493
8494 SDValue InsertElt = InVec;
8495 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8496 SDValue ExtractElt =
8497 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
8498 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
8499 }
8500
8501 return InsertElt;
8502}
8503
8504SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8505 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8506 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
8507 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8508}
8509
8510SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8511 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8512 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
8513 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8514}
8515
8516SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8517 SDLoc DL(N);
8518 EVT ResVT = N->getValueType(ResNo: 0);
8519
8520 // Widen the input as requested by the legalizer.
8521 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8522 EVT WideInVT = WideInOp.getValueType();
8523
8524 // Simple case: if widened input is still smaller than or equal to result,
8525 // just use it directly.
8526 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8527 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
8528
8529 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8530 // If widening makes the input larger than the original result, widen the
8531 // result to match, then extract back down.
8532 EVT ResEltVT = ResVT.getVectorElementType();
8533 unsigned EltBits = ResEltVT.getSizeInBits();
8534 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8535 "Widened input size must be a multiple of result element size");
8536
8537 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8538 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
8539
8540 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
8541 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
8542}
8543
8544SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8545 // We have to widen the value, but we want only to store the original
8546 // vector type.
8547 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
8548
8549 if (!ST->getMemoryVT().getScalarType().isByteSized())
8550 return TLI.scalarizeVectorStore(ST, DAG);
8551
8552 if (ST->isTruncatingStore())
8553 return TLI.scalarizeVectorStore(ST, DAG);
8554
8555 // Generate a vector-predicated store if it is custom/legal on the target.
8556 // To avoid possible recursion, only do this if the widened mask type is
8557 // legal.
8558 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8559 // removed from the IR by the ExpandVectorPredication pass but we're
8560 // reintroducing them here.
8561 SDValue StVal = ST->getValue();
8562 EVT StVT = StVal.getValueType();
8563 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
8564 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
8565
8566 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8567 TLI.isTypeLegal(VT: WideMaskVT)) {
8568 // Widen the value.
8569 SDLoc DL(N);
8570 StVal = GetWidenedVector(Op: StVal);
8571 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
8572 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
8573 EC: StVT.getVectorElementCount());
8574 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
8575 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
8576 AM: ST->getAddressingMode());
8577 }
8578
8579 SmallVector<SDValue, 16> StChain;
8580 if (GenWidenVectorStores(StChain, ST)) {
8581 if (StChain.size() == 1)
8582 return StChain[0];
8583
8584 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
8585 }
8586
8587 if (StVT.isVector()) {
8588 // If all else fails replace the store with a wide masked store.
8589 SDLoc DL(N);
8590 SDValue WideStVal = GetWidenedVector(Op: StVal);
8591 SDValue Mask =
8592 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
8593
8594 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
8595 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
8596 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
8597 IsTruncating: ST->isTruncatingStore());
8598 }
8599
8600 report_fatal_error(reason: "Unable to widen vector store");
8601}
8602
8603SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8604 EVT StVT = ST->getMemoryVT();
8605 SDLoc dl(ST);
8606
8607 SDValue StVal = GetWidenedVector(Op: ST->getVal());
8608 EVT WidenVT = StVal.getValueType();
8609
8610 TypeSize StWidth = StVT.getSizeInBits();
8611 TypeSize WidenWidth = WidenVT.getSizeInBits();
8612 TypeSize WidthDiff = WidenWidth - StWidth;
8613
8614 // Find the vector type that can store the original memory width in one
8615 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8616 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8617 // implies a 4-byte movl, writing undef bytes past its object).
8618 std::optional<EVT> FirstVT =
8619 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/Align: 0,
8620 WidenEx: WidthDiff.getKnownMinValue());
8621 if (!FirstVT)
8622 return SDValue();
8623
8624 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8625
8626 SDValue StOp =
8627 coerceStoredValue(StVal, FirstVT: *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8628
8629 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: *FirstVT, Chain: ST->getChain(), Ptr: StOp,
8630 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
8631}
8632
8633SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8634 assert((OpNo == 1 || OpNo == 3) &&
8635 "Can widen only data or mask operand of vp_store");
8636 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
8637 SDValue Mask = ST->getMask();
8638 SDValue StVal = ST->getValue();
8639 SDLoc dl(N);
8640
8641 if (OpNo == 1) {
8642 // Widen the value.
8643 StVal = GetWidenedVector(Op: StVal);
8644
8645 // We only handle the case where the mask needs widening to an
8646 // identically-sized type as the vector inputs.
8647 assert(getTypeAction(Mask.getValueType()) ==
8648 TargetLowering::TypeWidenVector &&
8649 "Unable to widen VP store");
8650 Mask = GetWidenedVector(Op: Mask);
8651 } else {
8652 Mask = GetWidenedVector(Op: Mask);
8653
8654 // We only handle the case where the stored value needs widening to an
8655 // identically-sized type as the mask.
8656 assert(getTypeAction(StVal.getValueType()) ==
8657 TargetLowering::TypeWidenVector &&
8658 "Unable to widen VP store");
8659 StVal = GetWidenedVector(Op: StVal);
8660 }
8661
8662 assert(Mask.getValueType().getVectorElementCount() ==
8663 StVal.getValueType().getVectorElementCount() &&
8664 "Mask and data vectors should have the same number of elements");
8665 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
8666 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
8667 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
8668 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
8669 IsCompressing: ST->isCompressingStore());
8670}
8671
8672SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8673 unsigned OpNo) {
8674 assert((OpNo == 1 || OpNo == 4) &&
8675 "Can widen only data or mask operand of vp_strided_store");
8676 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
8677 SDValue Mask = SST->getMask();
8678 SDValue StVal = SST->getValue();
8679 SDLoc DL(N);
8680
8681 if (OpNo == 1)
8682 assert(getTypeAction(Mask.getValueType()) ==
8683 TargetLowering::TypeWidenVector &&
8684 "Unable to widen VP strided store");
8685 else
8686 assert(getTypeAction(StVal.getValueType()) ==
8687 TargetLowering::TypeWidenVector &&
8688 "Unable to widen VP strided store");
8689
8690 StVal = GetWidenedVector(Op: StVal);
8691 Mask = GetWidenedVector(Op: Mask);
8692
8693 assert(StVal.getValueType().getVectorElementCount() ==
8694 Mask.getValueType().getVectorElementCount() &&
8695 "Data and mask vectors should have the same number of elements");
8696
8697 return DAG.getStridedStoreVP(
8698 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
8699 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
8700 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
8701 IsCompressing: SST->isCompressingStore());
8702}
8703
8704SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8705 assert((OpNo == 1 || OpNo == 4) &&
8706 "Can widen only data or mask operand of mstore");
8707 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
8708 SDValue Mask = MST->getMask();
8709 EVT MaskVT = Mask.getValueType();
8710 SDValue StVal = MST->getValue();
8711 EVT VT = StVal.getValueType();
8712 SDLoc dl(N);
8713
8714 EVT WideVT, WideMaskVT;
8715 if (OpNo == 1) {
8716 // Widen the value.
8717 StVal = GetWidenedVector(Op: StVal);
8718
8719 WideVT = StVal.getValueType();
8720 WideMaskVT =
8721 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8722 EC: WideVT.getVectorElementCount());
8723 } else {
8724 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8725
8726 EVT ValueVT = StVal.getValueType();
8727 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8728 EC: WideMaskVT.getVectorElementCount());
8729 }
8730
8731 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8732 TLI.isTypeLegal(VT: WideMaskVT) && !MST->isCompressingStore()) {
8733 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8734 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8735 EC: VT.getVectorElementCount());
8736 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8737 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8738 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8739 }
8740
8741 if (OpNo == 1) {
8742 // The mask should be widened as well.
8743 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8744 } else {
8745 // Widen the mask.
8746 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8747
8748 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8749 }
8750
8751 assert(Mask.getValueType().getVectorElementCount() ==
8752 StVal.getValueType().getVectorElementCount() &&
8753 "Mask and data vectors should have the same number of elements");
8754 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8755 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8756 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8757 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8758}
8759
8760SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8761 assert(OpNo == 4 && "Can widen only the index of mgather");
8762 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8763 SDValue DataOp = MG->getPassThru();
8764 SDValue Mask = MG->getMask();
8765 SDValue Scale = MG->getScale();
8766
8767 // Just widen the index. It's allowed to have extra elements.
8768 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8769
8770 SDLoc dl(N);
8771 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8772 Scale};
8773 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8774 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8775 ExtTy: MG->getExtensionType());
8776 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8777 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8778 return SDValue();
8779}
8780
8781SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8782 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8783 SDValue DataOp = MSC->getValue();
8784 SDValue Mask = MSC->getMask();
8785 SDValue Index = MSC->getIndex();
8786 SDValue Scale = MSC->getScale();
8787 EVT WideMemVT = MSC->getMemoryVT();
8788
8789 if (OpNo == 1) {
8790 DataOp = GetWidenedVector(Op: DataOp);
8791 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8792
8793 // Widen index.
8794 EVT IndexVT = Index.getValueType();
8795 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8796 VT: IndexVT.getVectorElementType(), EC: WideEC);
8797 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8798
8799 // The mask should be widened as well.
8800 EVT MaskVT = Mask.getValueType();
8801 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8802 VT: MaskVT.getVectorElementType(), EC: WideEC);
8803 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8804
8805 // Widen the MemoryType
8806 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8807 VT: MSC->getMemoryVT().getScalarType(), EC: WideEC);
8808 } else if (OpNo == 4) {
8809 // Just widen the index. It's allowed to have extra elements.
8810 Index = GetWidenedVector(Op: Index);
8811 } else
8812 llvm_unreachable("Can't widen this operand of mscatter");
8813
8814 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8815 Scale};
8816 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8817 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8818 IsTruncating: MSC->isTruncatingStore());
8819}
8820
8821SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8822 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8823 SDValue DataOp = VPSC->getValue();
8824 SDValue Mask = VPSC->getMask();
8825 SDValue Index = VPSC->getIndex();
8826 SDValue Scale = VPSC->getScale();
8827 EVT WideMemVT = VPSC->getMemoryVT();
8828
8829 if (OpNo == 1) {
8830 DataOp = GetWidenedVector(Op: DataOp);
8831 Index = GetWidenedVector(Op: Index);
8832 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8833 Mask = GetWidenedMask(Mask, EC: WideEC);
8834 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8835 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8836 } else if (OpNo == 3) {
8837 // Just widen the index. It's allowed to have extra elements.
8838 Index = GetWidenedVector(Op: Index);
8839 } else
8840 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8841
8842 SDValue Ops[] = {
8843 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8844 VPSC->getVectorLength()};
8845 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8846 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8847}
8848
8849SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8850 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8851 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8852 SDLoc dl(N);
8853 EVT VT = N->getValueType(ResNo: 0);
8854
8855 // WARNING: In this code we widen the compare instruction with garbage.
8856 // This garbage may contain denormal floats which may be slow. Is this a real
8857 // concern ? Should we zero the unused lanes if this is a float compare ?
8858
8859 // Get a new SETCC node to compare the newly widened operands.
8860 // Only some of the compared elements are legal.
8861 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8862 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8863 if (VT.getScalarType() == MVT::i1)
8864 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8865 EC: SVT.getVectorElementCount());
8866
8867 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8868 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8869
8870 // Extract the needed results from the result vector.
8871 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8872 VT: SVT.getVectorElementType(),
8873 EC: VT.getVectorElementCount());
8874 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8875
8876 EVT OpVT = N->getOperand(Num: 0).getValueType();
8877 ISD::NodeType ExtendCode =
8878 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8879 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8880}
8881
8882SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8883 SDValue Chain = N->getOperand(Num: 0);
8884 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8885 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8886 SDValue CC = N->getOperand(Num: 3);
8887 SDLoc dl(N);
8888
8889 EVT VT = N->getValueType(ResNo: 0);
8890 EVT EltVT = VT.getVectorElementType();
8891 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8892 unsigned NumElts = VT.getVectorNumElements();
8893
8894 // Unroll into a build vector.
8895 SmallVector<SDValue, 8> Scalars(NumElts);
8896 SmallVector<SDValue, 8> Chains(NumElts);
8897
8898 for (unsigned i = 0; i != NumElts; ++i) {
8899 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8900 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8901
8902 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8903 Ops: {Chain, LHSElem, RHSElem, CC});
8904 Chains[i] = Scalars[i].getValue(R: 1);
8905 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8906 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8907 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8908 }
8909
8910 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8911 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8912
8913 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8914}
8915
8916static unsigned getExtendForIntVecReduction(unsigned Opc) {
8917 switch (Opc) {
8918 default:
8919 llvm_unreachable("Expected integer vector reduction");
8920 case ISD::VECREDUCE_ADD:
8921 case ISD::VECREDUCE_MUL:
8922 case ISD::VECREDUCE_AND:
8923 case ISD::VECREDUCE_OR:
8924 case ISD::VECREDUCE_XOR:
8925 return ISD::ANY_EXTEND;
8926 case ISD::VECREDUCE_SMAX:
8927 case ISD::VECREDUCE_SMIN:
8928 return ISD::SIGN_EXTEND;
8929 case ISD::VECREDUCE_UMAX:
8930 case ISD::VECREDUCE_UMIN:
8931 return ISD::ZERO_EXTEND;
8932 }
8933}
8934
8935SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8936 SDLoc dl(N);
8937 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8938 EVT VT = N->getValueType(ResNo: 0);
8939 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8940 EVT WideVT = Op.getValueType();
8941 EVT ElemVT = OrigVT.getVectorElementType();
8942 SDNodeFlags Flags = N->getFlags();
8943
8944 unsigned Opc = N->getOpcode();
8945 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8946 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8947 assert(NeutralElem && "Neutral element must exist");
8948
8949 // Pad the vector with the neutral element.
8950 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8951 unsigned WideElts = WideVT.getVectorMinNumElements();
8952
8953 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8954 // needing to pad the source vector, because the inactive lanes can simply be
8955 // disabled and not contribute to the result.
8956 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8957 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8958 SDValue Start = NeutralElem;
8959 if (VT.isInteger())
8960 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8961 assert(Start.getValueType() == VT);
8962 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8963 EC: WideVT.getVectorElementCount());
8964 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8965 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8966 EC: OrigVT.getVectorElementCount());
8967 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8968 }
8969
8970 if (WideVT.isScalableVector()) {
8971 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8972 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8973 EC: ElementCount::getScalable(MinVal: GCD));
8974 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8975 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8976 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8977 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8978 }
8979
8980 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8981 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8982
8983 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8984}
8985
8986SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8987 SDLoc dl(N);
8988 SDValue AccOp = N->getOperand(Num: 0);
8989 SDValue VecOp = N->getOperand(Num: 1);
8990 SDValue Op = GetWidenedVector(Op: VecOp);
8991
8992 EVT VT = N->getValueType(ResNo: 0);
8993 EVT OrigVT = VecOp.getValueType();
8994 EVT WideVT = Op.getValueType();
8995 EVT ElemVT = OrigVT.getVectorElementType();
8996 SDNodeFlags Flags = N->getFlags();
8997
8998 unsigned Opc = N->getOpcode();
8999 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
9000 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
9001
9002 // Pad the vector with the neutral element.
9003 unsigned OrigElts = OrigVT.getVectorMinNumElements();
9004 unsigned WideElts = WideVT.getVectorMinNumElements();
9005
9006 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
9007 // needing to pad the source vector, because the inactive lanes can simply be
9008 // disabled and not contribute to the result.
9009 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
9010 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
9011 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
9012 EC: WideVT.getVectorElementCount());
9013 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
9014 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
9015 EC: OrigVT.getVectorElementCount());
9016 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
9017 }
9018
9019 if (WideVT.isScalableVector()) {
9020 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
9021 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
9022 EC: ElementCount::getScalable(MinVal: GCD));
9023 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
9024 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
9025 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
9026 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
9027 }
9028
9029 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
9030 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
9031
9032 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
9033}
9034
9035SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
9036 assert(N->isVPOpcode() && "Expected VP opcode");
9037
9038 SDLoc dl(N);
9039 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
9040 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
9041 EC: Op.getValueType().getVectorElementCount());
9042
9043 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
9044 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
9045 Flags: N->getFlags());
9046}
9047
9048SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
9049 // This only gets called in the case that the left and right inputs and
9050 // result are of a legal odd vector type, and the condition is illegal i1 of
9051 // the same odd width that needs widening.
9052 EVT VT = N->getValueType(ResNo: 0);
9053 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
9054
9055 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
9056 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
9057 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
9058 SDLoc DL(N);
9059
9060 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
9061 N2: LeftIn, N3: RightIn);
9062 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
9063}
9064
9065SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
9066 SDLoc DL(N);
9067 SDValue Source = N->getOperand(Num: 0);
9068 EVT SourceVT = Source.getValueType();
9069 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
9070
9071 SDValue WideSource;
9072 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
9073 WideSource = GetWidenedVector(Op: Source);
9074 } else {
9075 // Pad the widened portion with all-ones so the extra lanes appear as
9076 // active (non-zero) elements and do not contribute trailing zeros.
9077 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: WideVT);
9078 if (WideVT.isFixedLengthVector() &&
9079 getTypeAction(VT: WideVT) == TargetLowering::TypeSplitVector) {
9080 WideSource = GetWidenedVector(Op: Source);
9081 unsigned WideElts = WideVT.getVectorNumElements();
9082 SmallVector<int> Mask(WideElts);
9083 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
9084 for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
9085 Mask[I] += WideElts;
9086 WideSource = DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: WideSource, N2: AllOnes, Mask);
9087 } else {
9088 WideSource = DAG.getInsertSubvector(DL, Vec: AllOnes, SubVec: Source, Idx: 0);
9089 }
9090 }
9091
9092 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0), Operand: WideSource,
9093 Flags: N->getFlags());
9094}
9095
9096SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
9097 SDLoc DL(N);
9098 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
9099 EVT SrcVT = Source.getValueType();
9100 SDValue Mask =
9101 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
9102
9103 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
9104 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
9105}
9106
9107SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
9108 SDLoc DL(N);
9109 SDValue Mask = N->getOperand(Num: 0);
9110 EVT OrigMaskVT = Mask.getValueType();
9111 SDValue WideMask = GetWidenedVector(Op: Mask);
9112 EVT WideMaskVT = WideMask.getValueType();
9113
9114 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
9115 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
9116 unsigned WideElts = WideMaskVT.getVectorNumElements();
9117 if (OrigElts != WideElts) {
9118 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
9119 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
9120 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9121 }
9122
9123 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
9124 Operand: WideMask);
9125}
9126
9127SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
9128 if (OpNo == 0) {
9129 SDLoc DL(N);
9130 EVT ResVT = N->getValueType(ResNo: 0);
9131 EVT SourceVT = N->getOperand(Num: 0).getValueType();
9132 EVT WideSourceVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
9133 EVT WidenVT =
9134 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
9135 EC: WideSourceVT.getVectorElementCount());
9136
9137 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
9138 SubVec: N->getOperand(Num: 0), Idx: 0);
9139 SDValue WideMask = DAG.getInsertSubvector(
9140 DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT), SubVec: N->getOperand(Num: 2), Idx: 0);
9141 SDValue WideMatch = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
9142 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
9143 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideMatch, Idx: 0);
9144 }
9145
9146 // Note: The Mask (OpNo == 2) should be widened with the result.
9147 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
9148
9149 SDLoc DL(N);
9150 SDValue Needle = N->getOperand(Num: 1);
9151 EVT NeedleVT = Needle.getValueType();
9152 if (NeedleVT.getVectorNumElements() == 1)
9153 return TLI.expandVectorMatch(N, DAG);
9154
9155 EVT WidenNeedleVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: NeedleVT);
9156
9157 SDValue Fill =
9158 DAG.getExtractVectorElt(DL, VT: NeedleVT.getVectorElementType(), Vec: Needle, Idx: 0);
9159 SDValue WideNeedle = DAG.getSplatVector(VT: WidenNeedleVT, DL, Op: Fill);
9160 WideNeedle = DAG.getInsertSubvector(DL, Vec: WideNeedle, SubVec: Needle, Idx: 0);
9161
9162 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0),
9163 N1: N->getOperand(Num: 0), N2: WideNeedle, N3: N->getOperand(Num: 2),
9164 Flags: N->getFlags());
9165}
9166
9167//===----------------------------------------------------------------------===//
9168// Vector Widening Utilities
9169//===----------------------------------------------------------------------===//
9170
9171// Utility function to find the type to chop up a widen vector for load/store
9172// TLI: Target lowering used to determine legal types.
9173// Width: Width left need to load/store.
9174// WidenVT: The widen vector type to load to/store from
9175// Align: If 0, don't allow use of a wider type
9176// WidenEx: If Align is not 0, the amount additional we can load/store from.
9177
9178static std::optional<EVT> findMemType(SelectionDAG &DAG,
9179 const TargetLowering &TLI, unsigned Width,
9180 EVT WidenVT, unsigned Align = 0,
9181 unsigned WidenEx = 0) {
9182 EVT WidenEltVT = WidenVT.getVectorElementType();
9183 const bool Scalable = WidenVT.isScalableVector();
9184 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
9185 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
9186 unsigned AlignInBits = Align*8;
9187
9188 EVT RetVT = WidenEltVT;
9189 // Don't bother looking for an integer type if the vector is scalable, skip
9190 // to vector types.
9191 if (!Scalable) {
9192 // If we have one element to load/store, return it.
9193 if (Width == WidenEltWidth)
9194 return RetVT;
9195
9196 // See if there is larger legal integer than the element type to load/store.
9197 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
9198 unsigned MemVTWidth = MemVT.getSizeInBits();
9199 if (MemVT.getSizeInBits() <= WidenEltWidth)
9200 break;
9201 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
9202 if ((Action == TargetLowering::TypeLegal ||
9203 Action == TargetLowering::TypePromoteInteger) &&
9204 (WidenWidth % MemVTWidth) == 0 &&
9205 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
9206 (MemVTWidth <= Width ||
9207 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9208 if (MemVTWidth == WidenWidth)
9209 return MemVT;
9210 RetVT = MemVT;
9211 break;
9212 }
9213 }
9214 }
9215
9216 // See if there is a larger vector type to load/store that has the same vector
9217 // element type and is evenly divisible with the WidenVT.
9218 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
9219 // Skip vector MVTs which don't match the scalable property of WidenVT.
9220 if (Scalable != MemVT.isScalableVector())
9221 continue;
9222 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
9223 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
9224 if ((Action == TargetLowering::TypeLegal ||
9225 Action == TargetLowering::TypePromoteInteger) &&
9226 WidenEltVT == MemVT.getVectorElementType() &&
9227 (WidenWidth % MemVTWidth) == 0 &&
9228 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
9229 (MemVTWidth <= Width ||
9230 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
9231 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
9232 return MemVT;
9233 }
9234 }
9235
9236 // Using element-wise loads and stores for widening operations is not
9237 // supported for scalable vectors
9238 if (Scalable)
9239 return std::nullopt;
9240
9241 return RetVT;
9242}
9243
9244// Builds a vector type from scalar loads
9245// VecTy: Resulting Vector type
9246// LDOps: Load operators to build a vector type
9247// [Start,End) the list of loads to use.
9248static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
9249 SmallVectorImpl<SDValue> &LdOps,
9250 unsigned Start, unsigned End) {
9251 SDLoc dl(LdOps[Start]);
9252 EVT LdTy = LdOps[Start].getValueType();
9253 unsigned Width = VecTy.getSizeInBits();
9254 unsigned NumElts = Width / LdTy.getSizeInBits();
9255 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
9256
9257 unsigned Idx = 1;
9258 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
9259
9260 for (unsigned i = Start + 1; i != End; ++i) {
9261 EVT NewLdTy = LdOps[i].getValueType();
9262 if (NewLdTy != LdTy) {
9263 NumElts = Width / NewLdTy.getSizeInBits();
9264 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
9265 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
9266 // Readjust position and vector position based on new load type.
9267 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
9268 LdTy = NewLdTy;
9269 }
9270 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
9271 }
9272 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
9273}
9274
9275SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
9276 LoadSDNode *LD) {
9277 // The strategy assumes that we can efficiently load power-of-two widths.
9278 // The routine chops the vector into the largest vector loads with the same
9279 // element type or scalar loads and then recombines it to the widen vector
9280 // type.
9281 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9282 EVT LdVT = LD->getMemoryVT();
9283 SDLoc dl(LD);
9284 assert(LdVT.isVector() && WidenVT.isVector());
9285 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9286 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
9287
9288 // Load information
9289 SDValue Chain = LD->getChain();
9290 SDValue BasePtr = LD->getBasePtr();
9291 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9292 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9293
9294 TypeSize LdWidth = LdVT.getSizeInBits();
9295 TypeSize WidenWidth = WidenVT.getSizeInBits();
9296 TypeSize WidthDiff = WidenWidth - LdWidth;
9297 // Allow wider loads if they are sufficiently aligned to avoid memory faults
9298 // and if the original load is simple.
9299 unsigned LdAlign =
9300 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
9301
9302 // Find the vector type that can load from.
9303 std::optional<EVT> FirstVT =
9304 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
9305 WidenEx: WidthDiff.getKnownMinValue());
9306
9307 if (!FirstVT)
9308 return SDValue();
9309
9310 SmallVector<EVT, 8> MemVTs;
9311 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
9312
9313 // Unless we're able to load in one instruction we must work out how to load
9314 // the remainder.
9315 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
9316 std::optional<EVT> NewVT = FirstVT;
9317 TypeSize RemainingWidth = LdWidth;
9318 TypeSize NewVTWidth = FirstVTWidth;
9319 do {
9320 RemainingWidth -= NewVTWidth;
9321 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
9322 // The current type we are using is too large. Find a better size.
9323 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
9324 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
9325 if (!NewVT)
9326 return SDValue();
9327 NewVTWidth = NewVT->getSizeInBits();
9328 }
9329 MemVTs.push_back(Elt: *NewVT);
9330 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
9331 }
9332
9333 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9334 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9335 LdChain.push_back(Elt: LdOp.getValue(R: 1));
9336
9337 // Check if we can load the element with one instruction.
9338 if (MemVTs.empty())
9339 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
9340 DAG);
9341
9342 // Load vector by using multiple loads from largest vector to scalar.
9343 SmallVector<SDValue, 16> LdOps;
9344 LdOps.push_back(Elt: LdOp);
9345
9346 uint64_t ScaledOffset = 0;
9347 MachinePointerInfo MPI = LD->getPointerInfo();
9348
9349 // First incremement past the first load.
9350 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
9351 ScaledOffset: &ScaledOffset);
9352
9353 for (EVT MemVT : MemVTs) {
9354 Align NewAlign = ScaledOffset == 0
9355 ? LD->getBaseAlign()
9356 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
9357 SDValue L = DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags,
9358 Metadata);
9359
9360 LdOps.push_back(Elt: L);
9361 LdChain.push_back(Elt: L.getValue(R: 1));
9362 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
9363 }
9364
9365 // Build the vector from the load operations.
9366 unsigned End = LdOps.size();
9367 if (!LdOps[0].getValueType().isVector())
9368 // All the loads are scalar loads.
9369 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
9370
9371 // If the load contains vectors, build the vector using concat vector.
9372 // All of the vectors used to load are power-of-2, and the scalar loads can be
9373 // combined to make a power-of-2 vector.
9374 SmallVector<SDValue, 16> ConcatOps(End);
9375 int i = End - 1;
9376 int Idx = End;
9377 EVT LdTy = LdOps[i].getValueType();
9378 // First, combine the scalar loads to a vector.
9379 if (!LdTy.isVector()) {
9380 for (--i; i >= 0; --i) {
9381 LdTy = LdOps[i].getValueType();
9382 if (LdTy.isVector())
9383 break;
9384 }
9385 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
9386 }
9387
9388 ConcatOps[--Idx] = LdOps[i];
9389 for (--i; i >= 0; --i) {
9390 EVT NewLdTy = LdOps[i].getValueType();
9391 if (NewLdTy != LdTy) {
9392 // Create a larger vector.
9393 TypeSize LdTySize = LdTy.getSizeInBits();
9394 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
9395 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
9396 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
9397 unsigned NumOps =
9398 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
9399 SmallVector<SDValue, 16> WidenOps(NumOps);
9400 unsigned j = 0;
9401 for (; j != End-Idx; ++j)
9402 WidenOps[j] = ConcatOps[Idx+j];
9403 for (; j != NumOps; ++j)
9404 WidenOps[j] = DAG.getPOISON(VT: LdTy);
9405
9406 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
9407 Ops: WidenOps);
9408 Idx = End - 1;
9409 LdTy = NewLdTy;
9410 }
9411 ConcatOps[--Idx] = LdOps[i];
9412 }
9413
9414 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9415 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
9416 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
9417
9418 // We need to fill the rest with undefs to build the vector.
9419 unsigned NumOps =
9420 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9421 SmallVector<SDValue, 16> WidenOps(NumOps);
9422 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
9423 {
9424 unsigned i = 0;
9425 for (; i != End-Idx; ++i)
9426 WidenOps[i] = ConcatOps[Idx+i];
9427 for (; i != NumOps; ++i)
9428 WidenOps[i] = UndefVal;
9429 }
9430 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
9431}
9432
9433SDValue
9434DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9435 LoadSDNode *LD,
9436 ISD::LoadExtType ExtType) {
9437 // For extension loads, it may not be more efficient to chop up the vector
9438 // and then extend it. Instead, we unroll the load and build a new vector.
9439 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9440 EVT LdVT = LD->getMemoryVT();
9441 SDLoc dl(LD);
9442 assert(LdVT.isVector() && WidenVT.isVector());
9443 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9444
9445 // Load information
9446 SDValue Chain = LD->getChain();
9447 SDValue BasePtr = LD->getBasePtr();
9448 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9449 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
9450
9451 if (LdVT.isScalableVector())
9452 return SDValue();
9453
9454 EVT EltVT = WidenVT.getVectorElementType();
9455 EVT LdEltVT = LdVT.getVectorElementType();
9456 unsigned NumElts = LdVT.getVectorNumElements();
9457
9458 // Load each element and widen.
9459 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9460 SmallVector<SDValue, 16> Ops(WidenNumElts);
9461 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9462 Ops[0] =
9463 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9464 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9465 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
9466 unsigned i = 0, Offset = Increment;
9467 for (i=1; i < NumElts; ++i, Offset += Increment) {
9468 SDValue NewBasePtr =
9469 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
9470 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
9471 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
9472 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
9473 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
9474 }
9475
9476 // Fill the rest with undefs.
9477 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9478 for (; i != WidenNumElts; ++i)
9479 Ops[i] = UndefVal;
9480
9481 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
9482}
9483
9484bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9485 StoreSDNode *ST) {
9486 // The strategy assumes that we can efficiently store power-of-two widths.
9487 // The routine chops the vector into the largest vector stores with the same
9488 // element type or scalar stores.
9489 SDValue Chain = ST->getChain();
9490 SDValue BasePtr = ST->getBasePtr();
9491 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9492 MMOMetadata Metadata = ST->getMMOMetadataForSubAccess();
9493 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
9494 SDLoc dl(ST);
9495
9496 EVT StVT = ST->getMemoryVT();
9497 TypeSize StWidth = StVT.getSizeInBits();
9498 EVT ValVT = ValOp.getValueType();
9499 TypeSize ValWidth = ValVT.getSizeInBits();
9500 EVT ValEltVT = ValVT.getVectorElementType();
9501 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9502 assert(StVT.getVectorElementType() == ValEltVT);
9503 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9504 "Mismatch between store and value types");
9505
9506 int Idx = 0; // current index to store
9507
9508 MachinePointerInfo MPI = ST->getPointerInfo();
9509 uint64_t ScaledOffset = 0;
9510
9511 // A breakdown of how to widen this vector store. Each element of the vector
9512 // is a memory VT combined with the number of times it is to be stored to,
9513 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9514 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
9515
9516 while (StWidth.isNonZero()) {
9517 // Find the largest vector type we can store with.
9518 std::optional<EVT> NewVT =
9519 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
9520 if (!NewVT)
9521 return false;
9522 MemVTs.push_back(Elt: {*NewVT, 0});
9523 TypeSize NewVTWidth = NewVT->getSizeInBits();
9524
9525 do {
9526 StWidth -= NewVTWidth;
9527 MemVTs.back().second++;
9528 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
9529 }
9530
9531 for (const auto &Pair : MemVTs) {
9532 EVT NewVT = Pair.first;
9533 unsigned Count = Pair.second;
9534 TypeSize NewVTWidth = NewVT.getSizeInBits();
9535
9536 if (NewVT.isVector()) {
9537 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9538 do {
9539 Align NewAlign = ScaledOffset == 0
9540 ? ST->getBaseAlign()
9541 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
9542 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
9543 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
9544 MMOFlags, Metadata);
9545 StChain.push_back(Elt: PartStore);
9546
9547 Idx += NumVTElts;
9548 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
9549 ScaledOffset: &ScaledOffset);
9550 } while (--Count);
9551 } else {
9552 // Cast the vector to the scalar type we can store.
9553 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9554 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
9555 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
9556 // Readjust index position based on new vector type.
9557 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9558 do {
9559 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
9560 SDValue PartStore =
9561 DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: ST->getBaseAlign(),
9562 MMOFlags, Metadata);
9563 StChain.push_back(Elt: PartStore);
9564
9565 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
9566 } while (--Count);
9567 // Restore index back to be relative to the original widen element type.
9568 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9569 }
9570 }
9571
9572 return true;
9573}
9574
9575/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9576/// input vector must have the same element type as NVT.
9577/// FillWithZeroes specifies that the vector should be widened with zeroes.
9578SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9579 bool FillWithZeroes) {
9580 // Note that InOp might have been widened so it might already have
9581 // the right width or it might need be narrowed.
9582 EVT InVT = InOp.getValueType();
9583 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
9584 "input and widen element type must match");
9585 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9586 "cannot modify scalable vectors in this way");
9587 SDLoc dl(InOp);
9588
9589 // Check if InOp already has the right width.
9590 if (InVT == NVT)
9591 return InOp;
9592
9593 ElementCount InEC = InVT.getVectorElementCount();
9594 ElementCount WidenEC = NVT.getVectorElementCount();
9595 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
9596 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
9597 SmallVector<SDValue, 16> Ops(NumConcat);
9598 SDValue FillVal =
9599 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
9600 Ops[0] = InOp;
9601 for (unsigned i = 1; i != NumConcat; ++i)
9602 Ops[i] = FillVal;
9603
9604 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9605 }
9606
9607 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
9608 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
9609
9610 if (NVT.isScalableVector() && InVT.isScalableVector()) {
9611 // Split the input into the largest equal-sized scalable subvectors.
9612 unsigned InNumElts = InVT.getVectorMinNumElements();
9613 unsigned NewNumElts = NVT.getVectorMinNumElements();
9614 unsigned CommonFactor = std::gcd(m: InNumElts, n: NewNumElts);
9615 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(),
9616 EC: ElementCount::getScalable(MinVal: CommonFactor));
9617
9618 SmallVector<SDValue, 16> Ops;
9619 unsigned NumCopiedParts = std::min(a: InNumElts, b: NewNumElts) / CommonFactor;
9620 for (unsigned I = 0; I != NumCopiedParts; ++I)
9621 Ops.push_back(
9622 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: I * CommonFactor));
9623
9624 unsigned NumResultParts = NewNumElts / CommonFactor;
9625 if (NumResultParts > NumCopiedParts) {
9626 SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: PartVT)
9627 : DAG.getPOISON(VT: PartVT);
9628 Ops.append(NumInputs: NumResultParts - NumCopiedParts, Elt: FillVal);
9629 }
9630
9631 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9632 }
9633
9634 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9635 "Scalable vectors should have been handled already.");
9636
9637 unsigned InNumElts = InEC.getFixedValue();
9638 unsigned WidenNumElts = WidenEC.getFixedValue();
9639
9640 // Fall back to extract and build (+ mask, if padding with zeros).
9641 SmallVector<SDValue, 16> Ops(WidenNumElts);
9642 EVT EltVT = NVT.getVectorElementType();
9643 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
9644 unsigned Idx;
9645 for (Idx = 0; Idx < MinNumElts; ++Idx)
9646 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
9647
9648 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9649 for (; Idx < WidenNumElts; ++Idx)
9650 Ops[Idx] = UndefVal;
9651
9652 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
9653 if (!FillWithZeroes)
9654 return Widened;
9655
9656 assert(NVT.isInteger() &&
9657 "We expect to never want to FillWithZeroes for non-integral types.");
9658
9659 SmallVector<SDValue, 16> MaskOps;
9660 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
9661 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
9662
9663 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
9664 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
9665}
9666