1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file performs vector type splitting and scalarization for LegalizeTypes.
10// Scalarization is the act of changing a computation in an illegal one-element
11// vector type to be a computation in its scalar element type. For example,
12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15// types.
16// Splitting is the act of changing a computation in an invalid vector type to
17// be a computation in two vectors of half the size. For example, implementing
18// <128 x f32> operations in terms of two <64 x f32> operations.
19//
20//===----------------------------------------------------------------------===//
21
22#include "LegalizeTypes.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/Analysis/MemoryLocation.h"
25#include "llvm/Analysis/VectorUtils.h"
26#include "llvm/CodeGen/ISDOpcodes.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/TypeSize.h"
30#include "llvm/Support/raw_ostream.h"
31#include <numeric>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "legalize-types"
36
37//===----------------------------------------------------------------------===//
38// Result Vector Scalarization: <1 x ty> -> ty.
39//===----------------------------------------------------------------------===//
40
41void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": ";
43 N->dump(&DAG));
44 SDValue R = SDValue();
45
46 // See if the target wants to custom expand this node.
47 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
48 return;
49
50 switch (N->getOpcode()) {
51 default:
52#ifndef NDEBUG
53 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
54 N->dump(&DAG);
55 dbgs() << "\n";
56#endif
57 report_fatal_error(reason: "Do not know how to scalarize the result of this "
58 "operator!\n");
59
60 case ISD::LOOP_DEPENDENCE_WAR_MASK:
61 case ISD::LOOP_DEPENDENCE_RAW_MASK:
62 R = ScalarizeVecRes_LOOP_DEPENDENCE_MASK(N);
63 break;
64 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
65 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
66 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
67 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
68 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
69 case ISD::CONVERT_FROM_ARBITRARY_FP:
70 R = ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(N);
71 break;
72 case ISD::CONVERT_TO_ARBITRARY_FP:
73 R = ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(N);
74 break;
75 case ISD::AssertZext:
76 case ISD::AssertSext:
77 case ISD::FPOWI:
78 case ISD::AssertNoFPClass:
79 R = ScalarizeVecRes_UnaryOpWithExtraInput(N);
80 break;
81 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
82 case ISD::ATOMIC_LOAD:
83 R = ScalarizeVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
84 break;
85 case ISD::LOAD: R = ScalarizeVecRes_LOAD(N: cast<LoadSDNode>(Val: N));break;
86 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
87 case ISD::VECTOR_DEINTERLEAVE:
88 case ISD::VECTOR_INTERLEAVE:
89 R = ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
90 break;
91 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
92 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
93 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
94 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
95 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
96 case ISD::VECTOR_MATCH:
97 R = ScalarizeVecRes_VECTOR_MATCH(N);
98 break;
99 case ISD::POISON:
100 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
101 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
102 case ISD::IS_FPCLASS: R = ScalarizeVecRes_IS_FPCLASS(N); break;
103 case ISD::ANY_EXTEND_VECTOR_INREG:
104 case ISD::SIGN_EXTEND_VECTOR_INREG:
105 case ISD::ZERO_EXTEND_VECTOR_INREG:
106 R = ScalarizeVecRes_VecInregOp(N);
107 break;
108 case ISD::ABS:
109 case ISD::ABS_MIN_POISON:
110 case ISD::ANY_EXTEND:
111 case ISD::BITREVERSE:
112 case ISD::BSWAP:
113 case ISD::CTLZ:
114 case ISD::CTLZ_ZERO_POISON:
115 case ISD::CTPOP:
116 case ISD::CTTZ:
117 case ISD::CTTZ_ZERO_POISON:
118 case ISD::FABS:
119 case ISD::FACOS:
120 case ISD::FASIN:
121 case ISD::FATAN:
122 case ISD::FCEIL:
123 case ISD::FCOS:
124 case ISD::FCOSH:
125 case ISD::FEXP:
126 case ISD::FEXP2:
127 case ISD::FEXP10:
128 case ISD::FFLOOR:
129 case ISD::FLOG:
130 case ISD::FLOG10:
131 case ISD::FLOG2:
132 case ISD::FNEARBYINT:
133 case ISD::FNEG:
134 case ISD::FREEZE:
135 case ISD::ARITH_FENCE:
136 case ISD::FP_EXTEND:
137 case ISD::FP_TO_SINT:
138 case ISD::FP_TO_UINT:
139 case ISD::FRINT:
140 case ISD::LRINT:
141 case ISD::LLRINT:
142 case ISD::FROUND:
143 case ISD::FROUNDEVEN:
144 case ISD::LROUND:
145 case ISD::LLROUND:
146 case ISD::FSIN:
147 case ISD::FSINH:
148 case ISD::FSQRT:
149 case ISD::FTAN:
150 case ISD::FTANH:
151 case ISD::FTRUNC:
152 case ISD::SIGN_EXTEND:
153 case ISD::SINT_TO_FP:
154 case ISD::TRUNCATE:
155 case ISD::UINT_TO_FP:
156 case ISD::ZERO_EXTEND:
157 case ISD::FCANONICALIZE:
158 R = ScalarizeVecRes_UnaryOp(N);
159 break;
160 case ISD::ADDRSPACECAST:
161 R = ScalarizeVecRes_ADDRSPACECAST(N);
162 break;
163 case ISD::FMODF:
164 case ISD::FFREXP:
165 case ISD::FSINCOS:
166 case ISD::FSINCOSPI:
167 R = ScalarizeVecRes_UnaryOpWithTwoResults(N, ResNo);
168 break;
169 case ISD::ADD:
170 case ISD::AND:
171 case ISD::AVGCEILS:
172 case ISD::AVGCEILU:
173 case ISD::AVGFLOORS:
174 case ISD::AVGFLOORU:
175 case ISD::FADD:
176 case ISD::FCOPYSIGN:
177 case ISD::FDIV:
178 case ISD::FMUL:
179 case ISD::FMINNUM:
180 case ISD::FMAXNUM:
181 case ISD::FMINNUM_IEEE:
182 case ISD::FMAXNUM_IEEE:
183 case ISD::FMINIMUM:
184 case ISD::FMAXIMUM:
185 case ISD::FMINIMUMNUM:
186 case ISD::FMAXIMUMNUM:
187 case ISD::FLDEXP:
188 case ISD::ABDS:
189 case ISD::ABDU:
190 case ISD::SMIN:
191 case ISD::SMAX:
192 case ISD::UMIN:
193 case ISD::UMAX:
194
195 case ISD::SADDSAT:
196 case ISD::UADDSAT:
197 case ISD::SSUBSAT:
198 case ISD::USUBSAT:
199 case ISD::SSHLSAT:
200 case ISD::USHLSAT:
201
202 case ISD::FPOW:
203 case ISD::FATAN2:
204 case ISD::FREM:
205 case ISD::FSUB:
206 case ISD::MUL:
207 case ISD::MULHS:
208 case ISD::MULHU:
209 case ISD::OR:
210 case ISD::SDIV:
211 case ISD::SREM:
212 case ISD::SUB:
213 case ISD::UDIV:
214 case ISD::UREM:
215 case ISD::XOR:
216 case ISD::SHL:
217 case ISD::SRA:
218 case ISD::SRL:
219 case ISD::ROTL:
220 case ISD::ROTR:
221 case ISD::CLMUL:
222 case ISD::CLMULR:
223 case ISD::CLMULH:
224 case ISD::PEXT:
225 case ISD::PDEP:
226 R = ScalarizeVecRes_BinOp(N);
227 break;
228
229 case ISD::MASKED_UDIV:
230 case ISD::MASKED_SDIV:
231 case ISD::MASKED_UREM:
232 case ISD::MASKED_SREM:
233 R = ScalarizeVecRes_MaskedBinOp(N);
234 break;
235
236 case ISD::SCMP:
237 case ISD::UCMP:
238 R = ScalarizeVecRes_CMP(N);
239 break;
240
241 case ISD::FMA:
242 case ISD::FSHL:
243 case ISD::FSHR:
244 R = ScalarizeVecRes_TernaryOp(N);
245 break;
246
247#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
248 case ISD::STRICT_##DAGN:
249#include "llvm/IR/ConstrainedOps.def"
250 R = ScalarizeVecRes_StrictFPOp(N);
251 break;
252
253 case ISD::FP_TO_UINT_SAT:
254 case ISD::FP_TO_SINT_SAT:
255 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
256 break;
257
258 case ISD::UADDO:
259 case ISD::SADDO:
260 case ISD::USUBO:
261 case ISD::SSUBO:
262 case ISD::UMULO:
263 case ISD::SMULO:
264 R = ScalarizeVecRes_OverflowOp(N, ResNo);
265 break;
266 case ISD::SMULFIX:
267 case ISD::SMULFIXSAT:
268 case ISD::UMULFIX:
269 case ISD::UMULFIXSAT:
270 case ISD::SDIVFIX:
271 case ISD::SDIVFIXSAT:
272 case ISD::UDIVFIX:
273 case ISD::UDIVFIXSAT:
274 R = ScalarizeVecRes_FIX(N);
275 break;
276 }
277
278 // If R is null, the sub-method took care of registering the result.
279 if (R.getNode())
280 SetScalarizedVector(Op: SDValue(N, ResNo), Result: R);
281}
282
283SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
284 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
285 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
286 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
287 VT: LHS.getValueType(), N1: LHS, N2: RHS, Flags: N->getFlags());
288}
289
290SDValue DAGTypeLegalizer::ScalarizeVecRes_MaskedBinOp(SDNode *N) {
291 SDLoc DL(N);
292 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
293 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
294 SDValue Mask = N->getOperand(Num: 2);
295 EVT MaskVT = Mask.getValueType();
296 // The vselect result and input vectors need scalarizing, but it's
297 // not a given that the mask does. For instance, in AVX512 v1i1 is legal.
298 // See the similar logic in ScalarizeVecRes_SETCC.
299 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeScalarizeVector)
300 Mask = GetScalarizedVector(Op: Mask);
301 else
302 Mask = DAG.getExtractVectorElt(DL, VT: MaskVT.getVectorElementType(), Vec: Mask, Idx: 0);
303 // Vectors may have a different boolean contents to scalars, so truncate to i1
304 // and let type legalization promote appropriately.
305 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
306 // Masked binary ops don't have UB on disabled lanes but produce poison, so
307 // use 1 as the divisor to avoid division by zero and overflow.
308 SDValue Divisor = DAG.getSelect(DL, VT: LHS.getValueType(), Cond: Mask, LHS: RHS,
309 RHS: DAG.getConstant(Val: 1, DL, VT: LHS.getValueType()));
310 return DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL,
311 VT: LHS.getValueType(), N1: LHS, N2: Divisor);
312}
313
314SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) {
315 SDLoc DL(N);
316
317 SDValue LHS = N->getOperand(Num: 0);
318 SDValue RHS = N->getOperand(Num: 1);
319 if (getTypeAction(VT: LHS.getValueType()) ==
320 TargetLowering::TypeScalarizeVector) {
321 LHS = GetScalarizedVector(Op: LHS);
322 RHS = GetScalarizedVector(Op: RHS);
323 } else {
324 EVT VT = LHS.getValueType().getVectorElementType();
325 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
326 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
327 }
328
329 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
330 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: LHS, N2: RHS);
331}
332
333SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
334 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
335 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
336 SDValue Op2 = GetScalarizedVector(Op: N->getOperand(Num: 2));
337 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
338 N3: Op2, Flags: N->getFlags());
339}
340
341SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
342 SDValue Op0 = GetScalarizedVector(Op: N->getOperand(Num: 0));
343 SDValue Op1 = GetScalarizedVector(Op: N->getOperand(Num: 1));
344 SDValue Op2 = N->getOperand(Num: 2);
345 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op0.getValueType(), N1: Op0, N2: Op1,
346 N3: Op2, Flags: N->getFlags());
347}
348
349SDValue
350DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N,
351 unsigned ResNo) {
352 assert(N->getValueType(0).getVectorNumElements() == 1 &&
353 "Unexpected vector type!");
354 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
355
356 EVT VT0 = N->getValueType(ResNo: 0);
357 EVT VT1 = N->getValueType(ResNo: 1);
358 SDLoc dl(N);
359
360 SDNode *ScalarNode =
361 DAG.getNode(Opcode: N->getOpcode(), DL: dl,
362 ResultTys: {VT0.getScalarType(), VT1.getScalarType()}, Ops: Elt)
363 .getNode();
364
365 // Replace the other vector result not being explicitly scalarized here.
366 unsigned OtherNo = 1 - ResNo;
367 EVT OtherVT = N->getValueType(ResNo: OtherNo);
368 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
369 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
370 } else {
371 SDValue OtherVal = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: OtherVT,
372 Operand: SDValue(ScalarNode, OtherNo));
373 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
374 }
375
376 return SDValue(ScalarNode, ResNo);
377}
378
379SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
380 EVT VT = N->getValueType(ResNo: 0).getVectorElementType();
381 unsigned NumOpers = N->getNumOperands();
382 SDValue Chain = N->getOperand(Num: 0);
383 EVT ValueVTs[] = {VT, MVT::Other};
384 SDLoc dl(N);
385
386 SmallVector<SDValue, 4> Opers(NumOpers);
387
388 // The Chain is the first operand.
389 Opers[0] = Chain;
390
391 // Now process the remaining operands.
392 for (unsigned i = 1; i < NumOpers; ++i) {
393 SDValue Oper = N->getOperand(Num: i);
394 EVT OperVT = Oper.getValueType();
395
396 if (OperVT.isVector()) {
397 if (getTypeAction(VT: OperVT) == TargetLowering::TypeScalarizeVector)
398 Oper = GetScalarizedVector(Op: Oper);
399 else
400 Oper =
401 DAG.getExtractVectorElt(DL: dl, VT: OperVT.getVectorElementType(), Vec: Oper, Idx: 0);
402 }
403
404 Opers[i] = Oper;
405 }
406
407 SDValue Result = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: ValueVTs),
408 Ops: Opers, Flags: N->getFlags());
409
410 // Legalize the chain result - switch anything that used the old chain to
411 // use the new one.
412 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
413 return Result;
414}
415
416SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
417 unsigned ResNo) {
418 SDLoc DL(N);
419 EVT ResVT = N->getValueType(ResNo: 0);
420 EVT OvVT = N->getValueType(ResNo: 1);
421
422 SDValue ScalarLHS, ScalarRHS;
423 if (getTypeAction(VT: ResVT) == TargetLowering::TypeScalarizeVector) {
424 ScalarLHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
425 ScalarRHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
426 } else {
427 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
428 DAG.ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: ElemsLHS);
429 DAG.ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: ElemsRHS);
430 ScalarLHS = ElemsLHS[0];
431 ScalarRHS = ElemsRHS[0];
432 }
433
434 SDVTList ScalarVTs = DAG.getVTList(
435 VT1: ResVT.getVectorElementType(), VT2: OvVT.getVectorElementType());
436 SDNode *ScalarNode = DAG.getNode(Opcode: N->getOpcode(), DL, VTList: ScalarVTs,
437 Ops: {ScalarLHS, ScalarRHS}, Flags: N->getFlags())
438 .getNode();
439
440 // Replace the other vector result not being explicitly scalarized here.
441 unsigned OtherNo = 1 - ResNo;
442 EVT OtherVT = N->getValueType(ResNo: OtherNo);
443 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeScalarizeVector) {
444 SetScalarizedVector(Op: SDValue(N, OtherNo), Result: SDValue(ScalarNode, OtherNo));
445 } else {
446 SDValue OtherVal = DAG.getNode(
447 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: OtherVT, Operand: SDValue(ScalarNode, OtherNo));
448 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
449 }
450
451 return SDValue(ScalarNode, ResNo);
452}
453
454SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
455 unsigned ResNo) {
456 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
457 return GetScalarizedVector(Op);
458}
459
460SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
461 SDLoc DL(N);
462 // Reuse the expansion (which should scalarize).
463 SDValue Mask = TLI.expandLoopDependenceMask(N, DAG);
464 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
465 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
466 N2: DAG.getVectorIdxConstant(Val: 0, DL));
467}
468
469SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
470 SDValue Op = N->getOperand(Num: 0);
471 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypeScalarizeVector)
472 Op = GetScalarizedVector(Op);
473 EVT NewVT = N->getValueType(ResNo: 0).getVectorElementType();
474 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
475 VT: NewVT, Operand: Op);
476}
477
478SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
479 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
480 SDValue InOp = N->getOperand(Num: 0);
481 // The BUILD_VECTOR operands may be of wider element types and
482 // we may need to truncate them back to the requested return type.
483 if (EltVT.isInteger())
484 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
485 return InOp;
486}
487
488SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
489 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
490 VT: N->getValueType(ResNo: 0).getVectorElementType(),
491 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
492}
493
494SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
495 SDLoc DL(N);
496 SDValue Op = N->getOperand(Num: 0);
497 EVT OpVT = Op.getValueType();
498 // The result needs scalarizing, but it's not a given that the source does.
499 // See similar logic in ScalarizeVecRes_UnaryOp.
500 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
501 Op = GetScalarizedVector(Op);
502 } else {
503 EVT VT = OpVT.getVectorElementType();
504 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
505 }
506 return DAG.getNode(Opcode: ISD::FP_ROUND, DL,
507 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
508 N2: N->getOperand(Num: 1));
509}
510
511SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
512 SDLoc DL(N);
513 SDValue Op = N->getOperand(Num: 0);
514 EVT OpVT = Op.getValueType();
515 // The result needs scalarizing, but it's not a given that the source does.
516 // See similar logic in ScalarizeVecRes_UnaryOp.
517 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
518 Op = GetScalarizedVector(Op);
519 } else {
520 EVT VT = OpVT.getVectorElementType();
521 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
522 }
523 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL,
524 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
525 N2: N->getOperand(Num: 1));
526}
527
528SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
529 SDLoc DL(N);
530 SDValue Op = N->getOperand(Num: 0);
531 EVT OpVT = Op.getValueType();
532 // The result needs scalarizing, but it's not a given that the source does.
533 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
534 Op = GetScalarizedVector(Op);
535 } else {
536 EVT VT = OpVT.getVectorElementType();
537 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
538 }
539 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL,
540 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Op,
541 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
542}
543
544SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N) {
545 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
546 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: Op.getValueType(), N1: Op,
547 N2: N->getOperand(Num: 1));
548}
549
550SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
551 // The value to insert may have a wider type than the vector element type,
552 // so be sure to truncate it to the element type if necessary.
553 SDValue Op = N->getOperand(Num: 1);
554 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
555 if (Op.getValueType() != EltVT)
556 // FIXME: Can this happen for floating point types?
557 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: Op);
558 return Op;
559}
560
561SDValue DAGTypeLegalizer::ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N) {
562 SDValue Result = DAG.getAtomicLoad(
563 ExtType: N->getExtensionType(), dl: SDLoc(N), MemVT: N->getMemoryVT().getVectorElementType(),
564 VT: N->getValueType(ResNo: 0).getVectorElementType(), Chain: N->getChain(), Ptr: N->getBasePtr(),
565 MMO: N->getMemOperand());
566
567 // Legalize the chain result - switch anything that used the old chain to
568 // use the new one.
569 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
570 return Result;
571}
572
573SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
574 assert(N->isUnindexed() && "Indexed vector load?");
575
576 SDValue Result = DAG.getLoad(
577 AM: ISD::UNINDEXED, ExtType: N->getExtensionType(),
578 VT: N->getValueType(ResNo: 0).getVectorElementType(), dl: SDLoc(N), Chain: N->getChain(),
579 Ptr: N->getBasePtr(), Offset: DAG.getPOISON(VT: N->getBasePtr().getValueType()),
580 PtrInfo: N->getPointerInfo(), MemVT: N->getMemoryVT().getVectorElementType(),
581 Alignment: N->getBaseAlign(), MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getAAInfo());
582
583 // Legalize the chain result - switch anything that used the old chain to
584 // use the new one.
585 ReplaceValueWith(From: SDValue(N, 1), To: Result.getValue(R: 1));
586 return Result;
587}
588
589SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
590 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
591 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
592 SDValue Op = N->getOperand(Num: 0);
593 EVT OpVT = Op.getValueType();
594 SDLoc DL(N);
595 // The result needs scalarizing, but it's not a given that the source does.
596 // This is a workaround for targets where it's impossible to scalarize the
597 // result of a conversion, because the source type is legal.
598 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
599 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
600 // legal and was not scalarized.
601 // See the similar logic in ScalarizeVecRes_SETCC
602 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
603 Op = GetScalarizedVector(Op);
604 } else {
605 EVT VT = OpVT.getVectorElementType();
606 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
607 }
608 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: DestVT, Operand: Op, Flags: N->getFlags());
609}
610
611SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
612 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
613 EVT ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType();
614 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
615 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: EltVT,
616 N1: LHS, N2: DAG.getValueType(ExtVT));
617}
618
619SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
620 SDLoc DL(N);
621 SDValue Op = N->getOperand(Num: 0);
622
623 EVT OpVT = Op.getValueType();
624 EVT OpEltVT = OpVT.getVectorElementType();
625 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
626
627 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
628 Op = GetScalarizedVector(Op);
629 } else {
630 Op = DAG.getExtractVectorElt(DL, VT: OpEltVT, Vec: Op, Idx: 0);
631 }
632
633 switch (N->getOpcode()) {
634 case ISD::ANY_EXTEND_VECTOR_INREG:
635 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Op);
636 case ISD::SIGN_EXTEND_VECTOR_INREG:
637 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: EltVT, Operand: Op);
638 case ISD::ZERO_EXTEND_VECTOR_INREG:
639 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EltVT, Operand: Op);
640 }
641
642 llvm_unreachable("Illegal extend_vector_inreg opcode");
643}
644
645SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) {
646 EVT DestVT = N->getValueType(ResNo: 0).getVectorElementType();
647 SDValue Op = N->getOperand(Num: 0);
648 EVT OpVT = Op.getValueType();
649 SDLoc DL(N);
650 // The result needs scalarizing, but it's not a given that the source does.
651 // This is a workaround for targets where it's impossible to scalarize the
652 // result of a conversion, because the source type is legal.
653 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
654 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
655 // legal and was not scalarized.
656 // See the similar logic in ScalarizeVecRes_SETCC
657 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
658 Op = GetScalarizedVector(Op);
659 } else {
660 EVT VT = OpVT.getVectorElementType();
661 Op = DAG.getExtractVectorElt(DL, VT, Vec: Op, Idx: 0);
662 }
663 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
664 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
665 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
666 return DAG.getAddrSpaceCast(dl: DL, VT: DestVT, Ptr: Op, SrcAS, DestAS);
667}
668
669SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
670 // If the operand is wider than the vector element type then it is implicitly
671 // truncated. Make that explicit here.
672 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
673 SDValue InOp = N->getOperand(Num: 0);
674 if (InOp.getValueType() != EltVT)
675 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: EltVT, Operand: InOp);
676 return InOp;
677}
678
679SDValue
680DAGTypeLegalizer::ScalarizeVecRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N) {
681 assert(N->getNumValues() == N->getNumOperands() &&
682 "Expected one result per operand");
683
684 // Interleaving or deinterleaving one-element vectors leaves each result
685 // equal to the corresponding operand.
686 for (unsigned I = 0; I != N->getNumValues(); ++I)
687 SetScalarizedVector(Op: SDValue(N, I), Result: GetScalarizedVector(Op: N->getOperand(Num: I)));
688 return SDValue();
689}
690
691SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
692 SDValue Cond = N->getOperand(Num: 0);
693 EVT OpVT = Cond.getValueType();
694 SDLoc DL(N);
695 // The vselect result and true/value operands needs scalarizing, but it's
696 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
697 // See the similar logic in ScalarizeVecRes_SETCC
698 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
699 Cond = GetScalarizedVector(Op: Cond);
700 } else {
701 EVT VT = OpVT.getVectorElementType();
702 Cond = DAG.getExtractVectorElt(DL, VT, Vec: Cond, Idx: 0);
703 }
704
705 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
706 TargetLowering::BooleanContent ScalarBool =
707 TLI.getBooleanContents(isVec: false, isFloat: false);
708 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(isVec: true, isFloat: false);
709
710 // If integer and float booleans have different contents then we can't
711 // reliably optimize in all cases. There is a full explanation for this in
712 // DAGCombiner::visitSELECT() where the same issue affects folding
713 // (select C, 0, 1) to (xor C, 1).
714 if (TLI.getBooleanContents(isVec: false, isFloat: false) !=
715 TLI.getBooleanContents(isVec: false, isFloat: true)) {
716 // At least try the common case where the boolean is generated by a
717 // comparison.
718 if (Cond->getOpcode() == ISD::SETCC) {
719 EVT OpVT = Cond->getOperand(Num: 0).getValueType();
720 ScalarBool = TLI.getBooleanContents(Type: OpVT.getScalarType());
721 VecBool = TLI.getBooleanContents(Type: OpVT);
722 } else
723 ScalarBool = TargetLowering::UndefinedBooleanContent;
724 }
725
726 EVT CondVT = Cond.getValueType();
727 if (ScalarBool != VecBool) {
728 switch (ScalarBool) {
729 case TargetLowering::UndefinedBooleanContent:
730 break;
731 case TargetLowering::ZeroOrOneBooleanContent:
732 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
733 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
734 // Vector read from all ones, scalar expects a single 1 so mask.
735 Cond = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CondVT,
736 N1: Cond, N2: DAG.getConstant(Val: 1, DL: SDLoc(N), VT: CondVT));
737 break;
738 case TargetLowering::ZeroOrNegativeOneBooleanContent:
739 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
740 VecBool == TargetLowering::ZeroOrOneBooleanContent);
741 // Vector reads from a one, scalar from all ones so sign extend.
742 Cond = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SDLoc(N), VT: CondVT,
743 N1: Cond, N2: DAG.getValueType(MVT::i1));
744 break;
745 }
746 }
747
748 // Truncate the condition if needed
749 auto BoolVT = getSetCCResultType(VT: CondVT);
750 if (BoolVT.bitsLT(VT: CondVT))
751 Cond = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: BoolVT, Operand: Cond);
752
753 return DAG.getSelect(DL: SDLoc(N), VT: LHS.getValueType(), Cond, LHS,
754 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)), Flags: N->getFlags());
755}
756
757SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
758 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
759 return DAG.getSelect(DL: SDLoc(N),
760 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS,
761 RHS: GetScalarizedVector(Op: N->getOperand(Num: 2)));
762}
763
764SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
765 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
766 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: LHS.getValueType(),
767 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
768 N3: LHS, N4: GetScalarizedVector(Op: N->getOperand(Num: 3)),
769 N5: N->getOperand(Num: 4));
770}
771
772SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
773 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
774}
775
776SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
777 // Figure out if the scalar is the LHS or RHS and return it.
778 SDValue Arg = N->getOperand(Num: 2).getOperand(i: 0);
779 if (Arg.isUndef())
780 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0).getVectorElementType());
781 unsigned Op = !cast<ConstantSDNode>(Val&: Arg)->isZero();
782 return GetScalarizedVector(Op: N->getOperand(Num: Op));
783}
784
785SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
786 SDValue Src = N->getOperand(Num: 0);
787 EVT SrcVT = Src.getValueType();
788 SDLoc dl(N);
789
790 // Handle case where result is scalarized but operand is not
791 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeScalarizeVector)
792 Src = GetScalarizedVector(Op: Src);
793 else
794 Src = DAG.getNode(
795 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: SrcVT.getVectorElementType(), N1: Src,
796 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
797
798 EVT DstVT = N->getValueType(ResNo: 0).getVectorElementType();
799 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVT, N1: Src, N2: N->getOperand(Num: 1));
800}
801
802SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
803 assert(N->getValueType(0).isVector() &&
804 N->getOperand(0).getValueType().isVector() &&
805 "Operand types must be vectors");
806 SDValue LHS = N->getOperand(Num: 0);
807 SDValue RHS = N->getOperand(Num: 1);
808 EVT OpVT = LHS.getValueType();
809 EVT NVT = N->getValueType(ResNo: 0).getVectorElementType();
810 SDLoc DL(N);
811
812 // The result needs scalarizing, but it's not a given that the source does.
813 if (getTypeAction(VT: OpVT) == TargetLowering::TypeScalarizeVector) {
814 LHS = GetScalarizedVector(Op: LHS);
815 RHS = GetScalarizedVector(Op: RHS);
816 } else {
817 EVT VT = OpVT.getVectorElementType();
818 LHS = DAG.getExtractVectorElt(DL, VT, Vec: LHS, Idx: 0);
819 RHS = DAG.getExtractVectorElt(DL, VT, Vec: RHS, Idx: 0);
820 }
821
822 // Turn it into a scalar SETCC.
823 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
824 N3: N->getOperand(Num: 2));
825 // Vectors may have a different boolean contents to scalars. Promote the
826 // value appropriately.
827 ISD::NodeType ExtendCode =
828 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
829 return DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
830}
831
832SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) {
833 SDLoc DL(N);
834 SDValue Arg = N->getOperand(Num: 0);
835 SDValue Test = N->getOperand(Num: 1);
836 EVT ArgVT = Arg.getValueType();
837 EVT ResultVT = N->getValueType(ResNo: 0).getVectorElementType();
838
839 if (getTypeAction(VT: ArgVT) == TargetLowering::TypeScalarizeVector) {
840 Arg = GetScalarizedVector(Op: Arg);
841 } else {
842 EVT VT = ArgVT.getVectorElementType();
843 Arg = DAG.getExtractVectorElt(DL, VT, Vec: Arg, Idx: 0);
844 }
845
846 SDValue Res =
847 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, Ops: {Arg, Test}, Flags: N->getFlags());
848 // Vectors may have a different boolean contents to scalars. Promote the
849 // value appropriately.
850 ISD::NodeType ExtendCode =
851 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: ArgVT));
852 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: Res);
853}
854
855//===----------------------------------------------------------------------===//
856// Operand Vector Scalarization <1 x ty> -> ty.
857//===----------------------------------------------------------------------===//
858
859bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
860 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": ";
861 N->dump(&DAG));
862 SDValue Res = SDValue();
863
864 // See if the target wants to custom scalarize this node.
865 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
866 return false;
867
868 switch (N->getOpcode()) {
869 default:
870#ifndef NDEBUG
871 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
872 N->dump(&DAG);
873 dbgs() << "\n";
874#endif
875 report_fatal_error(reason: "Do not know how to scalarize this operator's "
876 "operand!\n");
877 case ISD::BITCAST:
878 Res = ScalarizeVecOp_BITCAST(N);
879 break;
880 case ISD::FAKE_USE:
881 Res = ScalarizeVecOp_FAKE_USE(N);
882 break;
883 case ISD::ANY_EXTEND:
884 case ISD::ZERO_EXTEND:
885 case ISD::SIGN_EXTEND:
886 case ISD::TRUNCATE:
887 case ISD::FP_TO_SINT:
888 case ISD::FP_TO_UINT:
889 case ISD::SINT_TO_FP:
890 case ISD::UINT_TO_FP:
891 case ISD::LROUND:
892 case ISD::LLROUND:
893 case ISD::LRINT:
894 case ISD::LLRINT:
895 Res = ScalarizeVecOp_UnaryOp(N);
896 break;
897 case ISD::FP_TO_SINT_SAT:
898 case ISD::FP_TO_UINT_SAT:
899 case ISD::CONVERT_FROM_ARBITRARY_FP:
900 Res = ScalarizeVecOp_UnaryOpWithExtraInput(N);
901 break;
902 case ISD::CONVERT_TO_ARBITRARY_FP: {
903 assert(N->getValueType(0).getVectorNumElements() == 1 &&
904 "Unexpected vector type!");
905 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
906 SDValue Op = DAG.getNode(
907 Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(), N1: Elt,
908 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
909 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
910 break;
911 }
912 case ISD::STRICT_SINT_TO_FP:
913 case ISD::STRICT_UINT_TO_FP:
914 case ISD::STRICT_FP_TO_SINT:
915 case ISD::STRICT_FP_TO_UINT:
916 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
917 break;
918 case ISD::CONCAT_VECTORS:
919 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
920 break;
921 case ISD::INSERT_SUBVECTOR:
922 Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
923 break;
924 case ISD::EXTRACT_VECTOR_ELT:
925 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
926 break;
927 case ISD::VSELECT:
928 Res = ScalarizeVecOp_VSELECT(N);
929 break;
930 case ISD::SETCC:
931 Res = ScalarizeVecOp_VSETCC(N);
932 break;
933 case ISD::STRICT_FSETCC:
934 case ISD::STRICT_FSETCCS:
935 Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
936 break;
937 case ISD::STORE:
938 Res = ScalarizeVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
939 break;
940 case ISD::ATOMIC_STORE:
941 Res = ScalarizeVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
942 break;
943 case ISD::STRICT_FP_ROUND:
944 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
945 break;
946 case ISD::FP_ROUND:
947 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
948 break;
949 case ISD::STRICT_FP_EXTEND:
950 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
951 break;
952 case ISD::FP_EXTEND:
953 Res = ScalarizeVecOp_FP_EXTEND(N);
954 break;
955 case ISD::VECREDUCE_FADD:
956 case ISD::VECREDUCE_FMUL:
957 case ISD::VECREDUCE_ADD:
958 case ISD::VECREDUCE_MUL:
959 case ISD::VECREDUCE_AND:
960 case ISD::VECREDUCE_OR:
961 case ISD::VECREDUCE_XOR:
962 case ISD::VECREDUCE_SMAX:
963 case ISD::VECREDUCE_SMIN:
964 case ISD::VECREDUCE_UMAX:
965 case ISD::VECREDUCE_UMIN:
966 case ISD::VECREDUCE_FMAX:
967 case ISD::VECREDUCE_FMIN:
968 case ISD::VECREDUCE_FMAXIMUM:
969 case ISD::VECREDUCE_FMINIMUM:
970 case ISD::VECREDUCE_FMAXIMUMNUM:
971 case ISD::VECREDUCE_FMINIMUMNUM:
972 Res = ScalarizeVecOp_VECREDUCE(N);
973 break;
974 case ISD::VECREDUCE_SEQ_FADD:
975 case ISD::VECREDUCE_SEQ_FMUL:
976 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
977 break;
978 case ISD::SCMP:
979 case ISD::UCMP:
980 Res = ScalarizeVecOp_CMP(N);
981 break;
982 case ISD::VECTOR_FIND_LAST_ACTIVE:
983 Res = ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(N);
984 break;
985 case ISD::CTTZ_ELTS:
986 case ISD::CTTZ_ELTS_ZERO_POISON:
987 Res = ScalarizeVecOp_CTTZ_ELTS(N);
988 break;
989 case ISD::VECTOR_MATCH:
990 Res = ScalarizeVecOp_VECTOR_MATCH(N, OpNo);
991 break;
992 case ISD::MASKED_UDIV:
993 case ISD::MASKED_SDIV:
994 case ISD::MASKED_UREM:
995 case ISD::MASKED_SREM:
996 Res = ScalarizeVecOp_MaskedBinOp(N, OpNo);
997 break;
998 }
999
1000 // If the result is null, the sub-method took care of registering results etc.
1001 if (!Res.getNode()) return false;
1002
1003 // If the result is N, the sub-method updated N in place. Tell the legalizer
1004 // core about this.
1005 if (Res.getNode() == N)
1006 return true;
1007
1008 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1009 "Invalid operand expansion");
1010
1011 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1012 return false;
1013}
1014
1015/// If the value to convert is a vector that needs to be scalarized, it must be
1016/// <1 x ty>. Convert the element instead.
1017SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
1018 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1019 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N),
1020 VT: N->getValueType(ResNo: 0), Operand: Elt);
1021}
1022
1023// Need to legalize vector operands of fake uses. Must be <1 x ty>.
1024SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) {
1025 assert(N->getOperand(1).getValueType().getVectorNumElements() == 1 &&
1026 "Fake Use: Unexpected vector type!");
1027 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1028 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Elt);
1029}
1030
1031/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1032/// Do the operation on the element instead.
1033SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
1034 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1035 "Unexpected vector type!");
1036 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1037 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1038 VT: N->getValueType(ResNo: 0).getScalarType(), Operand: Elt);
1039 // Revectorize the result so the types line up with what the uses of this
1040 // expression expect.
1041 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1042}
1043
1044/// Same as ScalarizeVecOp_UnaryOp with an extra operand (for example a
1045/// typesize).
1046SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N) {
1047 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1048 "Unexpected vector type!");
1049 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1050 SDValue Op =
1051 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0).getScalarType(),
1052 N1: Elt, N2: N->getOperand(Num: 1));
1053 // Revectorize the result so the types line up with what the uses of this
1054 // expression expect.
1055 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
1056}
1057
1058/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
1059/// Do the strict FP operation on the element instead.
1060SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
1061 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1062 "Unexpected vector type!");
1063 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1064 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
1065 ResultTys: { N->getValueType(ResNo: 0).getScalarType(), MVT::Other },
1066 Ops: { N->getOperand(Num: 0), Elt });
1067 // Legalize the chain result - switch anything that used the old chain to
1068 // use the new one.
1069 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1070 // Revectorize the result so the types line up with what the uses of this
1071 // expression expect.
1072 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1073
1074 // Do our own replacement and return SDValue() to tell the caller that we
1075 // handled all replacements since caller can only handle a single result.
1076 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1077 return SDValue();
1078}
1079
1080/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
1081SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
1082 SmallVector<SDValue, 8> Ops(N->getNumOperands());
1083 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
1084 Ops[i] = GetScalarizedVector(Op: N->getOperand(Num: i));
1085 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
1086}
1087
1088/// The inserted subvector is to be scalarized - use insert vector element
1089/// instead.
1090SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
1091 unsigned OpNo) {
1092 // We should not be attempting to scalarize the containing vector
1093 assert(OpNo == 1);
1094 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1095 SDValue ContainingVec = N->getOperand(Num: 0);
1096 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
1097 VT: ContainingVec.getValueType(), N1: ContainingVec, N2: Elt,
1098 N3: N->getOperand(Num: 2));
1099}
1100
1101/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
1102/// so just return the element, ignoring the index.
1103SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1104 EVT VT = N->getValueType(ResNo: 0);
1105 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1106 if (Res.getValueType() != VT)
1107 Res = VT.isFloatingPoint()
1108 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Res)
1109 : DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: Res);
1110 return Res;
1111}
1112
1113/// If the input condition is a vector that needs to be scalarized, it must be
1114/// <1 x i1>, so just convert to a normal ISD::SELECT
1115/// (still with vector output type since that was acceptable if we got here).
1116SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
1117 SDValue ScalarCond = GetScalarizedVector(Op: N->getOperand(Num: 0));
1118 EVT VT = N->getValueType(ResNo: 0);
1119
1120 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT, N1: ScalarCond, N2: N->getOperand(Num: 1),
1121 N3: N->getOperand(Num: 2));
1122}
1123
1124/// If the operand is a vector that needs to be scalarized then the
1125/// result must be a single-element vector, so just convert to a scalar
1126/// SETCC and wrap with a scalar_to_vector since the res type is legal
1127/// if we got here
1128SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
1129 assert(N->getValueType(0).isVector() &&
1130 N->getOperand(0).getValueType().isVector() &&
1131 "Operand types must be vectors");
1132 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1133 "Expected single-element vector type");
1134
1135 EVT VT = N->getValueType(ResNo: 0);
1136 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1137 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1138
1139 EVT OpVT = N->getOperand(Num: 0).getValueType();
1140 EVT NVT = VT.getVectorElementType();
1141 SDLoc DL(N);
1142 // Turn it into a scalar SETCC.
1143 SDValue Res = DAG.getNode(Opcode: ISD::SETCC, DL, VT: MVT::i1, N1: LHS, N2: RHS,
1144 N3: N->getOperand(Num: 2));
1145
1146 // Vectors may have a different boolean contents to scalars. Promote the
1147 // value appropriately.
1148 ISD::NodeType ExtendCode =
1149 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1150
1151 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1152
1153 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1154}
1155
1156// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
1157SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
1158 unsigned OpNo) {
1159 assert(OpNo == 1 && "Wrong operand for scalarization!");
1160 assert(N->getValueType(0).isVector() &&
1161 N->getOperand(1).getValueType().isVector() &&
1162 "Operand types must be vectors");
1163 assert(N->getValueType(0).getVectorNumElements() == 1 &&
1164 "Expected single-element vector type");
1165
1166 EVT VT = N->getValueType(ResNo: 0);
1167 SDValue Ch = N->getOperand(Num: 0);
1168 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1169 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 2));
1170 SDValue CC = N->getOperand(Num: 3);
1171
1172 EVT OpVT = N->getOperand(Num: 1).getValueType();
1173 EVT NVT = VT.getVectorElementType();
1174 SDLoc DL(N);
1175 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {MVT::i1, MVT::Other},
1176 Ops: {Ch, LHS, RHS, CC});
1177
1178 // Legalize the chain result - switch anything that used the old chain to
1179 // use the new one.
1180 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1181
1182 ISD::NodeType ExtendCode =
1183 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
1184
1185 Res = DAG.getNode(Opcode: ExtendCode, DL, VT: NVT, Operand: Res);
1186 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Res);
1187
1188 // Do our own replacement and return SDValue() to tell the caller that we
1189 // handled all replacements since caller can only handle a single result.
1190 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1191 return SDValue();
1192}
1193
1194/// If the value to store is a vector that needs to be scalarized, it must be
1195/// <1 x ty>. Just store the element.
1196SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
1197 assert(N->isUnindexed() && "Indexed store of one-element vector?");
1198 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
1199 SDLoc dl(N);
1200
1201 if (N->isTruncatingStore())
1202 return DAG.getTruncStore(
1203 Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1204 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(),
1205 SVT: N->getMemoryVT().getVectorElementType(), Alignment: N->getBaseAlign(),
1206 MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getAAInfo());
1207
1208 return DAG.getStore(Chain: N->getChain(), dl, Val: GetScalarizedVector(Op: N->getOperand(Num: 1)),
1209 Ptr: N->getBasePtr(), PtrInfo: N->getPointerInfo(), Alignment: N->getBaseAlign(),
1210 MMOFlags: N->getMemOperand()->getFlags(), Metadata: N->getAAInfo());
1211}
1212
1213/// If the value to store is a vector that needs to be scalarized, it must be
1214/// <1 x ty>. Just store the element.
1215SDValue DAGTypeLegalizer::ScalarizeVecOp_ATOMIC_STORE(AtomicSDNode *N) {
1216 SDValue ScalarVal = GetScalarizedVector(Op: N->getVal());
1217 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SDLoc(N),
1218 MemVT: N->getMemoryVT().getVectorElementType(), Chain: N->getChain(),
1219 Ptr: ScalarVal, Val: N->getBasePtr(), MMO: N->getMemOperand());
1220}
1221
1222/// If the value to round is a vector that needs to be scalarized, it must be
1223/// <1 x ty>. Convert the element instead.
1224SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
1225 assert(OpNo == 0 && "Wrong operand for scalarization!");
1226 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1227 SDValue Res = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
1228 VT: N->getValueType(ResNo: 0).getVectorElementType(), N1: Elt,
1229 N2: N->getOperand(Num: 1));
1230 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1231}
1232
1233SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
1234 unsigned OpNo) {
1235 assert(OpNo == 1 && "Wrong operand for scalarization!");
1236 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1237 SDValue Res =
1238 DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
1239 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1240 Ops: {N->getOperand(Num: 0), Elt, N->getOperand(Num: 2)});
1241 // Legalize the chain result - switch anything that used the old chain to
1242 // use the new one.
1243 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1244
1245 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1246
1247 // Do our own replacement and return SDValue() to tell the caller that we
1248 // handled all replacements since caller can only handle a single result.
1249 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1250 return SDValue();
1251}
1252
1253/// If the value to extend is a vector that needs to be scalarized, it must be
1254/// <1 x ty>. Convert the element instead.
1255SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
1256 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 0));
1257 SDValue Res = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N),
1258 VT: N->getValueType(ResNo: 0).getVectorElementType(), Operand: Elt);
1259 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1260}
1261
1262/// If the value to extend is a vector that needs to be scalarized, it must be
1263/// <1 x ty>. Convert the element instead.
1264SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
1265 SDValue Elt = GetScalarizedVector(Op: N->getOperand(Num: 1));
1266 SDValue Res =
1267 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
1268 ResultTys: {N->getValueType(ResNo: 0).getVectorElementType(), MVT::Other},
1269 Ops: {N->getOperand(Num: 0), Elt});
1270 // Legalize the chain result - switch anything that used the old chain to
1271 // use the new one.
1272 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
1273
1274 Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1275
1276 // Do our own replacement and return SDValue() to tell the caller that we
1277 // handled all replacements since caller can only handle a single result.
1278 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1279 return SDValue();
1280}
1281
1282SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
1283 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
1284 // Result type may be wider than element type.
1285 if (Res.getValueType() != N->getValueType(ResNo: 0))
1286 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Res);
1287 return Res;
1288}
1289
1290SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
1291 SDValue AccOp = N->getOperand(Num: 0);
1292 SDValue VecOp = N->getOperand(Num: 1);
1293
1294 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
1295
1296 SDValue Op = GetScalarizedVector(Op: VecOp);
1297 return DAG.getNode(Opcode: BaseOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1298 N1: AccOp, N2: Op, Flags: N->getFlags());
1299}
1300
1301SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) {
1302 SDValue LHS = GetScalarizedVector(Op: N->getOperand(Num: 0));
1303 SDValue RHS = GetScalarizedVector(Op: N->getOperand(Num: 1));
1304
1305 EVT ResVT = N->getValueType(ResNo: 0).getVectorElementType();
1306 SDValue Cmp = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, N1: LHS, N2: RHS);
1307 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Cmp);
1308}
1309
1310SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
1311 // Since there is no "none-active" result, the only valid return for <1 x ty>
1312 // is 0. Note: Since we check the high mask during splitting this is safe.
1313 // As e.g., a <2 x ty> operation would split to:
1314 // any_active(%hi_mask) ? (1 + last_active(%hi_mask))
1315 // : `last_active(%lo_mask)`
1316 // Which then scalarizes to:
1317 // %mask[1] ? 1 : 0
1318 EVT VT = N->getValueType(ResNo: 0);
1319 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
1320}
1321
1322SDValue DAGTypeLegalizer::ScalarizeVecOp_CTTZ_ELTS(SDNode *N) {
1323 // The number of trailing zero elements is 1 if the element is 0, and 0
1324 // otherwise.
1325 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
1326 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1327 SDValue Op = GetScalarizedVector(Op: N->getOperand(Num: 0));
1328 SDValue SetCC =
1329 DAG.getSetCC(DL: SDLoc(N), VT: MVT::i1, LHS: Op,
1330 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: Op.getValueType()), Cond: ISD::SETEQ);
1331 return DAG.getZExtOrTrunc(Op: SetCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
1332}
1333
1334SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_MATCH(SDNode *N) {
1335 SDLoc DL(N);
1336 // Reuse the expansion (which should scalarize).
1337 SDValue Mask = TLI.expandVectorMatch(N, DAG);
1338 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL,
1339 VT: N->getValueType(ResNo: 0).getScalarType(), N1: Mask,
1340 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1341}
1342
1343SDValue DAGTypeLegalizer::ScalarizeVecOp_VECTOR_MATCH(SDNode *N,
1344 unsigned OpNo) {
1345 return TLI.expandVectorMatch(N, DAG);
1346}
1347
1348SDValue DAGTypeLegalizer::ScalarizeVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
1349 assert(OpNo == 2 && "Can only scalarize mask operand");
1350 SDLoc DL(N);
1351 EVT VT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1352 SDValue LHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 0), Idx: 0);
1353 SDValue RHS = DAG.getExtractVectorElt(DL, VT, Vec: N->getOperand(Num: 1), Idx: 0);
1354 SDValue Mask = GetScalarizedVector(Op: N->getOperand(Num: 2));
1355 // Vectors may have a different boolean contents to scalars, so truncate to i1
1356 // and let type legalization promote appropriately.
1357 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Mask);
1358 // Masked binary ops don't have UB on disabled lanes but produce poison, so
1359 // use 1 as the divisor to avoid division by zero and overflow.
1360 SDValue BinOp =
1361 DAG.getNode(Opcode: ISD::getUnmaskedBinOpOpcode(MaskedOpc: N->getOpcode()), DL, VT, N1: LHS,
1362 N2: DAG.getSelect(DL, VT, Cond: Mask, LHS: RHS, RHS: DAG.getConstant(Val: 1, DL, VT)));
1363 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: N->getValueType(ResNo: 0), Operand: BinOp);
1364}
1365
1366//===----------------------------------------------------------------------===//
1367// Result Vector Splitting
1368//===----------------------------------------------------------------------===//
1369
1370/// This method is called when the specified result of the specified node is
1371/// found to need vector splitting. At this point, the node may also have
1372/// invalid operands or may have other results that need legalization, we just
1373/// know that (at least) one result needs vector splitting.
1374void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
1375 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG));
1376 SDValue Lo, Hi;
1377
1378 // See if the target wants to custom expand this node.
1379 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1380 return;
1381
1382 switch (N->getOpcode()) {
1383 default:
1384#ifndef NDEBUG
1385 dbgs() << "SplitVectorResult #" << ResNo << ": ";
1386 N->dump(&DAG);
1387 dbgs() << "\n";
1388#endif
1389 report_fatal_error(reason: "Do not know how to split the result of this "
1390 "operator!\n");
1391
1392 case ISD::LOOP_DEPENDENCE_RAW_MASK:
1393 case ISD::LOOP_DEPENDENCE_WAR_MASK:
1394 SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
1395 break;
1396 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1397 case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
1398 case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
1399 case ISD::VSELECT:
1400 case ISD::SELECT:
1401 case ISD::VP_MERGE: SplitRes_Select(N, Lo, Hi); break;
1402 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1403 case ISD::POISON:
1404 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1405 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
1406 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
1407 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
1408 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
1409 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
1410 case ISD::FPOWI:
1411 case ISD::FLDEXP:
1412 case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
1413 case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
1414 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1415 case ISD::SPLAT_VECTOR:
1416 case ISD::SCALAR_TO_VECTOR:
1417 SplitVecRes_ScalarOp(N, Lo, Hi);
1418 break;
1419 case ISD::STEP_VECTOR:
1420 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
1421 break;
1422 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
1423 case ISD::ATOMIC_LOAD:
1424 SplitVecRes_ATOMIC_LOAD(LD: cast<AtomicSDNode>(Val: N), Lo, Hi);
1425 break;
1426 case ISD::LOAD:
1427 SplitVecRes_LOAD(LD: cast<LoadSDNode>(Val: N), Lo, Hi);
1428 break;
1429 case ISD::VP_LOAD:
1430 SplitVecRes_VP_LOAD(LD: cast<VPLoadSDNode>(Val: N), Lo, Hi);
1431 break;
1432 case ISD::VP_LOAD_FF:
1433 SplitVecRes_VP_LOAD_FF(LD: cast<VPLoadFFSDNode>(Val: N), Lo, Hi);
1434 break;
1435 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1436 SplitVecRes_VP_STRIDED_LOAD(SLD: cast<VPStridedLoadSDNode>(Val: N), Lo, Hi);
1437 break;
1438 case ISD::MLOAD:
1439 SplitVecRes_MLOAD(MLD: cast<MaskedLoadSDNode>(Val: N), Lo, Hi);
1440 break;
1441 case ISD::MGATHER:
1442 case ISD::VP_GATHER:
1443 SplitVecRes_Gather(VPGT: cast<MemSDNode>(Val: N), Lo, Hi, /*SplitSETCC*/ true);
1444 break;
1445 case ISD::VECTOR_COMPRESS:
1446 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
1447 break;
1448 case ISD::SETCC:
1449 SplitVecRes_SETCC(N, Lo, Hi);
1450 break;
1451 case ISD::VECTOR_REVERSE:
1452 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
1453 break;
1454 case ISD::VECTOR_SHUFFLE:
1455 SplitVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N), Lo, Hi);
1456 break;
1457 case ISD::VECTOR_SPLICE_LEFT:
1458 case ISD::VECTOR_SPLICE_RIGHT:
1459 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
1460 break;
1461 case ISD::VECTOR_DEINTERLEAVE:
1462 SplitVecRes_VECTOR_DEINTERLEAVE(N);
1463 return;
1464 case ISD::VECTOR_INTERLEAVE:
1465 SplitVecRes_VECTOR_INTERLEAVE(N);
1466 return;
1467 case ISD::VAARG:
1468 SplitVecRes_VAARG(N, Lo, Hi);
1469 break;
1470
1471 case ISD::ANY_EXTEND_VECTOR_INREG:
1472 case ISD::SIGN_EXTEND_VECTOR_INREG:
1473 case ISD::ZERO_EXTEND_VECTOR_INREG:
1474 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1475 break;
1476
1477 case ISD::ABS:
1478 case ISD::ABS_MIN_POISON:
1479 case ISD::BITREVERSE:
1480 case ISD::BSWAP:
1481 case ISD::CTLZ:
1482 case ISD::CTTZ:
1483 case ISD::CTLZ_ZERO_POISON:
1484 case ISD::CTTZ_ZERO_POISON:
1485 case ISD::CTPOP:
1486 case ISD::FABS:
1487 case ISD::FACOS:
1488 case ISD::FASIN:
1489 case ISD::FATAN:
1490 case ISD::FCEIL:
1491 case ISD::FCOS:
1492 case ISD::FCOSH:
1493 case ISD::FEXP:
1494 case ISD::FEXP2:
1495 case ISD::FEXP10:
1496 case ISD::FFLOOR:
1497 case ISD::FLOG:
1498 case ISD::FLOG10:
1499 case ISD::FLOG2:
1500 case ISD::FNEARBYINT:
1501 case ISD::FNEG:
1502 case ISD::FREEZE:
1503 case ISD::ARITH_FENCE:
1504 case ISD::FP_EXTEND:
1505 case ISD::FP_ROUND:
1506 case ISD::FP_TO_SINT:
1507 case ISD::FP_TO_UINT:
1508 case ISD::FRINT:
1509 case ISD::LRINT:
1510 case ISD::LLRINT:
1511 case ISD::FROUND:
1512 case ISD::FROUNDEVEN:
1513 case ISD::LROUND:
1514 case ISD::LLROUND:
1515 case ISD::FSIN:
1516 case ISD::FSINH:
1517 case ISD::FSQRT:
1518 case ISD::FTAN:
1519 case ISD::FTANH:
1520 case ISD::FTRUNC:
1521 case ISD::SINT_TO_FP:
1522 case ISD::TRUNCATE:
1523 case ISD::UINT_TO_FP:
1524 case ISD::FCANONICALIZE:
1525 case ISD::AssertNoFPClass:
1526 case ISD::CONVERT_FROM_ARBITRARY_FP:
1527 case ISD::CONVERT_TO_ARBITRARY_FP:
1528 SplitVecRes_UnaryOp(N, Lo, Hi);
1529 break;
1530 case ISD::ADDRSPACECAST:
1531 SplitVecRes_ADDRSPACECAST(N, Lo, Hi);
1532 break;
1533 case ISD::FMODF:
1534 case ISD::FFREXP:
1535 case ISD::FSINCOS:
1536 case ISD::FSINCOSPI:
1537 SplitVecRes_UnaryOpWithTwoResults(N, ResNo, Lo, Hi);
1538 break;
1539
1540 case ISD::ANY_EXTEND:
1541 case ISD::SIGN_EXTEND:
1542 case ISD::ZERO_EXTEND:
1543 SplitVecRes_ExtendOp(N, Lo, Hi);
1544 break;
1545
1546 case ISD::ADD:
1547 case ISD::SUB:
1548 case ISD::MUL:
1549 case ISD::CLMUL:
1550 case ISD::CLMULR:
1551 case ISD::CLMULH:
1552 case ISD::PEXT:
1553 case ISD::PDEP:
1554 case ISD::MULHS:
1555 case ISD::MULHU:
1556 case ISD::ABDS:
1557 case ISD::ABDU:
1558 case ISD::AVGCEILS:
1559 case ISD::AVGCEILU:
1560 case ISD::AVGFLOORS:
1561 case ISD::AVGFLOORU:
1562 case ISD::FADD:
1563 case ISD::FSUB:
1564 case ISD::FMUL:
1565 case ISD::FMINNUM:
1566 case ISD::FMINNUM_IEEE:
1567 case ISD::FMAXNUM:
1568 case ISD::FMAXNUM_IEEE:
1569 case ISD::FMINIMUM:
1570 case ISD::FMAXIMUM:
1571 case ISD::FMINIMUMNUM:
1572 case ISD::FMAXIMUMNUM:
1573 case ISD::SDIV: case ISD::VP_SDIV:
1574 case ISD::UDIV: case ISD::VP_UDIV:
1575 case ISD::FDIV:
1576 case ISD::FPOW:
1577 case ISD::FATAN2:
1578 case ISD::AND:
1579 case ISD::OR:
1580 case ISD::XOR:
1581 case ISD::SHL:
1582 case ISD::SRA:
1583 case ISD::SRL:
1584 case ISD::UREM: case ISD::VP_UREM:
1585 case ISD::SREM: case ISD::VP_SREM:
1586 case ISD::FREM:
1587 case ISD::SMIN:
1588 case ISD::SMAX:
1589 case ISD::UMIN:
1590 case ISD::UMAX:
1591 case ISD::SADDSAT:
1592 case ISD::UADDSAT:
1593 case ISD::SSUBSAT:
1594 case ISD::USUBSAT:
1595 case ISD::SSHLSAT:
1596 case ISD::USHLSAT:
1597 case ISD::ROTL:
1598 case ISD::ROTR:
1599 SplitVecRes_BinOp(N, Lo, Hi);
1600 break;
1601 case ISD::MASKED_UDIV:
1602 case ISD::MASKED_SDIV:
1603 case ISD::MASKED_UREM:
1604 case ISD::MASKED_SREM:
1605 SplitVecRes_MaskedBinOp(N, Lo, Hi);
1606 break;
1607 case ISD::FMA:
1608 case ISD::FSHL:
1609 case ISD::FSHR:
1610 SplitVecRes_TernaryOp(N, Lo, Hi);
1611 break;
1612
1613 case ISD::SCMP: case ISD::UCMP:
1614 SplitVecRes_CMP(N, Lo, Hi);
1615 break;
1616
1617#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1618 case ISD::STRICT_##DAGN:
1619#include "llvm/IR/ConstrainedOps.def"
1620 SplitVecRes_StrictFPOp(N, Lo, Hi);
1621 break;
1622
1623 case ISD::FP_TO_UINT_SAT:
1624 case ISD::FP_TO_SINT_SAT:
1625 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1626 break;
1627
1628 case ISD::UADDO:
1629 case ISD::SADDO:
1630 case ISD::USUBO:
1631 case ISD::SSUBO:
1632 case ISD::UMULO:
1633 case ISD::SMULO:
1634 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1635 break;
1636 case ISD::SMULFIX:
1637 case ISD::SMULFIXSAT:
1638 case ISD::UMULFIX:
1639 case ISD::UMULFIXSAT:
1640 case ISD::SDIVFIX:
1641 case ISD::SDIVFIXSAT:
1642 case ISD::UDIVFIX:
1643 case ISD::UDIVFIXSAT:
1644 SplitVecRes_FIX(N, Lo, Hi);
1645 break;
1646 case ISD::EXPERIMENTAL_VP_SPLICE:
1647 SplitVecRes_VP_SPLICE(N, Lo, Hi);
1648 break;
1649 case ISD::EXPERIMENTAL_VP_REVERSE:
1650 SplitVecRes_VP_REVERSE(N, Lo, Hi);
1651 break;
1652 case ISD::PARTIAL_REDUCE_UMLA:
1653 case ISD::PARTIAL_REDUCE_SMLA:
1654 case ISD::PARTIAL_REDUCE_SUMLA:
1655 case ISD::PARTIAL_REDUCE_FMLA:
1656 SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi);
1657 break;
1658 case ISD::GET_ACTIVE_LANE_MASK:
1659 SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi);
1660 break;
1661 case ISD::VECTOR_MATCH:
1662 SplitVecRes_VECTOR_MATCH(N, Lo, Hi);
1663 break;
1664 }
1665
1666 // If Lo/Hi is null, the sub-method took care of registering results etc.
1667 if (Lo.getNode())
1668 SetSplitVector(Op: SDValue(N, ResNo), Lo, Hi);
1669}
1670
1671void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1672 MachinePointerInfo &MPI, SDValue &Ptr,
1673 uint64_t *ScaledOffset) {
1674 SDLoc DL(N);
1675 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinValue() / 8;
1676
1677 if (MemVT.isScalableVector()) {
1678 SDValue BytesIncrement = DAG.getVScale(
1679 DL, VT: Ptr.getValueType(),
1680 MulImm: APInt(Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1681 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1682 if (ScaledOffset)
1683 *ScaledOffset += IncrementSize;
1684 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: Ptr, N2: BytesIncrement,
1685 Flags: SDNodeFlags::NoUnsignedWrap);
1686 } else {
1687 MPI = N->getPointerInfo().getWithOffset(O: IncrementSize);
1688 // Increment the pointer to the other half.
1689 Ptr = DAG.getObjectPtrOffset(SL: DL, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
1690 }
1691}
1692
1693std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) {
1694 return SplitMask(Mask, DL: SDLoc(Mask));
1695}
1696
1697std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask,
1698 const SDLoc &DL) {
1699 SDValue MaskLo, MaskHi;
1700 EVT MaskVT = Mask.getValueType();
1701 if (getTypeAction(VT: MaskVT) == TargetLowering::TypeSplitVector)
1702 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
1703 else
1704 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
1705 return std::make_pair(x&: MaskLo, y&: MaskHi);
1706}
1707
1708void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) {
1709 SDValue LHSLo, LHSHi;
1710 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1711 SDValue RHSLo, RHSHi;
1712 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1713 SDLoc dl(N);
1714
1715 const SDNodeFlags Flags = N->getFlags();
1716 unsigned Opcode = N->getOpcode();
1717 if (N->getNumOperands() == 2) {
1718 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, Flags);
1719 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, Flags);
1720 return;
1721 }
1722
1723 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1724 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
1725 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
1726 "Expected VP opcode");
1727
1728 SDValue MaskLo, MaskHi;
1729 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
1730
1731 SDValue EVLLo, EVLHi;
1732 std::tie(args&: EVLLo, args&: EVLHi) =
1733 DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT: N->getValueType(ResNo: 0), DL: dl);
1734
1735 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(),
1736 Ops: {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1737 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(),
1738 Ops: {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1739}
1740
1741void DAGTypeLegalizer::SplitVecRes_MaskedBinOp(SDNode *N, SDValue &Lo,
1742 SDValue &Hi) {
1743 SDValue LHSLo, LHSHi;
1744 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1745 SDValue RHSLo, RHSHi;
1746 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1747
1748 SDValue MaskLo, MaskHi, Mask = N->getOperand(Num: 2);
1749 if (Mask.getOpcode() == ISD::SETCC)
1750 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
1751 else
1752 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask);
1753
1754 SDLoc dl(N);
1755
1756 const SDNodeFlags Flags = N->getFlags();
1757 unsigned Opcode = N->getOpcode();
1758 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: MaskLo,
1759 Flags);
1760 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: MaskHi,
1761 Flags);
1762}
1763
1764void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1765 SDValue &Hi) {
1766 SDValue Op0Lo, Op0Hi;
1767 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Op0Lo, Hi&: Op0Hi);
1768 SDValue Op1Lo, Op1Hi;
1769 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Op1Lo, Hi&: Op1Hi);
1770 SDValue Op2Lo, Op2Hi;
1771 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Op2Lo, Hi&: Op2Hi);
1772 SDLoc dl(N);
1773
1774 const SDNodeFlags Flags = N->getFlags();
1775 unsigned Opcode = N->getOpcode();
1776 Lo =
1777 DAG.getNode(Opcode, DL: dl, VT: Op0Lo.getValueType(), N1: Op0Lo, N2: Op1Lo, N3: Op2Lo, Flags);
1778 Hi =
1779 DAG.getNode(Opcode, DL: dl, VT: Op0Hi.getValueType(), N1: Op0Hi, N2: Op1Hi, N3: Op2Hi, Flags);
1780}
1781
1782void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) {
1783 LLVMContext &Ctxt = *DAG.getContext();
1784 SDLoc dl(N);
1785
1786 SDValue LHS = N->getOperand(Num: 0);
1787 SDValue RHS = N->getOperand(Num: 1);
1788
1789 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1790 if (getTypeAction(VT: LHS.getValueType()) == TargetLowering::TypeSplitVector) {
1791 GetSplitVector(Op: LHS, Lo&: LHSLo, Hi&: LHSHi);
1792 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
1793 } else {
1794 std::tie(args&: LHSLo, args&: LHSHi) = DAG.SplitVector(N: LHS, DL: dl);
1795 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: dl);
1796 }
1797
1798 EVT SplitResVT = N->getValueType(ResNo: 0).getHalfNumVectorElementsVT(Context&: Ctxt);
1799 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSLo, N2: RHSLo);
1800 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: SplitResVT, N1: LHSHi, N2: RHSHi);
1801}
1802
1803void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1804 SDValue LHSLo, LHSHi;
1805 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
1806 SDValue RHSLo, RHSHi;
1807 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
1808 SDLoc dl(N);
1809 SDValue Op2 = N->getOperand(Num: 2);
1810
1811 unsigned Opcode = N->getOpcode();
1812 Lo = DAG.getNode(Opcode, DL: dl, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo, N3: Op2,
1813 Flags: N->getFlags());
1814 Hi = DAG.getNode(Opcode, DL: dl, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi, N3: Op2,
1815 Flags: N->getFlags());
1816}
1817
1818void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1819 SDValue &Hi) {
1820 // We know the result is a vector. The input may be either a vector or a
1821 // scalar value.
1822 EVT LoVT, HiVT;
1823 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1824 SDLoc dl(N);
1825
1826 SDValue InOp = N->getOperand(Num: 0);
1827 EVT InVT = InOp.getValueType();
1828
1829 // Handle some special cases efficiently.
1830 switch (getTypeAction(VT: InVT)) {
1831 case TargetLowering::TypeLegal:
1832 case TargetLowering::TypePromoteInteger:
1833 case TargetLowering::TypeSoftPromoteHalf:
1834 case TargetLowering::TypeSoftenFloat:
1835 case TargetLowering::TypeScalarizeVector:
1836 case TargetLowering::TypeWidenVector:
1837 break;
1838 case TargetLowering::TypeExpandInteger:
1839 case TargetLowering::TypeExpandFloat:
1840 // A scalar to vector conversion, where the scalar needs expansion.
1841 // If the vector is being split in two then we can just convert the
1842 // expanded pieces.
1843 if (LoVT == HiVT) {
1844 GetExpandedOp(Op: InOp, Lo, Hi);
1845 if (DAG.getDataLayout().isBigEndian())
1846 std::swap(a&: Lo, b&: Hi);
1847 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1848 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1849 return;
1850 }
1851 break;
1852 case TargetLowering::TypeSplitVector:
1853 // If the input is a vector that needs to be split, convert each split
1854 // piece of the input now.
1855 GetSplitVector(Op: InOp, Lo, Hi);
1856 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1857 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1858 return;
1859 case TargetLowering::TypeScalarizeScalableVector:
1860 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
1861 }
1862
1863 if (LoVT.isScalableVector()) {
1864 auto [InLo, InHi] = DAG.SplitVectorOperand(N, OpNo: 0);
1865 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: InLo);
1866 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: InHi);
1867 return;
1868 }
1869
1870 // In the general case, convert the input to an integer and split it by hand.
1871 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
1872 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
1873 if (DAG.getDataLayout().isBigEndian())
1874 std::swap(a&: LoIntVT, b&: HiIntVT);
1875
1876 SplitInteger(Op: BitConvertToInteger(Op: InOp), LoVT: LoIntVT, HiVT: HiIntVT, Lo, Hi);
1877
1878 if (DAG.getDataLayout().isBigEndian())
1879 std::swap(a&: Lo, b&: Hi);
1880 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
1881 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
1882}
1883
1884void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
1885 SDValue &Hi) {
1886 SDLoc DL(N);
1887 EVT LoVT, HiVT;
1888 SDValue PtrA = N->getOperand(Num: 0);
1889 SDValue PtrB = N->getOperand(Num: 1);
1890 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1891
1892 // The lane offset for the "Lo" half of the mask is unchanged.
1893 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: PtrA, N2: PtrB,
1894 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1895 /*LaneOffset=*/N4: N->getOperand(Num: 3));
1896 // The lane offset for the "Hi" half of the mask is incremented by the number
1897 // of elements in the "Lo" half.
1898 unsigned LaneOffset =
1899 N->getConstantOperandVal(Num: 3) + LoVT.getVectorMinNumElements();
1900 // Note: The lane offset is implicitly scalable for scalable masks.
1901 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: PtrA, N2: PtrB,
1902 /*ElementSizeInBytes=*/N3: N->getOperand(Num: 2),
1903 /*LaneOffset=*/N4: DAG.getConstant(Val: LaneOffset, DL, VT: MVT::i64));
1904}
1905
1906void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1907 SDValue &Hi) {
1908 EVT LoVT, HiVT;
1909 SDLoc dl(N);
1910 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1911 unsigned LoNumElts = LoVT.getVectorNumElements();
1912 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1913 Lo = DAG.getBuildVector(VT: LoVT, DL: dl, Ops: LoOps);
1914
1915 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1916 Hi = DAG.getBuildVector(VT: HiVT, DL: dl, Ops: HiOps);
1917}
1918
1919void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1920 SDValue &Hi) {
1921 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1922 SDLoc dl(N);
1923 unsigned NumSubvectors = N->getNumOperands() / 2;
1924 if (NumSubvectors == 1) {
1925 Lo = N->getOperand(Num: 0);
1926 Hi = N->getOperand(Num: 1);
1927 return;
1928 }
1929
1930 EVT LoVT, HiVT;
1931 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1932
1933 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1934 Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: LoVT, Ops: LoOps);
1935
1936 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1937 Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: HiVT, Ops: HiOps);
1938}
1939
1940void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1941 SDValue &Hi) {
1942 SDValue Vec = N->getOperand(Num: 0);
1943 SDValue Idx = N->getOperand(Num: 1);
1944 SDLoc dl(N);
1945
1946 EVT LoVT, HiVT;
1947 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
1948
1949 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: LoVT, N1: Vec, N2: Idx);
1950 uint64_t IdxVal = Idx->getAsZExtVal();
1951 Hi = DAG.getNode(
1952 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: HiVT, N1: Vec,
1953 N2: DAG.getVectorIdxConstant(Val: IdxVal + LoVT.getVectorMinNumElements(), DL: dl));
1954}
1955
1956void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1957 SDValue &Hi) {
1958 SDValue Vec = N->getOperand(Num: 0);
1959 SDValue SubVec = N->getOperand(Num: 1);
1960 SDValue Idx = N->getOperand(Num: 2);
1961 SDLoc dl(N);
1962 GetSplitVector(Op: Vec, Lo, Hi);
1963
1964 EVT VecVT = Vec.getValueType();
1965 EVT LoVT = Lo.getValueType();
1966 EVT SubVecVT = SubVec.getValueType();
1967 unsigned VecElems = VecVT.getVectorMinNumElements();
1968 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1969 unsigned LoElems = LoVT.getVectorMinNumElements();
1970
1971 // If we know the index is in the first half, and we know the subvector
1972 // doesn't cross the boundary between the halves, we can avoid spilling the
1973 // vector, and insert into the lower half of the split vector directly.
1974 unsigned IdxVal = Idx->getAsZExtVal();
1975 if (IdxVal + SubElems <= LoElems) {
1976 Lo = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: LoVT, N1: Lo, N2: SubVec, N3: Idx);
1977 return;
1978 }
1979 // Similarly if the subvector is fully in the high half, but mind that we
1980 // can't tell whether a fixed-length subvector is fully within the high half
1981 // of a scalable vector.
1982 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1983 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1984 Hi = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: Hi.getValueType(), N1: Hi, N2: SubVec,
1985 N3: DAG.getVectorIdxConstant(Val: IdxVal - LoElems, DL: dl));
1986 return;
1987 }
1988
1989 if (getTypeAction(VT: SubVecVT) == TargetLowering::TypeWidenVector &&
1990 Vec.isUndef() && SubVecVT.getVectorElementType() == MVT::i1) {
1991 SDValue WideSubVec = GetWidenedVector(Op: SubVec);
1992 if (WideSubVec.getValueType() == VecVT) {
1993 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: WideSubVec, DL: SDLoc(WideSubVec));
1994 return;
1995 }
1996 }
1997
1998 // Spill the vector to the stack.
1999 // In cases where the vector is illegal it will be broken down into parts
2000 // and stored in parts - we should use the alignment for the smallest part.
2001 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2002 SDValue StackPtr =
2003 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2004 auto &MF = DAG.getMachineFunction();
2005 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2006 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2007
2008 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2009 Alignment: SmallestAlign);
2010
2011 // Store the new subvector into the specified index.
2012 SDValue SubVecPtr =
2013 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT, Index: Idx);
2014 Store = DAG.getStore(Chain: Store, dl, Val: SubVec, Ptr: SubVecPtr,
2015 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2016
2017 // Load the Lo part from the stack slot.
2018 Lo = DAG.getLoad(VT: Lo.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo,
2019 Alignment: SmallestAlign);
2020
2021 // Increment the pointer to the other part.
2022 auto *Load = cast<LoadSDNode>(Val&: Lo);
2023 MachinePointerInfo MPI = Load->getPointerInfo();
2024 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2025
2026 // Load the Hi part from the stack slot.
2027 Hi = DAG.getLoad(VT: Hi.getValueType(), dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2028}
2029
2030// Handle splitting an FP where the second operand does not match the first
2031// type. The second operand may be a scalar, or a vector that has exactly as
2032// many elements as the first
2033void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo,
2034 SDValue &Hi) {
2035 SDValue LHSLo, LHSHi;
2036 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2037 SDLoc DL(N);
2038
2039 SDValue RHSLo, RHSHi;
2040 SDValue RHS = N->getOperand(Num: 1);
2041 EVT RHSVT = RHS.getValueType();
2042 if (RHSVT.isVector()) {
2043 if (getTypeAction(VT: RHSVT) == TargetLowering::TypeSplitVector)
2044 GetSplitVector(Op: RHS, Lo&: RHSLo, Hi&: RHSHi);
2045 else
2046 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: RHS, DL: SDLoc(RHS));
2047
2048 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHSLo);
2049 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHSHi);
2050 } else {
2051 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo, N2: RHS);
2052 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi, N2: RHS);
2053 }
2054}
2055
2056void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo,
2057 SDValue &Hi) {
2058 SDLoc DL(N);
2059 SDValue ArgLo, ArgHi;
2060 SDValue Test = N->getOperand(Num: 1);
2061 SDValue FpValue = N->getOperand(Num: 0);
2062 if (getTypeAction(VT: FpValue.getValueType()) == TargetLowering::TypeSplitVector)
2063 GetSplitVector(Op: FpValue, Lo&: ArgLo, Hi&: ArgHi);
2064 else
2065 std::tie(args&: ArgLo, args&: ArgHi) = DAG.SplitVector(N: FpValue, DL: SDLoc(FpValue));
2066 EVT LoVT, HiVT;
2067 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2068
2069 Lo = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: LoVT, N1: ArgLo, N2: Test, Flags: N->getFlags());
2070 Hi = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: HiVT, N1: ArgHi, N2: Test, Flags: N->getFlags());
2071}
2072
2073void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
2074 SDValue &Hi) {
2075 SDValue LHSLo, LHSHi;
2076 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
2077 SDLoc dl(N);
2078
2079 EVT LoVT, HiVT;
2080 std::tie(args&: LoVT, args&: HiVT) =
2081 DAG.GetSplitDestVTs(VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT());
2082
2083 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSLo.getValueType(), N1: LHSLo,
2084 N2: DAG.getValueType(LoVT));
2085 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LHSHi.getValueType(), N1: LHSHi,
2086 N2: DAG.getValueType(HiVT));
2087}
2088
2089void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
2090 SDValue &Hi) {
2091 unsigned Opcode = N->getOpcode();
2092 SDValue N0 = N->getOperand(Num: 0);
2093
2094 SDLoc dl(N);
2095 SDValue InLo, InHi;
2096
2097 if (getTypeAction(VT: N0.getValueType()) == TargetLowering::TypeSplitVector)
2098 GetSplitVector(Op: N0, Lo&: InLo, Hi&: InHi);
2099 else
2100 std::tie(args&: InLo, args&: InHi) = DAG.SplitVectorOperand(N, OpNo: 0);
2101
2102 EVT InLoVT = InLo.getValueType();
2103 unsigned InNumElements = InLoVT.getVectorNumElements();
2104
2105 EVT OutLoVT, OutHiVT;
2106 std::tie(args&: OutLoVT, args&: OutHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2107 unsigned OutNumElements = OutLoVT.getVectorNumElements();
2108 assert((2 * OutNumElements) <= InNumElements &&
2109 "Illegal extend vector in reg split");
2110
2111 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
2112 // input vector (i.e. we only use InLo):
2113 // OutLo will extend the first OutNumElements from InLo.
2114 // OutHi will extend the next OutNumElements from InLo.
2115
2116 // Shuffle the elements from InLo for OutHi into the bottom elements to
2117 // create a 'fake' InHi.
2118 SmallVector<int, 8> SplitHi(InNumElements, -1);
2119 for (unsigned i = 0; i != OutNumElements; ++i)
2120 SplitHi[i] = i + OutNumElements;
2121 InHi = DAG.getVectorShuffle(VT: InLoVT, dl, N1: InLo, N2: DAG.getPOISON(VT: InLoVT), Mask: SplitHi);
2122
2123 Lo = DAG.getNode(Opcode, DL: dl, VT: OutLoVT, Operand: InLo);
2124 Hi = DAG.getNode(Opcode, DL: dl, VT: OutHiVT, Operand: InHi);
2125}
2126
2127void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
2128 SDValue &Hi) {
2129 unsigned NumOps = N->getNumOperands();
2130 SDValue Chain = N->getOperand(Num: 0);
2131 EVT LoVT, HiVT;
2132 SDLoc dl(N);
2133 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2134
2135 SmallVector<SDValue, 4> OpsLo(NumOps);
2136 SmallVector<SDValue, 4> OpsHi(NumOps);
2137
2138 // The Chain is the first operand.
2139 OpsLo[0] = Chain;
2140 OpsHi[0] = Chain;
2141
2142 // Now process the remaining operands.
2143 for (unsigned i = 1; i < NumOps; ++i) {
2144 SDValue Op = N->getOperand(Num: i);
2145 SDValue OpLo = Op;
2146 SDValue OpHi = Op;
2147
2148 EVT InVT = Op.getValueType();
2149 if (InVT.isVector()) {
2150 // If the input also splits, handle it directly for a
2151 // compile time speedup. Otherwise split it by hand.
2152 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2153 GetSplitVector(Op, Lo&: OpLo, Hi&: OpHi);
2154 else
2155 std::tie(args&: OpLo, args&: OpHi) = DAG.SplitVectorOperand(N, OpNo: i);
2156 }
2157
2158 OpsLo[i] = OpLo;
2159 OpsHi[i] = OpHi;
2160 }
2161
2162 EVT LoValueVTs[] = {LoVT, MVT::Other};
2163 EVT HiValueVTs[] = {HiVT, MVT::Other};
2164 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: LoValueVTs), Ops: OpsLo,
2165 Flags: N->getFlags());
2166 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VTs: HiValueVTs), Ops: OpsHi,
2167 Flags: N->getFlags());
2168
2169 // Build a factor node to remember that this Op is independent of the
2170 // other one.
2171 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
2172 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
2173
2174 // Legalize the chain result - switch anything that used the old chain to
2175 // use the new one.
2176 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2177}
2178
2179SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
2180 SDValue Chain = N->getOperand(Num: 0);
2181 EVT VT = N->getValueType(ResNo: 0);
2182 unsigned NE = VT.getVectorNumElements();
2183 EVT EltVT = VT.getVectorElementType();
2184 SDLoc dl(N);
2185
2186 SmallVector<SDValue, 8> Scalars;
2187 SmallVector<SDValue, 4> Operands(N->getNumOperands());
2188
2189 // If ResNE is 0, fully unroll the vector op.
2190 if (ResNE == 0)
2191 ResNE = NE;
2192 else if (NE > ResNE)
2193 NE = ResNE;
2194
2195 //The results of each unrolled operation, including the chain.
2196 SDVTList ChainVTs = DAG.getVTList(VT1: EltVT, VT2: MVT::Other);
2197 SmallVector<SDValue, 8> Chains;
2198
2199 unsigned i;
2200 for (i = 0; i != NE; ++i) {
2201 Operands[0] = Chain;
2202 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
2203 SDValue Operand = N->getOperand(Num: j);
2204 EVT OperandVT = Operand.getValueType();
2205 if (OperandVT.isVector()) {
2206 EVT OperandEltVT = OperandVT.getVectorElementType();
2207 Operands[j] = DAG.getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
2208 } else {
2209 Operands[j] = Operand;
2210 }
2211 }
2212 SDValue Scalar =
2213 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: ChainVTs, Ops: Operands, Flags: N->getFlags());
2214
2215 //Add in the scalar as well as its chain value to the
2216 //result vectors.
2217 Scalars.push_back(Elt: Scalar);
2218 Chains.push_back(Elt: Scalar.getValue(R: 1));
2219 }
2220
2221 for (; i < ResNE; ++i)
2222 Scalars.push_back(Elt: DAG.getPOISON(VT: EltVT));
2223
2224 // Build a new factor node to connect the chain back together.
2225 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
2226 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2227
2228 // Create a new BUILD_VECTOR node
2229 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: ResNE);
2230 return DAG.getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
2231}
2232
2233void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
2234 SDValue &Lo, SDValue &Hi) {
2235 SDLoc dl(N);
2236 EVT ResVT = N->getValueType(ResNo: 0);
2237 EVT OvVT = N->getValueType(ResNo: 1);
2238 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
2239 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: ResVT);
2240 std::tie(args&: LoOvVT, args&: HiOvVT) = DAG.GetSplitDestVTs(VT: OvVT);
2241
2242 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
2243 if (getTypeAction(VT: ResVT) == TargetLowering::TypeSplitVector) {
2244 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoLHS, Hi&: HiLHS);
2245 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: LoRHS, Hi&: HiRHS);
2246 } else {
2247 std::tie(args&: LoLHS, args&: HiLHS) = DAG.SplitVectorOperand(N, OpNo: 0);
2248 std::tie(args&: LoRHS, args&: HiRHS) = DAG.SplitVectorOperand(N, OpNo: 1);
2249 }
2250
2251 unsigned Opcode = N->getOpcode();
2252 SDVTList LoVTs = DAG.getVTList(VT1: LoResVT, VT2: LoOvVT);
2253 SDVTList HiVTs = DAG.getVTList(VT1: HiResVT, VT2: HiOvVT);
2254 SDNode *LoNode =
2255 DAG.getNode(Opcode, DL: dl, VTList: LoVTs, Ops: {LoLHS, LoRHS}, Flags: N->getFlags()).getNode();
2256 SDNode *HiNode =
2257 DAG.getNode(Opcode, DL: dl, VTList: HiVTs, Ops: {HiLHS, HiRHS}, Flags: N->getFlags()).getNode();
2258
2259 Lo = SDValue(LoNode, ResNo);
2260 Hi = SDValue(HiNode, ResNo);
2261
2262 // Replace the other vector result not being explicitly split here.
2263 unsigned OtherNo = 1 - ResNo;
2264 EVT OtherVT = N->getValueType(ResNo: OtherNo);
2265 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
2266 SetSplitVector(Op: SDValue(N, OtherNo),
2267 Lo: SDValue(LoNode, OtherNo), Hi: SDValue(HiNode, OtherNo));
2268 } else {
2269 SDValue OtherVal = DAG.getNode(
2270 Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT,
2271 N1: SDValue(LoNode, OtherNo), N2: SDValue(HiNode, OtherNo));
2272 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
2273 }
2274}
2275
2276void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
2277 SDValue &Hi) {
2278 SDValue Vec = N->getOperand(Num: 0);
2279 SDValue Elt = N->getOperand(Num: 1);
2280 SDValue Idx = N->getOperand(Num: 2);
2281 SDLoc dl(N);
2282 GetSplitVector(Op: Vec, Lo, Hi);
2283
2284 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
2285 unsigned IdxVal = CIdx->getZExtValue();
2286 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
2287 if (IdxVal < LoNumElts) {
2288 Lo = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl,
2289 VT: Lo.getValueType(), N1: Lo, N2: Elt, N3: Idx);
2290 return;
2291 } else if (!Vec.getValueType().isScalableVector()) {
2292 Hi = DAG.getInsertVectorElt(DL: dl, Vec: Hi, Elt, Idx: IdxVal - LoNumElts);
2293 return;
2294 }
2295 }
2296
2297 // Make the vector elements byte-addressable if they aren't already.
2298 EVT VecVT = Vec.getValueType();
2299 EVT EltVT = VecVT.getVectorElementType();
2300 if (!EltVT.isByteSized()) {
2301 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
2302 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
2303 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
2304 // Extend the element type to match if needed.
2305 if (EltVT.bitsGT(VT: Elt.getValueType()))
2306 Elt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: EltVT, Operand: Elt);
2307 }
2308
2309 // Spill the vector to the stack.
2310 // In cases where the vector is illegal it will be broken down into parts
2311 // and stored in parts - we should use the alignment for the smallest part.
2312 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
2313 SDValue StackPtr =
2314 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
2315 auto &MF = DAG.getMachineFunction();
2316 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
2317 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
2318
2319 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
2320 Alignment: SmallestAlign);
2321
2322 // Store the new element. This may be larger than the vector element type,
2323 // so use a truncating store.
2324 SDValue EltPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
2325 Store = DAG.getTruncStore(
2326 Chain: Store, dl, Val: Elt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF), SVT: EltVT,
2327 Alignment: commonAlignment(A: SmallestAlign,
2328 Offset: EltVT.getFixedSizeInBits() / 8));
2329
2330 EVT LoVT, HiVT;
2331 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: VecVT);
2332
2333 // Load the Lo part from the stack slot.
2334 Lo = DAG.getLoad(VT: LoVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
2335
2336 // Increment the pointer to the other part.
2337 auto Load = cast<LoadSDNode>(Val&: Lo);
2338 MachinePointerInfo MPI = Load->getPointerInfo();
2339 IncrementPointer(N: Load, MemVT: LoVT, MPI, Ptr&: StackPtr);
2340
2341 Hi = DAG.getLoad(VT: HiVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo: MPI, Alignment: SmallestAlign);
2342
2343 // If we adjusted the original type, we need to truncate the results.
2344 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2345 if (LoVT != Lo.getValueType())
2346 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LoVT, Operand: Lo);
2347 if (HiVT != Hi.getValueType())
2348 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiVT, Operand: Hi);
2349}
2350
2351void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
2352 SDValue &Hi) {
2353 EVT LoVT, HiVT;
2354 SDLoc dl(N);
2355 assert(N->getValueType(0).isScalableVector() &&
2356 "Only scalable vectors are supported for STEP_VECTOR");
2357 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2358 SDValue Step = N->getOperand(Num: 0);
2359
2360 Lo = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: LoVT, Operand: Step);
2361
2362 // Hi = Lo + (EltCnt * Step)
2363 EVT EltVT = Step.getValueType();
2364 APInt StepVal = Step->getAsAPIntVal();
2365 SDValue StartOfHi =
2366 DAG.getVScale(DL: dl, VT: EltVT, MulImm: StepVal * LoVT.getVectorMinNumElements());
2367 StartOfHi = DAG.getSExtOrTrunc(Op: StartOfHi, DL: dl, VT: HiVT.getVectorElementType());
2368 StartOfHi = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: HiVT, Operand: StartOfHi);
2369
2370 Hi = DAG.getNode(Opcode: ISD::STEP_VECTOR, DL: dl, VT: HiVT, Operand: Step);
2371 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiVT, N1: Hi, N2: StartOfHi);
2372}
2373
2374void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
2375 SDValue &Hi) {
2376 EVT LoVT, HiVT;
2377 SDLoc dl(N);
2378 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2379 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: N->getOperand(Num: 0));
2380 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
2381 Hi = DAG.getPOISON(VT: HiVT);
2382 } else {
2383 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
2384 Hi = Lo;
2385 }
2386}
2387
2388void DAGTypeLegalizer::SplitVecRes_ATOMIC_LOAD(AtomicSDNode *LD, SDValue &Lo,
2389 SDValue &Hi) {
2390 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&
2391 "Extended load during type legalization!");
2392 SDLoc dl(LD);
2393 EVT VT = LD->getValueType(ResNo: 0);
2394 EVT LoVT, HiVT;
2395 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT);
2396
2397 SDValue Ch = LD->getChain();
2398 SDValue Ptr = LD->getBasePtr();
2399
2400 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2401 EVT MemIntVT =
2402 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LD->getMemoryVT().getSizeInBits());
2403 SDValue ALD = DAG.getAtomicLoad(ExtType: LD->getExtensionType(), dl, MemVT: MemIntVT, VT: IntVT,
2404 Chain: Ch, Ptr, MMO: LD->getMemOperand());
2405
2406 EVT LoIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoVT.getSizeInBits());
2407 EVT HiIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HiVT.getSizeInBits());
2408 SDValue ExtractLo, ExtractHi;
2409 SplitInteger(Op: ALD, LoVT: LoIntVT, HiVT: HiIntVT, Lo&: ExtractLo, Hi&: ExtractHi);
2410
2411 Lo = DAG.getBitcast(VT: LoVT, V: ExtractLo);
2412 Hi = DAG.getBitcast(VT: HiVT, V: ExtractHi);
2413
2414 // Legalize the chain result - switch anything that used the old chain to
2415 // use the new one.
2416 ReplaceValueWith(From: SDValue(LD, 1), To: ALD.getValue(R: 1));
2417}
2418
2419void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
2420 SDValue &Hi) {
2421 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
2422 EVT LoVT, HiVT;
2423 SDLoc dl(LD);
2424 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2425
2426 ISD::LoadExtType ExtType = LD->getExtensionType();
2427 SDValue Ch = LD->getChain();
2428 SDValue Ptr = LD->getBasePtr();
2429 SDValue Offset = DAG.getPOISON(VT: Ptr.getValueType());
2430 EVT MemoryVT = LD->getMemoryVT();
2431 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
2432 AAMDNodes AAInfo = LD->getAAInfo();
2433
2434 EVT LoMemVT, HiMemVT;
2435 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2436
2437 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
2438 SDValue Value, NewChain;
2439 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
2440 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Value, DL: dl);
2441 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
2442 return;
2443 }
2444
2445 Lo = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2446 PtrInfo: LD->getPointerInfo(), MemVT: LoMemVT, Alignment: LD->getBaseAlign(), MMOFlags,
2447 Metadata: AAInfo);
2448
2449 MachinePointerInfo MPI;
2450 IncrementPointer(N: LD, MemVT: LoMemVT, MPI, Ptr);
2451
2452 Hi = DAG.getLoad(AM: ISD::UNINDEXED, ExtType, VT: HiVT, dl, Chain: Ch, Ptr, Offset, PtrInfo: MPI,
2453 MemVT: HiMemVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
2454
2455 // Build a factor node to remember that this load is independent of the
2456 // other one.
2457 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2458 N2: Hi.getValue(R: 1));
2459
2460 // Legalize the chain result - switch anything that used the old chain to
2461 // use the new one.
2462 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2463}
2464
2465void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
2466 SDValue &Hi) {
2467 assert(LD->isUnindexed() && "Indexed VP load during type legalization!");
2468 EVT LoVT, HiVT;
2469 SDLoc dl(LD);
2470 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2471
2472 ISD::LoadExtType ExtType = LD->getExtensionType();
2473 SDValue Ch = LD->getChain();
2474 SDValue Ptr = LD->getBasePtr();
2475 SDValue Offset = LD->getOffset();
2476 assert(Offset.isUndef() && "Unexpected indexed variable-length load offset");
2477 Align Alignment = LD->getBaseAlign();
2478 SDValue Mask = LD->getMask();
2479 SDValue EVL = LD->getVectorLength();
2480 EVT MemoryVT = LD->getMemoryVT();
2481
2482 EVT LoMemVT, HiMemVT;
2483 bool HiIsEmpty = false;
2484 std::tie(args&: LoMemVT, args&: HiMemVT) =
2485 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2486
2487 // Split Mask operand
2488 SDValue MaskLo, MaskHi;
2489 if (Mask.getOpcode() == ISD::SETCC) {
2490 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2491 } else {
2492 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2493 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2494 else
2495 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2496 }
2497
2498 // Split EVL operand
2499 SDValue EVLLo, EVLHi;
2500 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2501
2502 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2503 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2504 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2505 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2506
2507 Lo =
2508 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: LoVT, dl, Chain: Ch, Ptr, Offset,
2509 Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO, IsExpanding: LD->isExpandingLoad());
2510
2511 if (HiIsEmpty) {
2512 // The hi vp_load has zero storage size. We therefore simply set it to
2513 // the low vp_load and rely on subsequent removal from the chain.
2514 Hi = Lo;
2515 } else {
2516 // Generate hi vp_load.
2517 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2518 IsCompressedMemory: LD->isExpandingLoad());
2519
2520 MachinePointerInfo MPI;
2521 if (LoMemVT.isScalableVector())
2522 MPI = MachinePointerInfo(LD->getPointerInfo().getAddrSpace());
2523 else
2524 MPI = LD->getPointerInfo().getWithOffset(
2525 O: LoMemVT.getStoreSize().getFixedValue());
2526
2527 MMO = DAG.getMachineFunction().getMachineMemOperand(
2528 PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2529 BaseAlignment: Alignment, Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2530
2531 Hi = DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType, VT: HiVT, dl, Chain: Ch, Ptr,
2532 Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
2533 IsExpanding: LD->isExpandingLoad());
2534 }
2535
2536 // Build a factor node to remember that this load is independent of the
2537 // other one.
2538 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2539 N2: Hi.getValue(R: 1));
2540
2541 // Legalize the chain result - switch anything that used the old chain to
2542 // use the new one.
2543 ReplaceValueWith(From: SDValue(LD, 1), To: Ch);
2544}
2545
2546void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
2547 SDValue &Hi) {
2548 SDLoc dl(LD);
2549 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: LD->getValueType(ResNo: 0));
2550
2551 SDValue Ch = LD->getChain();
2552 SDValue Ptr = LD->getBasePtr();
2553 Align Alignment = LD->getBaseAlign();
2554 SDValue Mask = LD->getMask();
2555 SDValue EVL = LD->getVectorLength();
2556
2557 // Split Mask operand
2558 SDValue MaskLo, MaskHi;
2559 if (Mask.getOpcode() == ISD::SETCC) {
2560 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2561 } else {
2562 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2563 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2564 else
2565 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2566 }
2567
2568 // Split EVL operand
2569 auto [EVLLo, EVLHi] = DAG.SplitEVL(N: EVL, VecVT: LD->getValueType(ResNo: 0), DL: dl);
2570
2571 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2572 PtrInfo: LD->getPointerInfo(), F: MachineMemOperand::MOLoad,
2573 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2574 Metadata: MMOMetadata(LD->getAAInfo(), LD->getRanges()));
2575
2576 Lo = DAG.getLoadFFVP(VT: LoVT, DL: dl, Chain: Ch, Ptr, Mask: MaskLo, EVL: EVLLo, MMO);
2577
2578 // Fill the upper half with poison.
2579 Hi = DAG.getPOISON(VT: HiVT);
2580
2581 ReplaceValueWith(From: SDValue(LD, 1), To: Lo.getValue(R: 1));
2582 ReplaceValueWith(From: SDValue(LD, 2), To: Lo.getValue(R: 2));
2583}
2584
2585void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
2586 SDValue &Lo, SDValue &Hi) {
2587 assert(SLD->isUnindexed() &&
2588 "Indexed VP strided load during type legalization!");
2589 assert(SLD->getOffset().isUndef() &&
2590 "Unexpected indexed variable-length load offset");
2591
2592 SDLoc DL(SLD);
2593
2594 EVT LoVT, HiVT;
2595 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: SLD->getValueType(ResNo: 0));
2596
2597 EVT LoMemVT, HiMemVT;
2598 bool HiIsEmpty = false;
2599 std::tie(args&: LoMemVT, args&: HiMemVT) =
2600 DAG.GetDependentSplitDestVTs(VT: SLD->getMemoryVT(), EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2601
2602 SDValue Mask = SLD->getMask();
2603 SDValue LoMask, HiMask;
2604 if (Mask.getOpcode() == ISD::SETCC) {
2605 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
2606 } else {
2607 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2608 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
2609 else
2610 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
2611 }
2612
2613 SDValue LoEVL, HiEVL;
2614 std::tie(args&: LoEVL, args&: HiEVL) =
2615 DAG.SplitEVL(N: SLD->getVectorLength(), VecVT: SLD->getValueType(ResNo: 0), DL);
2616
2617 // Generate the low vp_strided_load
2618 Lo = DAG.getStridedLoadVP(
2619 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: LoVT, DL,
2620 Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(), Stride: SLD->getStride(),
2621 Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
2622
2623 if (HiIsEmpty) {
2624 // The high vp_strided_load has zero storage size. We therefore simply set
2625 // it to the low vp_strided_load and rely on subsequent removal from the
2626 // chain.
2627 Hi = Lo;
2628 } else {
2629 // Generate the high vp_strided_load.
2630 // To calculate the high base address, we need to sum to the low base
2631 // address stride number of bytes for each element already loaded by low,
2632 // that is: Ptr = Ptr + (LoEVL * Stride)
2633 EVT PtrVT = SLD->getBasePtr().getValueType();
2634 SDValue Increment =
2635 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
2636 N2: DAG.getSExtOrTrunc(Op: SLD->getStride(), DL, VT: PtrVT));
2637 SDValue Ptr =
2638 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SLD->getBasePtr(), N2: Increment);
2639
2640 Align Alignment = SLD->getBaseAlign();
2641 if (LoMemVT.isScalableVector())
2642 Alignment = commonAlignment(
2643 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
2644
2645 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2646 PtrInfo: MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
2647 F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
2648 BaseAlignment: Alignment, Metadata: MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
2649
2650 Hi = DAG.getStridedLoadVP(AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(),
2651 VT: HiVT, DL, Chain: SLD->getChain(), Ptr, Offset: SLD->getOffset(),
2652 Stride: SLD->getStride(), Mask: HiMask, EVL: HiEVL, MemVT: HiMemVT, MMO,
2653 IsExpanding: SLD->isExpandingLoad());
2654 }
2655
2656 // Build a factor node to remember that this load is independent of the
2657 // other one.
2658 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
2659 N2: Hi.getValue(R: 1));
2660
2661 // Legalize the chain result - switch anything that used the old chain to
2662 // use the new one.
2663 ReplaceValueWith(From: SDValue(SLD, 1), To: Ch);
2664}
2665
2666void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
2667 SDValue &Lo, SDValue &Hi) {
2668 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
2669 EVT LoVT, HiVT;
2670 SDLoc dl(MLD);
2671 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: MLD->getValueType(ResNo: 0));
2672
2673 SDValue Ch = MLD->getChain();
2674 SDValue Ptr = MLD->getBasePtr();
2675 SDValue Offset = MLD->getOffset();
2676 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
2677 SDValue Mask = MLD->getMask();
2678 SDValue PassThru = MLD->getPassThru();
2679 Align Alignment = MLD->getBaseAlign();
2680 ISD::LoadExtType ExtType = MLD->getExtensionType();
2681 MachineMemOperand::Flags MMOFlags = MLD->getMemOperand()->getFlags();
2682
2683 // Split Mask operand
2684 SDValue MaskLo, MaskHi;
2685 if (Mask.getOpcode() == ISD::SETCC) {
2686 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2687 } else {
2688 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
2689 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
2690 else
2691 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL: dl);
2692 }
2693
2694 EVT MemoryVT = MLD->getMemoryVT();
2695 EVT LoMemVT, HiMemVT;
2696 bool HiIsEmpty = false;
2697 std::tie(args&: LoMemVT, args&: HiMemVT) =
2698 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: LoVT, HiIsEmpty: &HiIsEmpty);
2699
2700 SDValue PassThruLo, PassThruHi;
2701 if (getTypeAction(VT: PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2702 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2703 else
2704 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2705
2706 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2707 PtrInfo: MLD->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2708 BaseAlignment: Alignment, Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
2709
2710 Lo = DAG.getMaskedLoad(VT: LoVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskLo, Src0: PassThruLo, MemVT: LoMemVT,
2711 MMO, AM: MLD->getAddressingMode(), ExtType,
2712 IsExpanding: MLD->isExpandingLoad());
2713
2714 if (HiIsEmpty) {
2715 // The hi masked load has zero storage size. We therefore simply set it to
2716 // the low masked load and rely on subsequent removal from the chain.
2717 Hi = Lo;
2718 } else {
2719 // Generate hi masked load.
2720 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL: dl, DataVT: LoMemVT, DAG,
2721 IsCompressedMemory: MLD->isExpandingLoad());
2722
2723 MachinePointerInfo MPI;
2724 if (LoMemVT.isScalableVector())
2725 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
2726 else
2727 MPI = MLD->getPointerInfo().getWithOffset(
2728 O: LoMemVT.getStoreSize().getFixedValue());
2729
2730 MMO = DAG.getMachineFunction().getMachineMemOperand(
2731 PtrInfo: MPI, F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
2732 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
2733
2734 Hi = DAG.getMaskedLoad(VT: HiVT, dl, Chain: Ch, Base: Ptr, Offset, Mask: MaskHi, Src0: PassThruHi,
2735 MemVT: HiMemVT, MMO, AM: MLD->getAddressingMode(), ExtType,
2736 IsExpanding: MLD->isExpandingLoad());
2737 }
2738
2739 // Build a factor node to remember that this load is independent of the
2740 // other one.
2741 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2742 N2: Hi.getValue(R: 1));
2743
2744 // Legalize the chain result - switch anything that used the old chain to
2745 // use the new one.
2746 ReplaceValueWith(From: SDValue(MLD, 1), To: Ch);
2747
2748}
2749
2750void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
2751 SDValue &Hi, bool SplitSETCC) {
2752 EVT LoVT, HiVT;
2753 SDLoc dl(N);
2754 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2755
2756 SDValue Ch = N->getChain();
2757 SDValue Ptr = N->getBasePtr();
2758 struct Operands {
2759 SDValue Mask;
2760 SDValue Index;
2761 SDValue Scale;
2762 } Ops = [&]() -> Operands {
2763 if (auto *MSC = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2764 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale()};
2765 }
2766 auto *VPSC = cast<VPGatherSDNode>(Val: N);
2767 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale()};
2768 }();
2769
2770 EVT MemoryVT = N->getMemoryVT();
2771 Align Alignment = N->getBaseAlign();
2772
2773 // Split Mask operand
2774 SDValue MaskLo, MaskHi;
2775 if (SplitSETCC && Ops.Mask.getOpcode() == ISD::SETCC) {
2776 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
2777 } else {
2778 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL: dl);
2779 }
2780
2781 EVT LoMemVT, HiMemVT;
2782 // Split MemoryVT
2783 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
2784
2785 SDValue IndexHi, IndexLo;
2786 if (getTypeAction(VT: Ops.Index.getValueType()) ==
2787 TargetLowering::TypeSplitVector)
2788 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
2789 else
2790 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL: dl);
2791
2792 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2793 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2794 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
2795 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
2796
2797 if (auto *MGT = dyn_cast<MaskedGatherSDNode>(Val: N)) {
2798 SDValue PassThru = MGT->getPassThru();
2799 SDValue PassThruLo, PassThruHi;
2800 if (getTypeAction(VT: PassThru.getValueType()) ==
2801 TargetLowering::TypeSplitVector)
2802 GetSplitVector(Op: PassThru, Lo&: PassThruLo, Hi&: PassThruHi);
2803 else
2804 std::tie(args&: PassThruLo, args&: PassThruHi) = DAG.SplitVector(N: PassThru, DL: dl);
2805
2806 ISD::LoadExtType ExtType = MGT->getExtensionType();
2807 ISD::MemIndexType IndexTy = MGT->getIndexType();
2808
2809 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Ops.Scale};
2810 Lo = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), MemVT: LoMemVT, dl,
2811 Ops: OpsLo, MMO, IndexType: IndexTy, ExtTy: ExtType);
2812
2813 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Ops.Scale};
2814 Hi = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), MemVT: HiMemVT, dl,
2815 Ops: OpsHi, MMO, IndexType: IndexTy, ExtTy: ExtType);
2816 } else {
2817 auto *VPGT = cast<VPGatherSDNode>(Val: N);
2818 SDValue EVLLo, EVLHi;
2819 std::tie(args&: EVLLo, args&: EVLHi) =
2820 DAG.SplitEVL(N: VPGT->getVectorLength(), VecVT: MemoryVT, DL: dl);
2821
2822 SDValue OpsLo[] = {Ch, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2823 Lo = DAG.getGatherVP(VTs: DAG.getVTList(VT1: LoVT, VT2: MVT::Other), VT: LoMemVT, dl, Ops: OpsLo,
2824 MMO, IndexType: VPGT->getIndexType());
2825
2826 SDValue OpsHi[] = {Ch, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2827 Hi = DAG.getGatherVP(VTs: DAG.getVTList(VT1: HiVT, VT2: MVT::Other), VT: HiMemVT, dl, Ops: OpsHi,
2828 MMO, IndexType: VPGT->getIndexType());
2829 }
2830
2831 // Build a factor node to remember that this load is independent of the
2832 // other one.
2833 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
2834 N2: Hi.getValue(R: 1));
2835
2836 // Legalize the chain result - switch anything that used the old chain to
2837 // use the new one.
2838 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
2839}
2840
2841void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo,
2842 SDValue &Hi) {
2843 // This is not "trivial", as there is a dependency between the two subvectors.
2844 // Depending on the number of 1s in the mask, the elements from the Hi vector
2845 // need to be moved to the Lo vector. Passthru values make this even harder.
2846 // We try to use VECTOR_COMPRESS if the target has custom lowering with
2847 // smaller types and passthru is undef, as it is most likely faster than the
2848 // fully expand path. Otherwise, just do the full expansion as one "big"
2849 // operation and then extract the Lo and Hi vectors from that. This gets
2850 // rid of VECTOR_COMPRESS and all other operands can be legalized later.
2851 SDLoc DL(N);
2852 EVT VecVT = N->getValueType(ResNo: 0);
2853
2854 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
2855 bool HasCustomLowering = false;
2856 EVT CheckVT = LoVT;
2857 while (CheckVT.getVectorMinNumElements() > 1) {
2858 // TLI.isOperationLegalOrCustom requires a legal type, but we could have a
2859 // custom lowering for illegal types. So we do the checks separately.
2860 if (TLI.isOperationLegal(Op: ISD::VECTOR_COMPRESS, VT: CheckVT) ||
2861 TLI.isOperationCustom(Op: ISD::VECTOR_COMPRESS, VT: CheckVT)) {
2862 HasCustomLowering = true;
2863 break;
2864 }
2865 CheckVT = CheckVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
2866 }
2867
2868 SDValue Passthru = N->getOperand(Num: 2);
2869 if (!HasCustomLowering) {
2870 SDValue Compressed = TLI.expandVECTOR_COMPRESS(Node: N, DAG);
2871 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL, LoVT, HiVT);
2872 return;
2873 }
2874
2875 // Try to VECTOR_COMPRESS smaller vectors and combine via a stack store+load.
2876 SDValue Mask = N->getOperand(Num: 1);
2877 SDValue LoMask, HiMask;
2878 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2879 std::tie(args&: LoMask, args&: HiMask) = SplitMask(Mask);
2880
2881 SDValue UndefPassthru = DAG.getPOISON(VT: LoVT);
2882 Lo = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: LoVT, N1: Lo, N2: LoMask, N3: UndefPassthru);
2883 Hi = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: HiVT, N1: Hi, N2: HiMask, N3: UndefPassthru);
2884
2885 SDValue StackPtr = DAG.CreateStackTemporary(
2886 Bytes: VecVT.getStoreSize(), Alignment: DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false));
2887 MachineFunction &MF = DAG.getMachineFunction();
2888 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(
2889 MF, FI: cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex());
2890
2891 EVT MaskVT = LoMask.getValueType();
2892 assert(MaskVT.getScalarType() == MVT::i1 && "Expected vector of i1s");
2893
2894 // We store LoVec and then insert HiVec starting at offset=|1s| in LoMask.
2895 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
2896 EC: MaskVT.getVectorElementCount());
2897 SDValue WideMask = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideMaskVT, Operand: LoMask);
2898 SDValue Offset = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: WideMask);
2899 Offset = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Offset);
2900
2901 SDValue Chain = DAG.getEntryNode();
2902 Chain = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr: StackPtr, PtrInfo);
2903 Chain = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr: Offset,
2904 PtrInfo: MachinePointerInfo::getUnknownStack(MF));
2905
2906 SDValue Compressed = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
2907 if (!Passthru.isUndef()) {
2908 Compressed =
2909 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Mask, N2: Compressed, N3: Passthru);
2910 }
2911 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Compressed, DL);
2912}
2913
2914void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
2915 assert(N->getValueType(0).isVector() &&
2916 N->getOperand(0).getValueType().isVector() &&
2917 "Operand types must be vectors");
2918
2919 EVT LoVT, HiVT;
2920 SDLoc DL(N);
2921 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2922
2923 // If the input also splits, handle it directly. Otherwise split it by hand.
2924 SDValue LL, LH, RL, RH;
2925 if (getTypeAction(VT: N->getOperand(Num: 0).getValueType()) ==
2926 TargetLowering::TypeSplitVector)
2927 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LL, Hi&: LH);
2928 else
2929 std::tie(args&: LL, args&: LH) = DAG.SplitVectorOperand(N, OpNo: 0);
2930
2931 if (getTypeAction(VT: N->getOperand(Num: 1).getValueType()) ==
2932 TargetLowering::TypeSplitVector)
2933 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RL, Hi&: RH);
2934 else
2935 std::tie(args&: RL, args&: RH) = DAG.SplitVectorOperand(N, OpNo: 1);
2936
2937 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LoVT, N1: LL, N2: RL, N3: N->getOperand(Num: 2));
2938 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HiVT, N1: LH, N2: RH, N3: N->getOperand(Num: 2));
2939}
2940
2941void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
2942 SDValue &Hi) {
2943 // Get the dest types - they may not match the input types, e.g. int_to_fp.
2944 EVT LoVT, HiVT;
2945 SDLoc dl(N);
2946 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2947
2948 // If the input also splits, handle it directly for a compile time speedup.
2949 // Otherwise split it by hand.
2950 EVT InVT = N->getOperand(Num: 0).getValueType();
2951 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2952 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2953 else
2954 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2955
2956 const SDNodeFlags Flags = N->getFlags();
2957 unsigned Opcode = N->getOpcode();
2958 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP) {
2959 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2960 N4: N->getOperand(Num: 3), Flags);
2961 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
2962 N4: N->getOperand(Num: 3), Flags);
2963 return;
2964 }
2965
2966 if (Opcode == ISD::FP_ROUND || Opcode == ISD::AssertNoFPClass ||
2967 Opcode == ISD::CONVERT_FROM_ARBITRARY_FP) {
2968 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, N1: Lo, N2: N->getOperand(Num: 1), Flags);
2969 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, N1: Hi, N2: N->getOperand(Num: 1), Flags);
2970 } else {
2971 Lo = DAG.getNode(Opcode, DL: dl, VT: LoVT, Operand: Lo, Flags);
2972 Hi = DAG.getNode(Opcode, DL: dl, VT: HiVT, Operand: Hi, Flags);
2973 }
2974}
2975
2976void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo,
2977 SDValue &Hi) {
2978 SDLoc dl(N);
2979 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
2980
2981 // If the input also splits, handle it directly for a compile time speedup.
2982 // Otherwise split it by hand.
2983 EVT InVT = N->getOperand(Num: 0).getValueType();
2984 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
2985 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
2986 else
2987 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
2988
2989 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
2990 unsigned SrcAS = AddrSpaceCastN->getSrcAddressSpace();
2991 unsigned DestAS = AddrSpaceCastN->getDestAddressSpace();
2992 Lo = DAG.getAddrSpaceCast(dl, VT: LoVT, Ptr: Lo, SrcAS, DestAS);
2993 Hi = DAG.getAddrSpaceCast(dl, VT: HiVT, Ptr: Hi, SrcAS, DestAS);
2994}
2995
2996void DAGTypeLegalizer::SplitVecRes_UnaryOpWithTwoResults(SDNode *N,
2997 unsigned ResNo,
2998 SDValue &Lo,
2999 SDValue &Hi) {
3000 SDLoc dl(N);
3001 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3002 auto [LoVT1, HiVT1] = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 1));
3003
3004 // If the input also splits, handle it directly for a compile time speedup.
3005 // Otherwise split it by hand.
3006 EVT InVT = N->getOperand(Num: 0).getValueType();
3007 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector)
3008 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
3009 else
3010 std::tie(args&: Lo, args&: Hi) = DAG.SplitVectorOperand(N, OpNo: 0);
3011
3012 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {LoVT, LoVT1}, Ops: Lo, Flags: N->getFlags());
3013 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {HiVT, HiVT1}, Ops: Hi, Flags: N->getFlags());
3014
3015 SDNode *HiNode = Hi.getNode();
3016 SDNode *LoNode = Lo.getNode();
3017
3018 // Replace the other vector result not being explicitly split here.
3019 unsigned OtherNo = 1 - ResNo;
3020 EVT OtherVT = N->getValueType(ResNo: OtherNo);
3021 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeSplitVector) {
3022 SetSplitVector(Op: SDValue(N, OtherNo), Lo: SDValue(LoNode, OtherNo),
3023 Hi: SDValue(HiNode, OtherNo));
3024 } else {
3025 SDValue OtherVal =
3026 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: OtherVT, N1: SDValue(LoNode, OtherNo),
3027 N2: SDValue(HiNode, OtherNo));
3028 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
3029 }
3030}
3031
3032void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
3033 SDValue &Hi) {
3034 SDLoc dl(N);
3035 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3036 EVT DestVT = N->getValueType(ResNo: 0);
3037 EVT LoVT, HiVT;
3038 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: DestVT);
3039
3040 // We can do better than a generic split operation if the extend is doing
3041 // more than just doubling the width of the elements and the following are
3042 // true:
3043 // - The number of vector elements is even,
3044 // - the source type is legal,
3045 // - the type of a split source is illegal,
3046 // - the type of an extended (by doubling element size) source is legal, and
3047 // - the type of that extended source when split is legal.
3048 //
3049 // This won't necessarily completely legalize the operation, but it will
3050 // more effectively move in the right direction and prevent falling down
3051 // to scalarization in many cases due to the input vector being split too
3052 // far.
3053 if (SrcVT.getVectorElementCount().isKnownEven() &&
3054 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
3055 LLVMContext &Ctx = *DAG.getContext();
3056 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Context&: Ctx);
3057 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Context&: Ctx);
3058
3059 EVT SplitLoVT, SplitHiVT;
3060 std::tie(args&: SplitLoVT, args&: SplitHiVT) = DAG.GetSplitDestVTs(VT: NewSrcVT);
3061 if (TLI.isTypeLegal(VT: SrcVT) && !TLI.isTypeLegal(VT: SplitSrcVT) &&
3062 TLI.isTypeLegal(VT: NewSrcVT) && TLI.isTypeLegal(VT: SplitLoVT)) {
3063 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
3064 N->dump(&DAG); dbgs() << "\n");
3065 // Extend the source vector by one step.
3066 SDValue NewSrc =
3067 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewSrcVT, Operand: N->getOperand(Num: 0));
3068 // Get the low and high halves of the new, extended one step, vector.
3069 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: NewSrc, DL: dl);
3070 // Extend those vector halves the rest of the way.
3071 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: LoVT, Operand: Lo);
3072 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: HiVT, Operand: Hi);
3073 return;
3074 }
3075 }
3076 // Fall back to the generic unary operator splitting otherwise.
3077 SplitVecRes_UnaryOp(N, Lo, Hi);
3078}
3079
3080void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
3081 SDValue &Lo, SDValue &Hi) {
3082 // The low and high parts of the original input give four input vectors.
3083 SDValue Inputs[4];
3084 SDLoc DL(N);
3085 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: Inputs[0], Hi&: Inputs[1]);
3086 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Inputs[2], Hi&: Inputs[3]);
3087 EVT NewVT = Inputs[0].getValueType();
3088 unsigned NewElts = NewVT.getVectorNumElements();
3089
3090 auto &&IsConstant = [](const SDValue &N) {
3091 APInt SplatValue;
3092 return N.getResNo() == 0 &&
3093 (ISD::isConstantSplatVector(N: N.getNode(), SplatValue) ||
3094 ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()));
3095 };
3096 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &DL](SDValue &Input1,
3097 SDValue &Input2,
3098 ArrayRef<int> Mask) {
3099 assert(Input1->getOpcode() == ISD::BUILD_VECTOR &&
3100 Input2->getOpcode() == ISD::BUILD_VECTOR &&
3101 "Expected build vector node.");
3102 EVT EltVT = NewVT.getVectorElementType();
3103 SmallVector<SDValue> Ops(NewElts, DAG.getPOISON(VT: EltVT));
3104 for (unsigned I = 0; I < NewElts; ++I) {
3105 if (Mask[I] == PoisonMaskElem)
3106 continue;
3107 unsigned Idx = Mask[I];
3108 if (Idx >= NewElts)
3109 Ops[I] = Input2.getOperand(i: Idx - NewElts);
3110 else
3111 Ops[I] = Input1.getOperand(i: Idx);
3112 // Make the type of all elements the same as the element type.
3113 if (Ops[I].getValueType().bitsGT(VT: EltVT))
3114 Ops[I] = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Ops[I]);
3115 }
3116 return DAG.getBuildVector(VT: NewVT, DL, Ops);
3117 };
3118
3119 // If Lo or Hi uses elements from at most two of the four input vectors, then
3120 // express it as a vector shuffle of those two inputs. Otherwise extract the
3121 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
3122 SmallVector<int> OrigMask(N->getMask());
3123 // Try to pack incoming shuffles/inputs.
3124 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT, this, NewElts,
3125 &DL](SmallVectorImpl<int> &Mask) {
3126 // Check if all inputs are shuffles of the same operands or non-shuffles.
3127 MapVector<std::pair<SDValue, SDValue>, SmallVector<unsigned>> ShufflesIdxs;
3128 for (unsigned Idx = 0; Idx < std::size(Inputs); ++Idx) {
3129 SDValue Input = Inputs[Idx];
3130 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Input.getNode());
3131 if (!Shuffle ||
3132 Input.getOperand(i: 0).getValueType() != Input.getValueType())
3133 continue;
3134 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 0), y: Input.getOperand(i: 1))]
3135 .push_back(Elt: Idx);
3136 ShufflesIdxs[std::make_pair(x: Input.getOperand(i: 1), y: Input.getOperand(i: 0))]
3137 .push_back(Elt: Idx);
3138 }
3139 for (auto &P : ShufflesIdxs) {
3140 if (P.second.size() < 2)
3141 continue;
3142 // Use shuffles operands instead of shuffles themselves.
3143 // 1. Adjust mask.
3144 for (int &Idx : Mask) {
3145 if (Idx == PoisonMaskElem)
3146 continue;
3147 unsigned SrcRegIdx = Idx / NewElts;
3148 if (Inputs[SrcRegIdx].isUndef()) {
3149 Idx = PoisonMaskElem;
3150 continue;
3151 }
3152 auto *Shuffle =
3153 dyn_cast<ShuffleVectorSDNode>(Val: Inputs[SrcRegIdx].getNode());
3154 if (!Shuffle || !is_contained(Range&: P.second, Element: SrcRegIdx))
3155 continue;
3156 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3157 if (MaskElt == PoisonMaskElem) {
3158 Idx = PoisonMaskElem;
3159 continue;
3160 }
3161 Idx = MaskElt % NewElts +
3162 P.second[Shuffle->getOperand(Num: MaskElt / NewElts) == P.first.first
3163 ? 0
3164 : 1] *
3165 NewElts;
3166 }
3167 // 2. Update inputs.
3168 Inputs[P.second[0]] = P.first.first;
3169 Inputs[P.second[1]] = P.first.second;
3170 // Clear the pair data.
3171 P.second.clear();
3172 ShufflesIdxs[std::make_pair(x&: P.first.second, y&: P.first.first)].clear();
3173 }
3174 // Check if any concat_vectors can be simplified.
3175 SmallBitVector UsedSubVector(2 * std::size(Inputs));
3176 for (int &Idx : Mask) {
3177 if (Idx == PoisonMaskElem)
3178 continue;
3179 unsigned SrcRegIdx = Idx / NewElts;
3180 if (Inputs[SrcRegIdx].isUndef()) {
3181 Idx = PoisonMaskElem;
3182 continue;
3183 }
3184 TargetLowering::LegalizeTypeAction TypeAction =
3185 getTypeAction(VT: Inputs[SrcRegIdx].getValueType());
3186 if (Inputs[SrcRegIdx].getOpcode() == ISD::CONCAT_VECTORS &&
3187 Inputs[SrcRegIdx].getNumOperands() == 2 &&
3188 !Inputs[SrcRegIdx].getOperand(i: 1).isUndef() &&
3189 (TypeAction == TargetLowering::TypeLegal ||
3190 TypeAction == TargetLowering::TypeWidenVector))
3191 UsedSubVector.set(2 * SrcRegIdx + (Idx % NewElts) / (NewElts / 2));
3192 }
3193 if (UsedSubVector.count() > 1) {
3194 SmallVector<SmallVector<std::pair<unsigned, int>, 2>> Pairs;
3195 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3196 if (UsedSubVector.test(Idx: 2 * I) == UsedSubVector.test(Idx: 2 * I + 1))
3197 continue;
3198 if (Pairs.empty() || Pairs.back().size() == 2)
3199 Pairs.emplace_back();
3200 if (UsedSubVector.test(Idx: 2 * I)) {
3201 Pairs.back().emplace_back(Args&: I, Args: 0);
3202 } else {
3203 assert(UsedSubVector.test(2 * I + 1) &&
3204 "Expected to be used one of the subvectors.");
3205 Pairs.back().emplace_back(Args&: I, Args: 1);
3206 }
3207 }
3208 if (!Pairs.empty() && Pairs.front().size() > 1) {
3209 // Adjust mask.
3210 for (int &Idx : Mask) {
3211 if (Idx == PoisonMaskElem)
3212 continue;
3213 unsigned SrcRegIdx = Idx / NewElts;
3214 auto *It = find_if(
3215 Range&: Pairs, P: [SrcRegIdx](ArrayRef<std::pair<unsigned, int>> Idxs) {
3216 return Idxs.front().first == SrcRegIdx ||
3217 Idxs.back().first == SrcRegIdx;
3218 });
3219 if (It == Pairs.end())
3220 continue;
3221 Idx = It->front().first * NewElts + (Idx % NewElts) % (NewElts / 2) +
3222 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
3223 }
3224 // Adjust inputs.
3225 for (ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
3226 Inputs[Idxs.front().first] = DAG.getNode(
3227 Opcode: ISD::CONCAT_VECTORS, DL,
3228 VT: Inputs[Idxs.front().first].getValueType(),
3229 N1: Inputs[Idxs.front().first].getOperand(i: Idxs.front().second),
3230 N2: Inputs[Idxs.back().first].getOperand(i: Idxs.back().second));
3231 }
3232 }
3233 }
3234 bool Changed;
3235 do {
3236 // Try to remove extra shuffles (except broadcasts) and shuffles with the
3237 // reused operands.
3238 Changed = false;
3239 for (unsigned I = 0; I < std::size(Inputs); ++I) {
3240 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: Inputs[I].getNode());
3241 if (!Shuffle)
3242 continue;
3243 if (Shuffle->getOperand(Num: 0).getValueType() != NewVT)
3244 continue;
3245 int Op = -1;
3246 if (!Inputs[I].hasOneUse() && Shuffle->getOperand(Num: 1).isUndef() &&
3247 !Shuffle->isSplat()) {
3248 Op = 0;
3249 } else if (!Inputs[I].hasOneUse() &&
3250 !Shuffle->getOperand(Num: 1).isUndef()) {
3251 // Find the only used operand, if possible.
3252 for (int &Idx : Mask) {
3253 if (Idx == PoisonMaskElem)
3254 continue;
3255 unsigned SrcRegIdx = Idx / NewElts;
3256 if (SrcRegIdx != I)
3257 continue;
3258 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3259 if (MaskElt == PoisonMaskElem) {
3260 Idx = PoisonMaskElem;
3261 continue;
3262 }
3263 int OpIdx = MaskElt / NewElts;
3264 if (Op == -1) {
3265 Op = OpIdx;
3266 continue;
3267 }
3268 if (Op != OpIdx) {
3269 Op = -1;
3270 break;
3271 }
3272 }
3273 }
3274 if (Op < 0) {
3275 // Try to check if one of the shuffle operands is used already.
3276 for (int OpIdx = 0; OpIdx < 2; ++OpIdx) {
3277 if (Shuffle->getOperand(Num: OpIdx).isUndef())
3278 continue;
3279 auto *It = find(Range&: Inputs, Val: Shuffle->getOperand(Num: OpIdx));
3280 if (It == std::end(arr&: Inputs))
3281 continue;
3282 int FoundOp = std::distance(first: std::begin(arr&: Inputs), last: It);
3283 // Found that operand is used already.
3284 // 1. Fix the mask for the reused operand.
3285 for (int &Idx : Mask) {
3286 if (Idx == PoisonMaskElem)
3287 continue;
3288 unsigned SrcRegIdx = Idx / NewElts;
3289 if (SrcRegIdx != I)
3290 continue;
3291 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3292 if (MaskElt == PoisonMaskElem) {
3293 Idx = PoisonMaskElem;
3294 continue;
3295 }
3296 int MaskIdx = MaskElt / NewElts;
3297 if (OpIdx == MaskIdx)
3298 Idx = MaskElt % NewElts + FoundOp * NewElts;
3299 }
3300 // 2. Set Op to the unused OpIdx.
3301 Op = (OpIdx + 1) % 2;
3302 break;
3303 }
3304 }
3305 if (Op >= 0) {
3306 Changed = true;
3307 Inputs[I] = Shuffle->getOperand(Num: Op);
3308 // Adjust mask.
3309 for (int &Idx : Mask) {
3310 if (Idx == PoisonMaskElem)
3311 continue;
3312 unsigned SrcRegIdx = Idx / NewElts;
3313 if (SrcRegIdx != I)
3314 continue;
3315 int MaskElt = Shuffle->getMaskElt(Idx: Idx % NewElts);
3316 int OpIdx = MaskElt / NewElts;
3317 if (OpIdx != Op)
3318 continue;
3319 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
3320 }
3321 }
3322 }
3323 } while (Changed);
3324 };
3325 TryPeekThroughShufflesInputs(OrigMask);
3326 // Proces unique inputs.
3327 auto &&MakeUniqueInputs = [&Inputs, &IsConstant,
3328 NewElts](SmallVectorImpl<int> &Mask) {
3329 SetVector<SDValue> UniqueInputs;
3330 SetVector<SDValue> UniqueConstantInputs;
3331 for (const auto &I : Inputs) {
3332 if (IsConstant(I))
3333 UniqueConstantInputs.insert(X: I);
3334 else if (!I.isUndef())
3335 UniqueInputs.insert(X: I);
3336 }
3337 // Adjust mask in case of reused inputs. Also, need to insert constant
3338 // inputs at first, otherwise it affects the final outcome.
3339 if (UniqueInputs.size() != std::size(Inputs)) {
3340 auto &&UniqueVec = UniqueInputs.takeVector();
3341 auto &&UniqueConstantVec = UniqueConstantInputs.takeVector();
3342 unsigned ConstNum = UniqueConstantVec.size();
3343 for (int &Idx : Mask) {
3344 if (Idx == PoisonMaskElem)
3345 continue;
3346 unsigned SrcRegIdx = Idx / NewElts;
3347 if (Inputs[SrcRegIdx].isUndef()) {
3348 Idx = PoisonMaskElem;
3349 continue;
3350 }
3351 const auto It = find(Range&: UniqueConstantVec, Val: Inputs[SrcRegIdx]);
3352 if (It != UniqueConstantVec.end()) {
3353 Idx = (Idx % NewElts) +
3354 NewElts * std::distance(first: UniqueConstantVec.begin(), last: It);
3355 assert(Idx >= 0 && "Expected defined mask idx.");
3356 continue;
3357 }
3358 const auto RegIt = find(Range&: UniqueVec, Val: Inputs[SrcRegIdx]);
3359 assert(RegIt != UniqueVec.end() && "Cannot find non-const value.");
3360 Idx = (Idx % NewElts) +
3361 NewElts * (std::distance(first: UniqueVec.begin(), last: RegIt) + ConstNum);
3362 assert(Idx >= 0 && "Expected defined mask idx.");
3363 }
3364 copy(Range&: UniqueConstantVec, Out: std::begin(arr&: Inputs));
3365 copy(Range&: UniqueVec, Out: std::next(x: std::begin(arr&: Inputs), n: ConstNum));
3366 }
3367 };
3368 MakeUniqueInputs(OrigMask);
3369 SDValue OrigInputs[4];
3370 copy(Range&: Inputs, Out: std::begin(arr&: OrigInputs));
3371 for (unsigned High = 0; High < 2; ++High) {
3372 SDValue &Output = High ? Hi : Lo;
3373
3374 // Build a shuffle mask for the output, discovering on the fly which
3375 // input vectors to use as shuffle operands.
3376 unsigned FirstMaskIdx = High * NewElts;
3377 SmallVector<int> Mask(NewElts * std::size(Inputs), PoisonMaskElem);
3378 copy(Range: ArrayRef(OrigMask).slice(N: FirstMaskIdx, M: NewElts), Out: Mask.begin());
3379 assert(!Output && "Expected default initialized initial value.");
3380 TryPeekThroughShufflesInputs(Mask);
3381 MakeUniqueInputs(Mask);
3382 SDValue TmpInputs[4];
3383 copy(Range&: Inputs, Out: std::begin(arr&: TmpInputs));
3384 // Track changes in the output registers.
3385 int UsedIdx = -1;
3386 bool SecondIteration = false;
3387 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](unsigned Idx) {
3388 if (UsedIdx < 0) {
3389 UsedIdx = Idx;
3390 return false;
3391 }
3392 if (UsedIdx >= 0 && static_cast<unsigned>(UsedIdx) == Idx)
3393 SecondIteration = true;
3394 return SecondIteration;
3395 };
3396 processShuffleMasks(
3397 Mask, NumOfSrcRegs: std::size(Inputs), NumOfDestRegs: std::size(Inputs),
3398 /*NumOfUsedRegs=*/1,
3399 NoInputAction: [&Output, &DAG = DAG, NewVT]() { Output = DAG.getPOISON(VT: NewVT); },
3400 SingleInputAction: [&Output, &DAG = DAG, NewVT, &DL, &Inputs,
3401 &BuildVector](ArrayRef<int> Mask, unsigned Idx, unsigned /*Unused*/) {
3402 if (Inputs[Idx]->getOpcode() == ISD::BUILD_VECTOR)
3403 Output = BuildVector(Inputs[Idx], Inputs[Idx], Mask);
3404 else
3405 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx],
3406 N2: DAG.getPOISON(VT: NewVT), Mask);
3407 Inputs[Idx] = Output;
3408 },
3409 ManyInputsAction: [&AccumulateResults, &Output, &DAG = DAG, NewVT, &DL, &Inputs,
3410 &TmpInputs, &BuildVector](ArrayRef<int> Mask, unsigned Idx1,
3411 unsigned Idx2, bool /*Unused*/) {
3412 if (AccumulateResults(Idx1)) {
3413 if (Inputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3414 Inputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3415 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
3416 else
3417 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: Inputs[Idx1],
3418 N2: Inputs[Idx2], Mask);
3419 } else {
3420 if (TmpInputs[Idx1]->getOpcode() == ISD::BUILD_VECTOR &&
3421 TmpInputs[Idx2]->getOpcode() == ISD::BUILD_VECTOR)
3422 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
3423 else
3424 Output = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: TmpInputs[Idx1],
3425 N2: TmpInputs[Idx2], Mask);
3426 }
3427 Inputs[Idx1] = Output;
3428 });
3429 copy(Range&: OrigInputs, Out: std::begin(arr&: Inputs));
3430 }
3431}
3432
3433void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3434 EVT OVT = N->getValueType(ResNo: 0);
3435 EVT NVT = OVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
3436 SDValue Chain = N->getOperand(Num: 0);
3437 SDValue Ptr = N->getOperand(Num: 1);
3438 SDValue SV = N->getOperand(Num: 2);
3439 SDLoc dl(N);
3440
3441 const Align Alignment =
3442 DAG.getDataLayout().getABITypeAlign(Ty: NVT.getTypeForEVT(Context&: *DAG.getContext()));
3443
3444 Lo = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV, Align: Alignment.value());
3445 Hi = DAG.getVAArg(VT: NVT, dl, Chain: Lo.getValue(R: 1), Ptr, SV, Align: Alignment.value());
3446 Chain = Hi.getValue(R: 1);
3447
3448 // Modified the chain - switch anything that used the old chain to use
3449 // the new one.
3450 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
3451}
3452
3453void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3454 SDValue &Hi) {
3455 EVT DstVTLo, DstVTHi;
3456 std::tie(args&: DstVTLo, args&: DstVTHi) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3457 SDLoc dl(N);
3458
3459 SDValue SrcLo, SrcHi;
3460 EVT SrcVT = N->getOperand(Num: 0).getValueType();
3461 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeSplitVector)
3462 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SrcLo, Hi&: SrcHi);
3463 else
3464 std::tie(args&: SrcLo, args&: SrcHi) = DAG.SplitVectorOperand(N, OpNo: 0);
3465
3466 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTLo, N1: SrcLo, N2: N->getOperand(Num: 1));
3467 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: DstVTHi, N1: SrcHi, N2: N->getOperand(Num: 1));
3468}
3469
3470void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
3471 SDValue &Hi) {
3472 SDValue InLo, InHi;
3473 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: InLo, Hi&: InHi);
3474 SDLoc DL(N);
3475
3476 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InHi.getValueType(), Operand: InHi);
3477 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: InLo.getValueType(), Operand: InLo);
3478}
3479
3480void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
3481 SDValue &Hi) {
3482 SDLoc DL(N);
3483
3484 SDValue Expanded = TLI.expandVectorSplice(Node: N, DAG);
3485 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Expanded, DL);
3486}
3487
3488void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo,
3489 SDValue &Hi) {
3490 EVT VT = N->getValueType(ResNo: 0);
3491 SDValue Val = N->getOperand(Num: 0);
3492 SDValue Mask = N->getOperand(Num: 1);
3493 SDValue EVL = N->getOperand(Num: 2);
3494 SDLoc DL(N);
3495
3496 // The stack round-trip uses a byte stride, so a sub-byte element (e.g. i1)
3497 // would get stride 0 and alias every lane. Widen to a byte integer, reverse,
3498 // then truncate back.
3499 EVT OrigVT = VT;
3500 if (!VT.getVectorElementType().isByteSized()) {
3501 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3502 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3503 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3504 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Val);
3505 }
3506
3507 // Fallback to VP_STRIDED_STORE to stack followed by VP_LOAD.
3508 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3509
3510 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3511 EC: VT.getVectorElementCount());
3512 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3513 EVT PtrVT = StackPtr.getValueType();
3514 auto &MF = DAG.getMachineFunction();
3515 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3516 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3517
3518 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3519 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3520 BaseAlignment: Alignment);
3521 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3522 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3523 BaseAlignment: Alignment);
3524
3525 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3526 SDValue NumElemMinus1 =
3527 DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: DAG.getZExtOrTrunc(Op: EVL, DL, VT: PtrVT),
3528 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
3529 SDValue StartOffset = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: NumElemMinus1,
3530 N2: DAG.getConstant(Val: EltWidth, DL, VT: PtrVT));
3531 SDValue StorePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: StartOffset);
3532 SDValue Stride = DAG.getConstant(Val: -(int64_t)EltWidth, DL, VT: PtrVT);
3533
3534 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3535 SDValue Store = DAG.getStridedStoreVP(Chain: DAG.getEntryNode(), DL, Val, Ptr: StorePtr,
3536 Offset: DAG.getPOISON(VT: PtrVT), Stride, Mask: TrueMask,
3537 EVL, MemVT, MMO: StoreMMO, AM: ISD::UNINDEXED);
3538
3539 SDValue Load = DAG.getLoadVP(VT, dl: DL, Chain: Store, Ptr: StackPtr, Mask, EVL, MMO: LoadMMO);
3540
3541 // Truncate back if we widened above.
3542 if (OrigVT != VT)
3543 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3544
3545 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Load, DL);
3546}
3547
3548void DAGTypeLegalizer::SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo,
3549 SDValue &Hi) {
3550 EVT VT = N->getValueType(ResNo: 0);
3551 SDValue V1 = N->getOperand(Num: 0);
3552 SDValue V2 = N->getOperand(Num: 1);
3553 int64_t Imm = cast<ConstantSDNode>(Val: N->getOperand(Num: 2))->getSExtValue();
3554 SDValue Mask = N->getOperand(Num: 3);
3555 SDValue EVL1 = N->getOperand(Num: 4);
3556 SDValue EVL2 = N->getOperand(Num: 5);
3557 SDLoc DL(N);
3558
3559 // Since EVL2 is considered the real VL it gets promoted during
3560 // SelectionDAGBuilder. Promote EVL1 here if needed.
3561 if (getTypeAction(VT: EVL1.getValueType()) == TargetLowering::TypePromoteInteger)
3562 EVL1 = ZExtPromotedInteger(Op: EVL1);
3563
3564 // The stack splice addresses elements by byte offset/stride, which breaks for
3565 // a sub-byte element (e.g. i1): getVectorElementPointer asserts and the
3566 // stride is 0. Widen to a byte integer, splice, then truncate back.
3567 EVT OrigVT = VT;
3568 if (!VT.getVectorElementType().isByteSized()) {
3569 EVT WideEltVT = VT.getVectorElementType().changeTypeToInteger();
3570 WideEltVT = WideEltVT.getRoundIntegerType(Context&: *DAG.getContext());
3571 VT = VT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: WideEltVT);
3572 V1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V1);
3573 V2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: V2);
3574 }
3575
3576 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
3577
3578 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
3579 EC: VT.getVectorElementCount() * 2);
3580 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
3581 EVT PtrVT = StackPtr.getValueType();
3582 auto &MF = DAG.getMachineFunction();
3583 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
3584 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
3585
3586 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
3587 PtrInfo, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
3588 BaseAlignment: Alignment);
3589 MachineMemOperand *LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
3590 PtrInfo, F: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer(),
3591 BaseAlignment: Alignment);
3592
3593 SDValue EltByteSize =
3594 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
3595 SDValue EVL1Ptr = DAG.getZExtOrTrunc(Op: EVL1, DL, VT: PtrVT);
3596 SDValue EVL1Bytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: EVL1Ptr, N2: EltByteSize);
3597 // Clip EVL1Bytes to make sure we stay within the stack object.
3598 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
3599 EVL1Bytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: EVL1Bytes, N2: VTBytes);
3600 SDValue StackPtr2 = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: EVL1Bytes, DL);
3601 SDValue PoisonPtr = DAG.getPOISON(VT: PtrVT);
3602
3603 SDValue TrueMask = DAG.getBoolConstant(V: true, DL, VT: Mask.getValueType(), OpVT: VT);
3604 SDValue StoreV1 =
3605 DAG.getStoreVP(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, Offset: PoisonPtr, Mask: TrueMask,
3606 EVL: EVL1, MemVT: V1.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3607
3608 SDValue StoreV2 =
3609 DAG.getStoreVP(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, Offset: PoisonPtr, Mask: TrueMask, EVL: EVL2,
3610 MemVT: V2.getValueType(), MMO: StoreMMO, AM: ISD::UNINDEXED);
3611
3612 SDValue Load;
3613 if (Imm >= 0) {
3614 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index: N->getOperand(Num: 2));
3615 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr, Mask, EVL: EVL2, MMO: LoadMMO);
3616 } else {
3617 uint64_t TrailingElts = -Imm;
3618 unsigned EltWidth = VT.getScalarSizeInBits() / 8;
3619 SDValue TrailingBytes = DAG.getConstant(Val: TrailingElts * EltWidth, DL, VT: PtrVT);
3620
3621 // Make sure TrailingBytes doesn't exceed the size of vec1.
3622 SDValue OffsetToV2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: StackPtr);
3623 TrailingBytes =
3624 DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: OffsetToV2);
3625
3626 // Calculate the start address of the spliced result.
3627 StackPtr2 = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
3628 Load = DAG.getLoadVP(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr2, Mask, EVL: EVL2, MMO: LoadMMO);
3629 }
3630
3631 // Truncate back if we widened above.
3632 if (OrigVT != VT)
3633 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: Load);
3634
3635 EVT LoVT, HiVT;
3636 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: OrigVT);
3637 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: Load,
3638 N2: DAG.getVectorIdxConstant(Val: 0, DL));
3639 Hi =
3640 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: Load,
3641 N2: DAG.getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
3642}
3643
3644void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo,
3645 SDValue &Hi) {
3646 SDLoc DL(N);
3647 SDValue Acc = N->getOperand(Num: 0);
3648 SDValue Input1 = N->getOperand(Num: 1);
3649 SDValue Input2 = N->getOperand(Num: 2);
3650
3651 SDValue AccLo, AccHi;
3652 GetSplitVector(Op: Acc, Lo&: AccLo, Hi&: AccHi);
3653 unsigned Opcode = N->getOpcode();
3654
3655 // If the input types don't need splitting, just accumulate into the
3656 // low part of the accumulator.
3657 if (getTypeAction(VT: Input1.getValueType()) != TargetLowering::TypeSplitVector) {
3658 Lo = DAG.getNode(Opcode, DL, VT: AccLo.getValueType(), N1: AccLo, N2: Input1, N3: Input2);
3659 Hi = AccHi;
3660 return;
3661 }
3662
3663 SDValue Input1Lo, Input1Hi;
3664 SDValue Input2Lo, Input2Hi;
3665 GetSplitVector(Op: Input1, Lo&: Input1Lo, Hi&: Input1Hi);
3666 GetSplitVector(Op: Input2, Lo&: Input2Lo, Hi&: Input2Hi);
3667 EVT ResultVT = AccLo.getValueType();
3668
3669 Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccLo, N2: Input1Lo, N3: Input2Lo);
3670 Hi = DAG.getNode(Opcode, DL, VT: ResultVT, N1: AccHi, N2: Input1Hi, N3: Input2Hi);
3671}
3672
3673void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo,
3674 SDValue &Hi) {
3675 SDLoc DL(N);
3676 SDValue Op0 = N->getOperand(Num: 0);
3677 SDValue Op1 = N->getOperand(Num: 1);
3678 EVT OpVT = Op0.getValueType();
3679
3680 EVT LoVT, HiVT;
3681 std::tie(args&: LoVT, args&: HiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
3682
3683 Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Op0, N2: Op1);
3684 SDValue LoElts = DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
3685 SDValue HiStartVal = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Op0, N2: LoElts);
3686 Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStartVal, N2: Op1);
3687}
3688
3689void DAGTypeLegalizer::SplitVecRes_VECTOR_MATCH(SDNode *N, SDValue &Lo,
3690 SDValue &Hi) {
3691 SDValue SourceLo, SourceHi;
3692 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: SourceLo, Hi&: SourceHi);
3693 SDValue MaskLo, MaskHi;
3694 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
3695 SDLoc DL(N);
3696
3697 Lo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskLo.getValueType(), N1: SourceLo,
3698 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
3699 Hi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: MaskHi.getValueType(), N1: SourceHi,
3700 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
3701}
3702
3703void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
3704 unsigned Factor = N->getNumOperands();
3705
3706 SmallVector<SDValue, 8> Ops(Factor * 2);
3707 for (unsigned i = 0; i != Factor; ++i) {
3708 SDValue OpLo, OpHi;
3709 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3710 Ops[i * 2] = OpLo;
3711 Ops[i * 2 + 1] = OpHi;
3712 }
3713
3714 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3715
3716 SDLoc DL(N);
3717 SDValue ResLo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3718 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
3719 SDValue ResHi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
3720 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
3721
3722 for (unsigned i = 0; i != Factor; ++i)
3723 SetSplitVector(Op: SDValue(N, i), Lo: ResLo.getValue(R: i), Hi: ResHi.getValue(R: i));
3724}
3725
3726void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) {
3727 unsigned Factor = N->getNumOperands();
3728
3729 SmallVector<SDValue, 8> Ops(Factor * 2);
3730 for (unsigned i = 0; i != Factor; ++i) {
3731 SDValue OpLo, OpHi;
3732 GetSplitVector(Op: N->getOperand(Num: i), Lo&: OpLo, Hi&: OpHi);
3733 Ops[i] = OpLo;
3734 Ops[i + Factor] = OpHi;
3735 }
3736
3737 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
3738
3739 SDLoc DL(N);
3740 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3741 Ops: ArrayRef(Ops).slice(N: 0, M: Factor)),
3742 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
3743 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor))};
3744
3745 for (unsigned i = 0; i != Factor; ++i) {
3746 unsigned IdxLo = 2 * i;
3747 unsigned IdxHi = 2 * i + 1;
3748 SetSplitVector(Op: SDValue(N, i), Lo: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
3749 Hi: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
3750 }
3751}
3752
3753//===----------------------------------------------------------------------===//
3754// Operand Vector Splitting
3755//===----------------------------------------------------------------------===//
3756
3757/// This method is called when the specified operand of the specified node is
3758/// found to need vector splitting. At this point, all of the result types of
3759/// the node are known to be legal, but other operands of the node may need
3760/// legalization as well as the specified one.
3761bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
3762 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG));
3763 SDValue Res = SDValue();
3764
3765 // See if the target wants to custom split this node.
3766 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
3767 return false;
3768
3769 switch (N->getOpcode()) {
3770 default:
3771#ifndef NDEBUG
3772 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
3773 N->dump(&DAG);
3774 dbgs() << "\n";
3775#endif
3776 report_fatal_error(reason: "Do not know how to split this operator's "
3777 "operand!\n");
3778
3779 case ISD::STRICT_FSETCC:
3780 case ISD::STRICT_FSETCCS:
3781 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
3782 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
3783 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
3784 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
3785 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
3786 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
3787 case ISD::VECTOR_FIND_LAST_ACTIVE:
3788 Res = SplitVecOp_VECTOR_FIND_LAST_ACTIVE(N);
3789 break;
3790 case ISD::TRUNCATE:
3791 Res = SplitVecOp_TruncateHelper(N);
3792 break;
3793 case ISD::STRICT_FP_ROUND:
3794 case ISD::FP_ROUND:
3795 case ISD::CONVERT_FROM_ARBITRARY_FP:
3796 case ISD::CONVERT_TO_ARBITRARY_FP:
3797 Res = SplitVecOp_FP_ROUND(N);
3798 break;
3799 case ISD::FCOPYSIGN: Res = SplitVecOp_FPOpDifferentTypes(N); break;
3800 case ISD::STORE:
3801 Res = SplitVecOp_STORE(N: cast<StoreSDNode>(Val: N), OpNo);
3802 break;
3803 case ISD::ATOMIC_STORE:
3804 Res = SplitVecOp_ATOMIC_STORE(N: cast<AtomicSDNode>(Val: N));
3805 break;
3806 case ISD::VP_STORE:
3807 Res = SplitVecOp_VP_STORE(N: cast<VPStoreSDNode>(Val: N), OpNo);
3808 break;
3809 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
3810 Res = SplitVecOp_VP_STRIDED_STORE(N: cast<VPStridedStoreSDNode>(Val: N), OpNo);
3811 break;
3812 case ISD::MSTORE:
3813 Res = SplitVecOp_MSTORE(N: cast<MaskedStoreSDNode>(Val: N), OpNo);
3814 break;
3815 case ISD::MSCATTER:
3816 case ISD::VP_SCATTER:
3817 Res = SplitVecOp_Scatter(N: cast<MemSDNode>(Val: N), OpNo);
3818 break;
3819 case ISD::MGATHER:
3820 case ISD::VP_GATHER:
3821 Res = SplitVecOp_Gather(MGT: cast<MemSDNode>(Val: N), OpNo);
3822 break;
3823 case ISD::VSELECT:
3824 Res = SplitVecOp_VSELECT(N, OpNo);
3825 break;
3826 case ISD::MASKED_UDIV:
3827 case ISD::MASKED_SDIV:
3828 case ISD::MASKED_UREM:
3829 case ISD::MASKED_SREM:
3830 Res = SplitVecOp_MaskedBinOp(N, OpNo);
3831 break;
3832 case ISD::VECTOR_COMPRESS:
3833 Res = SplitVecOp_VECTOR_COMPRESS(N, OpNo);
3834 break;
3835 case ISD::STRICT_SINT_TO_FP:
3836 case ISD::STRICT_UINT_TO_FP:
3837 case ISD::SINT_TO_FP:
3838 case ISD::UINT_TO_FP:
3839 if (N->getValueType(ResNo: 0).bitsLT(
3840 VT: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType()))
3841 Res = SplitVecOp_TruncateHelper(N);
3842 else
3843 Res = SplitVecOp_UnaryOp(N);
3844 break;
3845 case ISD::FP_TO_SINT_SAT:
3846 case ISD::FP_TO_UINT_SAT:
3847 Res = SplitVecOp_FP_TO_XINT_SAT(N);
3848 break;
3849 case ISD::FP_TO_SINT:
3850 case ISD::FP_TO_UINT:
3851 case ISD::STRICT_FP_TO_SINT:
3852 case ISD::STRICT_FP_TO_UINT:
3853 case ISD::STRICT_FP_EXTEND:
3854 case ISD::FP_EXTEND:
3855 case ISD::SIGN_EXTEND:
3856 case ISD::ZERO_EXTEND:
3857 case ISD::ANY_EXTEND:
3858 case ISD::FTRUNC:
3859 case ISD::LROUND:
3860 case ISD::LLROUND:
3861 case ISD::LRINT:
3862 case ISD::LLRINT:
3863 Res = SplitVecOp_UnaryOp(N);
3864 break;
3865 case ISD::FLDEXP:
3866 Res = SplitVecOp_FPOpDifferentTypes(N);
3867 break;
3868
3869 case ISD::SCMP:
3870 case ISD::UCMP:
3871 Res = SplitVecOp_CMP(N);
3872 break;
3873
3874 case ISD::FAKE_USE:
3875 Res = SplitVecOp_FAKE_USE(N);
3876 break;
3877 case ISD::ANY_EXTEND_VECTOR_INREG:
3878 case ISD::SIGN_EXTEND_VECTOR_INREG:
3879 case ISD::ZERO_EXTEND_VECTOR_INREG:
3880 Res = SplitVecOp_ExtVecInRegOp(N);
3881 break;
3882
3883 case ISD::VECREDUCE_FADD:
3884 case ISD::VECREDUCE_FMUL:
3885 case ISD::VECREDUCE_ADD:
3886 case ISD::VECREDUCE_MUL:
3887 case ISD::VECREDUCE_AND:
3888 case ISD::VECREDUCE_OR:
3889 case ISD::VECREDUCE_XOR:
3890 case ISD::VECREDUCE_SMAX:
3891 case ISD::VECREDUCE_SMIN:
3892 case ISD::VECREDUCE_UMAX:
3893 case ISD::VECREDUCE_UMIN:
3894 case ISD::VECREDUCE_FMAX:
3895 case ISD::VECREDUCE_FMIN:
3896 case ISD::VECREDUCE_FMAXIMUM:
3897 case ISD::VECREDUCE_FMINIMUM:
3898 case ISD::VECREDUCE_FMAXIMUMNUM:
3899 case ISD::VECREDUCE_FMINIMUMNUM:
3900 Res = SplitVecOp_VECREDUCE(N, OpNo);
3901 break;
3902 case ISD::VECREDUCE_SEQ_FADD:
3903 case ISD::VECREDUCE_SEQ_FMUL:
3904 Res = SplitVecOp_VECREDUCE_SEQ(N);
3905 break;
3906 case ISD::VP_REDUCE_FADD:
3907 case ISD::VP_REDUCE_SEQ_FADD:
3908 case ISD::VP_REDUCE_FMUL:
3909 case ISD::VP_REDUCE_SEQ_FMUL:
3910 case ISD::VP_REDUCE_ADD:
3911 case ISD::VP_REDUCE_MUL:
3912 case ISD::VP_REDUCE_AND:
3913 case ISD::VP_REDUCE_OR:
3914 case ISD::VP_REDUCE_XOR:
3915 case ISD::VP_REDUCE_SMAX:
3916 case ISD::VP_REDUCE_SMIN:
3917 case ISD::VP_REDUCE_UMAX:
3918 case ISD::VP_REDUCE_UMIN:
3919 case ISD::VP_REDUCE_FMAX:
3920 case ISD::VP_REDUCE_FMIN:
3921 case ISD::VP_REDUCE_FMAXIMUM:
3922 case ISD::VP_REDUCE_FMINIMUM:
3923 Res = SplitVecOp_VP_REDUCE(N, OpNo);
3924 break;
3925 case ISD::CTTZ_ELTS:
3926 case ISD::CTTZ_ELTS_ZERO_POISON:
3927 Res = SplitVecOp_CttzElts(N);
3928 break;
3929 case ISD::VP_CTTZ_ELTS:
3930 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
3931 Res = SplitVecOp_VP_CttzElements(N);
3932 break;
3933 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
3934 Res = SplitVecOp_VECTOR_HISTOGRAM(N);
3935 break;
3936 case ISD::PARTIAL_REDUCE_UMLA:
3937 case ISD::PARTIAL_REDUCE_SMLA:
3938 case ISD::PARTIAL_REDUCE_SUMLA:
3939 case ISD::PARTIAL_REDUCE_FMLA:
3940 Res = SplitVecOp_PARTIAL_REDUCE_MLA(N);
3941 break;
3942 case ISD::VECTOR_MATCH:
3943 Res = SplitVecOp_VECTOR_MATCH(N, OpNo);
3944 break;
3945 }
3946
3947 // If the result is null, the sub-method took care of registering results etc.
3948 if (!Res.getNode()) return false;
3949
3950 // If the result is N, the sub-method updated N in place. Tell the legalizer
3951 // core about this.
3952 if (Res.getNode() == N)
3953 return true;
3954
3955 if (N->isStrictFPOpcode())
3956 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
3957 "Invalid operand expansion");
3958 else
3959 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3960 "Invalid operand expansion");
3961
3962 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3963 return false;
3964}
3965
3966SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
3967 SDLoc DL(N);
3968
3969 SDValue LoMask, HiMask;
3970 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LoMask, Hi&: HiMask);
3971
3972 EVT VT = N->getValueType(ResNo: 0);
3973 EVT SplitVT = LoMask.getValueType();
3974 ElementCount SplitEC = SplitVT.getVectorElementCount();
3975
3976 // Find the last active in both the low and the high masks.
3977 SDValue LoFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: LoMask);
3978 SDValue HiFind = DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT, Operand: HiMask);
3979
3980 // Check if any lane is active in the high mask.
3981 // FIXME: This would not be necessary if VECTOR_FIND_LAST_ACTIVE returned a
3982 // sentinel value for "none active".
3983 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: HiMask);
3984 SDValue Cond = DAG.getBoolExtOrTrunc(Op: AnyHiActive, SL: DL,
3985 VT: getSetCCResultType(VT: MVT::i1), OpVT: MVT::i1);
3986
3987 // Return: AnyHiActive ? (HiFind + SplitEC) : LoFind;
3988 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
3989 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: HiFind,
3990 N2: DAG.getElementCount(DL, VT, EC: SplitEC)),
3991 N3: LoFind);
3992}
3993
3994SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
3995 // The only possibility for an illegal operand is the mask, since result type
3996 // legalization would have handled this node already otherwise.
3997 assert(OpNo == 0 && "Illegal operand must be mask");
3998
3999 SDValue Mask = N->getOperand(Num: 0);
4000 SDValue Src0 = N->getOperand(Num: 1);
4001 SDValue Src1 = N->getOperand(Num: 2);
4002 EVT Src0VT = Src0.getValueType();
4003 SDLoc DL(N);
4004 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
4005
4006 SDValue Lo, Hi;
4007 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4008 assert(Lo.getValueType() == Hi.getValueType() &&
4009 "Lo and Hi have differing types");
4010
4011 EVT LoOpVT, HiOpVT;
4012 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: Src0VT);
4013 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
4014
4015 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
4016 std::tie(args&: LoOp0, args&: HiOp0) = DAG.SplitVector(N: Src0, DL);
4017 std::tie(args&: LoOp1, args&: HiOp1) = DAG.SplitVector(N: Src1, DL);
4018 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4019
4020 SDValue LoSelect =
4021 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: LoOpVT, N1: LoMask, N2: LoOp0, N3: LoOp1);
4022 SDValue HiSelect =
4023 DAG.getNode(Opcode: ISD::VSELECT, DL, VT: HiOpVT, N1: HiMask, N2: HiOp0, N3: HiOp1);
4024
4025 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Src0VT, N1: LoSelect, N2: HiSelect);
4026}
4027
4028SDValue DAGTypeLegalizer::SplitVecOp_MaskedBinOp(SDNode *N, unsigned OpNo) {
4029 assert(OpNo == 2 && "Illegal operand must be mask");
4030
4031 SDLoc DL(N);
4032 auto [LHSLo, LHSHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
4033 auto [RHSLo, RHSHi] = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
4034 SDValue MaskLo, MaskHi;
4035 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: MaskLo, Hi&: MaskHi);
4036
4037 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLo.getValueType(), N1: LHSLo,
4038 N2: RHSLo, N3: MaskLo, Flags: N->getFlags());
4039 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHi.getValueType(), N1: LHSHi,
4040 N2: RHSHi, N3: MaskHi, Flags: N->getFlags());
4041 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4042}
4043
4044SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) {
4045 // The only possibility for an illegal operand is the mask, since result type
4046 // legalization would have handled this node already otherwise.
4047 assert(OpNo == 1 && "Illegal operand must be mask");
4048
4049 // To split the mask, we need to split the result type too, so we can just
4050 // reuse that logic here.
4051 SDValue Lo, Hi;
4052 SplitVecRes_VECTOR_COMPRESS(N, Lo, Hi);
4053
4054 EVT VecVT = N->getValueType(ResNo: 0);
4055 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: VecVT, N1: Lo, N2: Hi);
4056}
4057
4058SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
4059 EVT ResVT = N->getValueType(ResNo: 0);
4060 SDValue Lo, Hi;
4061 SDLoc dl(N);
4062
4063 SDValue VecOp = N->getOperand(Num: OpNo);
4064 EVT VecVT = VecOp.getValueType();
4065 assert(VecVT.isVector() && "Can only split reduce vector operand");
4066 GetSplitVector(Op: VecOp, Lo, Hi);
4067 EVT LoOpVT, HiOpVT;
4068 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4069
4070 // Use the appropriate scalar instruction on the split subvectors before
4071 // reducing the now partially reduced smaller vector.
4072 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: N->getOpcode());
4073 SDValue Partial = DAG.getNode(Opcode: CombineOpc, DL: dl, VT: LoOpVT, N1: Lo, N2: Hi, Flags: N->getFlags());
4074 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, Operand: Partial, Flags: N->getFlags());
4075}
4076
4077SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
4078 EVT ResVT = N->getValueType(ResNo: 0);
4079 SDValue Lo, Hi;
4080 SDLoc dl(N);
4081
4082 SDValue AccOp = N->getOperand(Num: 0);
4083 SDValue VecOp = N->getOperand(Num: 1);
4084 SDNodeFlags Flags = N->getFlags();
4085
4086 EVT VecVT = VecOp.getValueType();
4087 assert(VecVT.isVector() && "Can only split reduce vector operand");
4088 GetSplitVector(Op: VecOp, Lo, Hi);
4089 EVT LoOpVT, HiOpVT;
4090 std::tie(args&: LoOpVT, args&: HiOpVT) = DAG.GetSplitDestVTs(VT: VecVT);
4091
4092 // Reduce low half.
4093 SDValue Partial = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: AccOp, N2: Lo, Flags);
4094
4095 // Reduce high half, using low half result as initial value.
4096 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: Partial, N2: Hi, Flags);
4097}
4098
4099SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
4100 assert(N->isVPOpcode() && "Expected VP opcode");
4101 assert(OpNo == 1 && "Can only split reduce vector operand");
4102
4103 unsigned Opc = N->getOpcode();
4104 EVT ResVT = N->getValueType(ResNo: 0);
4105 SDValue Lo, Hi;
4106 SDLoc dl(N);
4107
4108 SDValue VecOp = N->getOperand(Num: OpNo);
4109 EVT VecVT = VecOp.getValueType();
4110 assert(VecVT.isVector() && "Can only split reduce vector operand");
4111 GetSplitVector(Op: VecOp, Lo, Hi);
4112
4113 SDValue MaskLo, MaskHi;
4114 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: N->getOperand(Num: 2));
4115
4116 SDValue EVLLo, EVLHi;
4117 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: N->getOperand(Num: 3), VecVT, DL: dl);
4118
4119 const SDNodeFlags Flags = N->getFlags();
4120
4121 SDValue ResLo =
4122 DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {N->getOperand(Num: 0), Lo, MaskLo, EVLLo}, Flags);
4123 return DAG.getNode(Opcode: Opc, DL: dl, VT: ResVT, Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags);
4124}
4125
4126SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
4127 // The result has a legal vector type, but the input needs splitting.
4128 EVT ResVT = N->getValueType(ResNo: 0);
4129 SDValue Lo, Hi;
4130 SDLoc dl(N);
4131 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4132 EVT InVT = Lo.getValueType();
4133
4134 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4135 EC: InVT.getVectorElementCount());
4136
4137 if (N->isStrictFPOpcode()) {
4138 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4139 Ops: {N->getOperand(Num: 0), Lo});
4140 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {OutVT, MVT::Other},
4141 Ops: {N->getOperand(Num: 0), Hi});
4142
4143 // Build a factor node to remember that this operation is independent
4144 // of the other one.
4145 SDValue Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
4146 N2: Hi.getValue(R: 1));
4147
4148 // Legalize the chain result - switch anything that used the old chain to
4149 // use the new one.
4150 ReplaceValueWith(From: SDValue(N, 1), To: Ch);
4151 } else {
4152 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Lo);
4153 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: OutVT, Operand: Hi);
4154 }
4155
4156 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4157}
4158
4159// Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part.
4160SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) {
4161 SDValue Lo, Hi;
4162 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4163 SDValue Chain =
4164 DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0), N2: Lo);
4165 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: Chain, N2: Hi);
4166}
4167
4168SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
4169 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
4170 // end up being split all the way down to individual components. Convert the
4171 // split pieces into integers and reassemble.
4172 EVT ResVT = N->getValueType(ResNo: 0);
4173 SDValue Lo, Hi;
4174 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4175 SDLoc dl(N);
4176
4177 if (ResVT.isScalableVector()) {
4178 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: ResVT);
4179 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoVT, Operand: Lo);
4180 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: HiVT, Operand: Hi);
4181 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
4182 }
4183
4184 Lo = BitConvertToInteger(Op: Lo);
4185 Hi = BitConvertToInteger(Op: Hi);
4186
4187 if (DAG.getDataLayout().isBigEndian())
4188 std::swap(a&: Lo, b&: Hi);
4189
4190 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResVT, Operand: JoinIntegers(Lo, Hi));
4191}
4192
4193SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
4194 unsigned OpNo) {
4195 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
4196 // We know that the result type is legal.
4197 EVT ResVT = N->getValueType(ResNo: 0);
4198
4199 SDValue Vec = N->getOperand(Num: 0);
4200 SDValue SubVec = N->getOperand(Num: 1);
4201 SDValue Idx = N->getOperand(Num: 2);
4202 SDLoc dl(N);
4203
4204 SDValue Lo, Hi;
4205 GetSplitVector(Op: SubVec, Lo, Hi);
4206
4207 uint64_t IdxVal = Idx->getAsZExtVal();
4208 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4209
4210 SDValue FirstInsertion =
4211 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: Vec, N2: Lo, N3: Idx);
4212 SDValue SecondInsertion =
4213 DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: ResVT, N1: FirstInsertion, N2: Hi,
4214 N3: DAG.getVectorIdxConstant(Val: IdxVal + LoElts, DL: dl));
4215
4216 return SecondInsertion;
4217}
4218
4219SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4220 // We know that the extracted result type is legal.
4221 EVT SubVT = N->getValueType(ResNo: 0);
4222 SDValue Idx = N->getOperand(Num: 1);
4223 SDLoc dl(N);
4224 SDValue Lo, Hi;
4225
4226 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
4227
4228 ElementCount LoElts = Lo.getValueType().getVectorElementCount();
4229 // Note: For scalable vectors, the index is scaled by vscale.
4230 ElementCount IdxVal =
4231 ElementCount::get(MinVal: Idx->getAsZExtVal(), Scalable: SubVT.isScalableVector());
4232 uint64_t IdxValMin = IdxVal.getKnownMinValue();
4233
4234 EVT SrcVT = N->getOperand(Num: 0).getValueType();
4235 ElementCount NumResultElts = SubVT.getVectorElementCount();
4236
4237 // If the extracted elements are all in the low half, do a simple extract.
4238 if (ElementCount::isKnownLE(LHS: IdxVal + NumResultElts, RHS: LoElts))
4239 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT, N1: Lo, N2: Idx);
4240
4241 unsigned LoEltsMin = LoElts.getKnownMinValue();
4242 if (IdxValMin < LoEltsMin && SubVT.isFixedLengthVector() &&
4243 SrcVT.isFixedLengthVector()) {
4244 // Extracted subvector crosses vector split, so we need to blend the two
4245 // halves.
4246 // TODO: May be able to emit partial extract_subvector.
4247 SmallVector<SDValue, 8> Elts;
4248 Elts.reserve(N: NumResultElts.getFixedValue());
4249
4250 // This is not valid for scalable vectors. If SubVT is scalable, this is the
4251 // same as unrolling a scalable dimension (invalid). If ScrVT is scalable,
4252 // `Lo[LoEltsMin]` may not be the last element of `Lo`.
4253 DAG.ExtractVectorElements(Op: Lo, Args&: Elts, /*Start=*/IdxValMin,
4254 /*Count=*/LoEltsMin - IdxValMin);
4255 DAG.ExtractVectorElements(Op: Hi, Args&: Elts, /*Start=*/0,
4256 /*Count=*/SubVT.getVectorNumElements() -
4257 Elts.size());
4258 return DAG.getBuildVector(VT: SubVT, DL: dl, Ops: Elts);
4259 }
4260
4261 if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
4262 ElementCount ExtractIdx = IdxVal - LoElts;
4263 if (ExtractIdx.isKnownMultipleOf(RHS: NumResultElts))
4264 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Hi,
4265 Idx: ExtractIdx.getKnownMinValue());
4266
4267 EVT HiVT = Hi.getValueType();
4268 assert(HiVT.isFixedLengthVector() &&
4269 "Only fixed-vector extracts are supported in this case");
4270
4271 // We cannot create an extract_subvector that isn't a multiple of the
4272 // result size, which may go out of bounds for the last elements. Shuffle
4273 // the desired elements down to 0 and do a simple 0 extract.
4274 SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
4275 for (int I = 0; I != int(NumResultElts.getFixedValue()); ++I)
4276 Mask[I] = int(ExtractIdx.getFixedValue()) + I;
4277
4278 SDValue Shuffle =
4279 DAG.getVectorShuffle(VT: HiVT, dl, N1: Hi, N2: DAG.getPOISON(VT: HiVT), Mask);
4280 return DAG.getExtractSubvector(DL: dl, VT: SubVT, Vec: Shuffle, Idx: 0);
4281 }
4282
4283 // After this point the DAG node only permits extracting fixed-width
4284 // subvectors from scalable vectors.
4285 assert(SubVT.isFixedLengthVector() &&
4286 "Extracting scalable subvector from fixed-width unsupported");
4287
4288 // If the element type is i1 and we're not promoting the result, then we may
4289 // end up loading the wrong data since the bits are packed tightly into
4290 // bytes. For example, if we extract a v4i1 (legal) from a nxv4i1 (legal)
4291 // type at index 4, then we will load a byte starting at index 0.
4292 if (SubVT.getScalarType() == MVT::i1)
4293 report_fatal_error(reason: "Don't know how to extract fixed-width predicate "
4294 "subvector from a scalable predicate vector");
4295
4296 // Spill the vector to the stack. We should use the alignment for
4297 // the smallest part.
4298 SDValue Vec = N->getOperand(Num: 0);
4299 EVT VecVT = Vec.getValueType();
4300 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4301 SDValue StackPtr =
4302 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4303 auto &MF = DAG.getMachineFunction();
4304 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4305 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4306
4307 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4308 Alignment: SmallestAlign);
4309
4310 // Extract the subvector by loading the correct part.
4311 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: SubVT, Index: Idx);
4312
4313 return DAG.getLoad(
4314 VT: SubVT, dl, Chain: Store, Ptr: StackPtr,
4315 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
4316}
4317
4318SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4319 SDValue Vec = N->getOperand(Num: 0);
4320 SDValue Idx = N->getOperand(Num: 1);
4321 EVT VecVT = Vec.getValueType();
4322
4323 if (const ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Val&: Idx)) {
4324 uint64_t IdxVal = Index->getZExtValue();
4325
4326 SDValue Lo, Hi;
4327 GetSplitVector(Op: Vec, Lo, Hi);
4328
4329 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
4330
4331 if (IdxVal < LoElts)
4332 return SDValue(DAG.UpdateNodeOperands(N, Op1: Lo, Op2: Idx), 0);
4333 else if (!Vec.getValueType().isScalableVector())
4334 return SDValue(DAG.UpdateNodeOperands(N, Op1: Hi,
4335 Op2: DAG.getConstant(Val: IdxVal - LoElts, DL: SDLoc(N),
4336 VT: Idx.getValueType())), 0);
4337 }
4338
4339 // See if the target wants to custom expand this node.
4340 if (CustomLowerNode(N, VT: N->getValueType(ResNo: 0), LegalizeResult: true))
4341 return SDValue();
4342
4343 // Make the vector elements byte-addressable if they aren't already.
4344 SDLoc dl(N);
4345 EVT EltVT = VecVT.getVectorElementType();
4346 if (!EltVT.isByteSized()) {
4347 EltVT = EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
4348 VecVT = VecVT.changeElementType(Context&: *DAG.getContext(), EltVT);
4349 Vec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VecVT, Operand: Vec);
4350 SDValue NewExtract =
4351 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Vec, N2: Idx);
4352 return DAG.getAnyExtOrTrunc(Op: NewExtract, DL: dl, VT: N->getValueType(ResNo: 0));
4353 }
4354
4355 // Store the vector to the stack.
4356 // In cases where the vector is illegal it will be broken down into parts
4357 // and stored in parts - we should use the alignment for the smallest part.
4358 Align SmallestAlign = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
4359 SDValue StackPtr =
4360 DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment: SmallestAlign);
4361 auto &MF = DAG.getMachineFunction();
4362 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
4363 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
4364 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
4365 Alignment: SmallestAlign);
4366
4367 // Load back the required element.
4368 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
4369
4370 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
4371 // type, leaving the high bits undefined. But it can't truncate.
4372 assert(N->getValueType(0).bitsGE(EltVT) && "Illegal EXTRACT_VECTOR_ELT.");
4373
4374 return DAG.getExtLoad(
4375 ExtType: ISD::EXTLOAD, dl, VT: N->getValueType(ResNo: 0), Chain: Store, Ptr: StackPtr,
4376 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()), MemVT: EltVT,
4377 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
4378}
4379
4380SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
4381 SDValue Lo, Hi;
4382
4383 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
4384 // splitting the result has the same effect as splitting the input operand.
4385 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
4386
4387 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4388}
4389
4390SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) {
4391 (void)OpNo;
4392 SDValue Lo, Hi;
4393 SplitVecRes_Gather(N, Lo, Hi);
4394
4395 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: N, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
4396 ReplaceValueWith(From: SDValue(N, 0), To: Res);
4397 return SDValue();
4398}
4399
4400SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
4401 assert(N->isUnindexed() && "Indexed vp_store of vector?");
4402 SDValue Ch = N->getChain();
4403 SDValue Ptr = N->getBasePtr();
4404 SDValue Offset = N->getOffset();
4405 assert(Offset.isUndef() && "Unexpected VP store offset");
4406 SDValue Mask = N->getMask();
4407 SDValue EVL = N->getVectorLength();
4408 SDValue Data = N->getValue();
4409 Align Alignment = N->getBaseAlign();
4410 SDLoc DL(N);
4411
4412 SDValue DataLo, DataHi;
4413 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4414 // Split Data operand
4415 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4416 else
4417 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4418
4419 // Split Mask operand
4420 SDValue MaskLo, MaskHi;
4421 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4422 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4423 } else {
4424 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4425 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4426 else
4427 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4428 }
4429
4430 EVT MemoryVT = N->getMemoryVT();
4431 EVT LoMemVT, HiMemVT;
4432 bool HiIsEmpty = false;
4433 std::tie(args&: LoMemVT, args&: HiMemVT) =
4434 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4435
4436 // Split EVL
4437 SDValue EVLLo, EVLHi;
4438 std::tie(args&: EVLLo, args&: EVLHi) = DAG.SplitEVL(N: EVL, VecVT: Data.getValueType(), DL);
4439
4440 SDValue Lo, Hi;
4441 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4442 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4443 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4444 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4445
4446 Lo = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataLo, Ptr, Offset, Mask: MaskLo, EVL: EVLLo, MemVT: LoMemVT, MMO,
4447 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4448 IsCompressing: N->isCompressingStore());
4449
4450 // If the hi vp_store has zero storage size, only the lo vp_store is needed.
4451 if (HiIsEmpty)
4452 return Lo;
4453
4454 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4455 IsCompressedMemory: N->isCompressingStore());
4456
4457 MachinePointerInfo MPI;
4458 if (LoMemVT.isScalableVector()) {
4459 Alignment = commonAlignment(A: Alignment,
4460 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4461 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4462 } else
4463 MPI = N->getPointerInfo().getWithOffset(
4464 O: LoMemVT.getStoreSize().getFixedValue());
4465
4466 MMO = DAG.getMachineFunction().getMachineMemOperand(
4467 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4468 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4469
4470 Hi = DAG.getStoreVP(Chain: Ch, dl: DL, Val: DataHi, Ptr, Offset, Mask: MaskHi, EVL: EVLHi, MemVT: HiMemVT, MMO,
4471 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4472 IsCompressing: N->isCompressingStore());
4473
4474 // Build a factor node to remember that this store is independent of the
4475 // other one.
4476 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4477}
4478
4479SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
4480 unsigned OpNo) {
4481 assert(N->isUnindexed() && "Indexed vp_strided_store of a vector?");
4482 assert(N->getOffset().isUndef() && "Unexpected VP strided store offset");
4483
4484 SDLoc DL(N);
4485
4486 SDValue Data = N->getValue();
4487 SDValue LoData, HiData;
4488 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4489 GetSplitVector(Op: Data, Lo&: LoData, Hi&: HiData);
4490 else
4491 std::tie(args&: LoData, args&: HiData) = DAG.SplitVector(N: Data, DL);
4492
4493 EVT LoMemVT, HiMemVT;
4494 bool HiIsEmpty = false;
4495 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetDependentSplitDestVTs(
4496 VT: N->getMemoryVT(), EnvVT: LoData.getValueType(), HiIsEmpty: &HiIsEmpty);
4497
4498 SDValue Mask = N->getMask();
4499 SDValue LoMask, HiMask;
4500 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC)
4501 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: LoMask, Hi&: HiMask);
4502 else if (getTypeAction(VT: Mask.getValueType()) ==
4503 TargetLowering::TypeSplitVector)
4504 GetSplitVector(Op: Mask, Lo&: LoMask, Hi&: HiMask);
4505 else
4506 std::tie(args&: LoMask, args&: HiMask) = DAG.SplitVector(N: Mask, DL);
4507
4508 SDValue LoEVL, HiEVL;
4509 std::tie(args&: LoEVL, args&: HiEVL) =
4510 DAG.SplitEVL(N: N->getVectorLength(), VecVT: Data.getValueType(), DL);
4511
4512 // Generate the low vp_strided_store
4513 SDValue Lo = DAG.getStridedStoreVP(
4514 Chain: N->getChain(), DL, Val: LoData, Ptr: N->getBasePtr(), Offset: N->getOffset(),
4515 Stride: N->getStride(), Mask: LoMask, EVL: LoEVL, MemVT: LoMemVT, MMO: N->getMemOperand(),
4516 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(), IsCompressing: N->isCompressingStore());
4517
4518 // If the high vp_strided_store has zero storage size, only the low
4519 // vp_strided_store is needed.
4520 if (HiIsEmpty)
4521 return Lo;
4522
4523 // Generate the high vp_strided_store.
4524 // To calculate the high base address, we need to sum to the low base
4525 // address stride number of bytes for each element already stored by low,
4526 // that is: Ptr = Ptr + (LoEVL * Stride)
4527 EVT PtrVT = N->getBasePtr().getValueType();
4528 SDValue Increment =
4529 DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: LoEVL,
4530 N2: DAG.getSExtOrTrunc(Op: N->getStride(), DL, VT: PtrVT));
4531 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: N->getBasePtr(), N2: Increment);
4532
4533 Align Alignment = N->getBaseAlign();
4534 if (LoMemVT.isScalableVector())
4535 Alignment = commonAlignment(A: Alignment,
4536 Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4537
4538 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4539 PtrInfo: MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
4540 F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4541 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4542
4543 SDValue Hi = DAG.getStridedStoreVP(
4544 Chain: N->getChain(), DL, Val: HiData, Ptr, Offset: N->getOffset(), Stride: N->getStride(), Mask: HiMask,
4545 EVL: HiEVL, MemVT: HiMemVT, MMO, AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4546 IsCompressing: N->isCompressingStore());
4547
4548 // Build a factor node to remember that this store is independent of the
4549 // other one.
4550 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4551}
4552
4553SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
4554 unsigned OpNo) {
4555 assert(N->isUnindexed() && "Indexed masked store of vector?");
4556 SDValue Ch = N->getChain();
4557 SDValue Ptr = N->getBasePtr();
4558 SDValue Offset = N->getOffset();
4559 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
4560 SDValue Mask = N->getMask();
4561 SDValue Data = N->getValue();
4562 Align Alignment = N->getBaseAlign();
4563 SDLoc DL(N);
4564
4565 SDValue DataLo, DataHi;
4566 if (getTypeAction(VT: Data.getValueType()) == TargetLowering::TypeSplitVector)
4567 // Split Data operand
4568 GetSplitVector(Op: Data, Lo&: DataLo, Hi&: DataHi);
4569 else
4570 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Data, DL);
4571
4572 // Split Mask operand
4573 SDValue MaskLo, MaskHi;
4574 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
4575 SplitVecRes_SETCC(N: Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4576 } else {
4577 if (getTypeAction(VT: Mask.getValueType()) == TargetLowering::TypeSplitVector)
4578 GetSplitVector(Op: Mask, Lo&: MaskLo, Hi&: MaskHi);
4579 else
4580 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: Mask, DL);
4581 }
4582
4583 EVT MemoryVT = N->getMemoryVT();
4584 EVT LoMemVT, HiMemVT;
4585 bool HiIsEmpty = false;
4586 std::tie(args&: LoMemVT, args&: HiMemVT) =
4587 DAG.GetDependentSplitDestVTs(VT: MemoryVT, EnvVT: DataLo.getValueType(), HiIsEmpty: &HiIsEmpty);
4588
4589 SDValue Lo, Hi, Res;
4590 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4591 PtrInfo: N->getPointerInfo(), F: MachineMemOperand::MOStore,
4592 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment,
4593 Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4594
4595 Lo = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataLo, Base: Ptr, Offset, Mask: MaskLo, MemVT: LoMemVT, MMO,
4596 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4597 IsCompressing: N->isCompressingStore());
4598
4599 if (HiIsEmpty) {
4600 // The hi masked store has zero storage size.
4601 // Only the lo masked store is needed.
4602 Res = Lo;
4603 } else {
4604
4605 Ptr = TLI.IncrementMemoryAddress(Addr: Ptr, Mask: MaskLo, DL, DataVT: LoMemVT, DAG,
4606 IsCompressedMemory: N->isCompressingStore());
4607
4608 MachinePointerInfo MPI;
4609 if (LoMemVT.isScalableVector()) {
4610 Alignment = commonAlignment(
4611 A: Alignment, Offset: LoMemVT.getSizeInBits().getKnownMinValue() / 8);
4612 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
4613 } else
4614 MPI = N->getPointerInfo().getWithOffset(
4615 O: LoMemVT.getStoreSize().getFixedValue());
4616
4617 MMO = DAG.getMachineFunction().getMachineMemOperand(
4618 PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer(),
4619 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4620
4621 Hi = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: DataHi, Base: Ptr, Offset, Mask: MaskHi, MemVT: HiMemVT, MMO,
4622 AM: N->getAddressingMode(), IsTruncating: N->isTruncatingStore(),
4623 IsCompressing: N->isCompressingStore());
4624
4625 // Build a factor node to remember that this store is independent of the
4626 // other one.
4627 Res = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4628 }
4629
4630 return Res;
4631}
4632
4633SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
4634 SDValue Ch = N->getChain();
4635 SDValue Ptr = N->getBasePtr();
4636 EVT MemoryVT = N->getMemoryVT();
4637 Align Alignment = N->getBaseAlign();
4638 SDLoc DL(N);
4639 struct Operands {
4640 SDValue Mask;
4641 SDValue Index;
4642 SDValue Scale;
4643 SDValue Data;
4644 } Ops = [&]() -> Operands {
4645 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4646 return {.Mask: MSC->getMask(), .Index: MSC->getIndex(), .Scale: MSC->getScale(),
4647 .Data: MSC->getValue()};
4648 }
4649 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4650 return {.Mask: VPSC->getMask(), .Index: VPSC->getIndex(), .Scale: VPSC->getScale(),
4651 .Data: VPSC->getValue()};
4652 }();
4653 // Split all operands
4654
4655 EVT LoMemVT, HiMemVT;
4656 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4657
4658 SDValue DataLo, DataHi;
4659 if (getTypeAction(VT: Ops.Data.getValueType()) == TargetLowering::TypeSplitVector)
4660 // Split Data operand
4661 GetSplitVector(Op: Ops.Data, Lo&: DataLo, Hi&: DataHi);
4662 else
4663 std::tie(args&: DataLo, args&: DataHi) = DAG.SplitVector(N: Ops.Data, DL);
4664
4665 // Split Mask operand
4666 SDValue MaskLo, MaskHi;
4667 if (OpNo == 1 && Ops.Mask.getOpcode() == ISD::SETCC) {
4668 SplitVecRes_SETCC(N: Ops.Mask.getNode(), Lo&: MaskLo, Hi&: MaskHi);
4669 } else {
4670 std::tie(args&: MaskLo, args&: MaskHi) = SplitMask(Mask: Ops.Mask, DL);
4671 }
4672
4673 SDValue IndexHi, IndexLo;
4674 if (getTypeAction(VT: Ops.Index.getValueType()) ==
4675 TargetLowering::TypeSplitVector)
4676 GetSplitVector(Op: Ops.Index, Lo&: IndexLo, Hi&: IndexHi);
4677 else
4678 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: Ops.Index, DL);
4679
4680 SDValue Lo;
4681 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4682 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4683 PtrInfo: N->getPointerInfo(), F: MMOFlags, Size: LocationSize::beforeOrAfterPointer(),
4684 BaseAlignment: Alignment, Metadata: MMOMetadata(N->getAAInfo(), N->getRanges()));
4685
4686 if (auto *MSC = dyn_cast<MaskedScatterSDNode>(Val: N)) {
4687 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
4688 Lo =
4689 DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4690 IndexType: MSC->getIndexType(), IsTruncating: MSC->isTruncatingStore());
4691
4692 // The order of the Scatter operation after split is well defined. The "Hi"
4693 // part comes after the "Lo". So these two operations should be chained one
4694 // after another.
4695 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Ops.Scale};
4696 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: HiMemVT, dl: DL, Ops: OpsHi,
4697 MMO, IndexType: MSC->getIndexType(),
4698 IsTruncating: MSC->isTruncatingStore());
4699 }
4700 auto *VPSC = cast<VPScatterSDNode>(Val: N);
4701 SDValue EVLLo, EVLHi;
4702 std::tie(args&: EVLLo, args&: EVLHi) =
4703 DAG.SplitEVL(N: VPSC->getVectorLength(), VecVT: Ops.Data.getValueType(), DL);
4704
4705 SDValue OpsLo[] = {Ch, DataLo, Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
4706 Lo = DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: LoMemVT, dl: DL, Ops: OpsLo, MMO,
4707 IndexType: VPSC->getIndexType());
4708
4709 // The order of the Scatter operation after split is well defined. The "Hi"
4710 // part comes after the "Lo". So these two operations should be chained one
4711 // after another.
4712 SDValue OpsHi[] = {Lo, DataHi, Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
4713 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: HiMemVT, dl: DL, Ops: OpsHi, MMO,
4714 IndexType: VPSC->getIndexType());
4715}
4716
4717SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
4718 assert(N->isUnindexed() && "Indexed store of vector?");
4719 assert(OpNo == 1 && "Can only split the stored value");
4720 SDLoc DL(N);
4721
4722 bool isTruncating = N->isTruncatingStore();
4723 SDValue Ch = N->getChain();
4724 SDValue Ptr = N->getBasePtr();
4725 EVT MemoryVT = N->getMemoryVT();
4726 Align Alignment = N->getBaseAlign();
4727 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
4728 AAMDNodes AAInfo = N->getAAInfo();
4729 SDValue Lo, Hi;
4730 GetSplitVector(Op: N->getOperand(Num: 1), Lo, Hi);
4731
4732 EVT LoMemVT, HiMemVT;
4733 std::tie(args&: LoMemVT, args&: HiMemVT) = DAG.GetSplitDestVTs(VT: MemoryVT);
4734
4735 // Scalarize if the split halves are not byte-sized.
4736 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
4737 return TLI.scalarizeVectorStore(ST: N, DAG);
4738
4739 if (isTruncating)
4740 Lo = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), SVT: LoMemVT,
4741 Alignment, MMOFlags, Metadata: AAInfo);
4742 else
4743 Lo = DAG.getStore(Chain: Ch, dl: DL, Val: Lo, Ptr, PtrInfo: N->getPointerInfo(), Alignment, MMOFlags,
4744 Metadata: AAInfo);
4745
4746 MachinePointerInfo MPI;
4747 IncrementPointer(N, MemVT: LoMemVT, MPI, Ptr);
4748
4749 if (isTruncating)
4750 Hi = DAG.getTruncStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI,
4751 SVT: HiMemVT, Alignment, MMOFlags, Metadata: AAInfo);
4752 else
4753 Hi = DAG.getStore(Chain: Ch, dl: DL, Val: Hi, Ptr, PtrInfo: MPI, Alignment, MMOFlags, Metadata: AAInfo);
4754
4755 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
4756}
4757
4758SDValue DAGTypeLegalizer::SplitVecOp_ATOMIC_STORE(AtomicSDNode *N) {
4759 SDLoc DL(N);
4760 LLVMContext &Ctx = *DAG.getContext();
4761 SDValue StVal = N->getVal();
4762 EVT VT = StVal.getValueType();
4763 EVT MemIntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: N->getMemoryVT().getSizeInBits());
4764
4765 // The store needs a single value spanning the full memory width. If the
4766 // value can be held in a legal vector register, keep it there and extract
4767 // the low integer element of the memory width. This lets the store be issued
4768 // directly from a vector register (e.g. a single MOVQ/MOVD) instead of
4769 // bitcasting the split vector straight to a scalar integer, which would
4770 // reassemble the value element by element in GPRs.
4771 //
4772 // Reinterpret the value as a same-shaped integer vector first: an FP element
4773 // type may not have a legal vector form (e.g. bfloat on SSE2) while the
4774 // integer-of-element-size form does. Ask the target which legal vector type
4775 // it widens to.
4776 EVT IntVecVT = VT.changeVectorElementTypeToInteger();
4777 EVT IntEltVT = IntVecVT.getVectorElementType();
4778 EVT WideVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT: IntVecVT);
4779 if (DAG.getDataLayout().isLittleEndian() && TLI.isTypeLegal(VT: MemIntVT) &&
4780 WideVT.isVector() && WideVT.getVectorElementType() == IntEltVT &&
4781 IntEltVT.getSizeInBits() <= MemIntVT.getSizeInBits() &&
4782 WideVT.getSizeInBits() % MemIntVT.getSizeInBits() == 0) {
4783 SDValue Wide = ModifyToType(InOp: DAG.getBitcast(VT: IntVecVT, V: StVal), NVT: WideVT);
4784 unsigned NumMemElts = WideVT.getSizeInBits() / MemIntVT.getSizeInBits();
4785 EVT MemVecVT = EVT::getVectorVT(Context&: Ctx, VT: MemIntVT, NumElements: NumMemElts);
4786 SDValue Elt = DAG.getExtractVectorElt(DL, VT: MemIntVT,
4787 Vec: DAG.getBitcast(VT: MemVecVT, V: Wide), Idx: 0);
4788 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: Elt,
4789 Val: N->getBasePtr(), MMO: N->getMemOperand());
4790 }
4791
4792 // Otherwise issue a single atomic store of an integer that spans the full
4793 // memory width. Bitcasting the (illegal) vector value to that integer lets
4794 // the type legalizer further legalize the BITCAST input as needed, while the
4795 // ATOMIC_STORE itself uses only the legal integer type.
4796 EVT IntVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: VT.getSizeInBits());
4797 SDValue AsInt = DAG.getBitcast(VT: IntVT, V: StVal);
4798 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MemIntVT, Chain: N->getChain(), Ptr: AsInt,
4799 Val: N->getBasePtr(), MMO: N->getMemOperand());
4800}
4801
4802SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
4803 SDLoc DL(N);
4804
4805 // The input operands all must have the same type, and we know the result
4806 // type is valid. Convert this to a buildvector which extracts all the
4807 // input elements.
4808 // TODO: If the input elements are power-two vectors, we could convert this to
4809 // a new CONCAT_VECTORS node with elements that are half-wide.
4810 SmallVector<SDValue, 32> Elts;
4811 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
4812 for (const SDValue &Op : N->op_values()) {
4813 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
4814 i != e; ++i) {
4815 Elts.push_back(Elt: DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op, Idx: i));
4816 }
4817 }
4818
4819 return DAG.getBuildVector(VT: N->getValueType(ResNo: 0), DL, Ops: Elts);
4820}
4821
4822SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
4823 // The result type is legal, but the input type is illegal. If splitting
4824 // ends up with the result type of each half still being legal, just
4825 // do that. If, however, that would result in an illegal result type,
4826 // we can try to get more clever with power-two vectors. Specifically,
4827 // split the input type, but also widen the result element size, then
4828 // concatenate the halves and truncate again. For example, consider a target
4829 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
4830 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
4831 // %inlo = v4i32 extract_subvector %in, 0
4832 // %inhi = v4i32 extract_subvector %in, 4
4833 // %lo16 = v4i16 trunc v4i32 %inlo
4834 // %hi16 = v4i16 trunc v4i32 %inhi
4835 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
4836 // %res = v8i8 trunc v8i16 %in16
4837 //
4838 // Without this transform, the original truncate would end up being
4839 // scalarized, which is pretty much always a last resort.
4840 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4841 SDValue InVec = N->getOperand(Num: OpNo);
4842 EVT InVT = InVec->getValueType(ResNo: 0);
4843 EVT OutVT = N->getValueType(ResNo: 0);
4844 ElementCount NumElements = OutVT.getVectorElementCount();
4845 bool IsFloat = OutVT.isFloatingPoint();
4846
4847 unsigned InElementSize = InVT.getScalarSizeInBits();
4848 unsigned OutElementSize = OutVT.getScalarSizeInBits();
4849
4850 // Determine the split output VT. If its legal we can just split dirctly.
4851 EVT LoOutVT, HiOutVT;
4852 std::tie(args&: LoOutVT, args&: HiOutVT) = DAG.GetSplitDestVTs(VT: OutVT);
4853 assert(LoOutVT == HiOutVT && "Unequal split?");
4854
4855 // If the input elements are only 1/2 the width of the result elements,
4856 // just use the normal splitting. Our trick only work if there's room
4857 // to split more than once.
4858 if (isTypeLegal(VT: LoOutVT) || InElementSize <= OutElementSize * 2 ||
4859 (IsFloat && !isPowerOf2_32(Value: InElementSize)))
4860 return SplitVecOp_UnaryOp(N);
4861 SDLoc DL(N);
4862
4863 // Don't touch if this will be scalarized.
4864 EVT FinalVT = InVT;
4865 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
4866 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
4867
4868 if (getTypeAction(VT: FinalVT) == TargetLowering::TypeScalarizeVector)
4869 return SplitVecOp_UnaryOp(N);
4870
4871 // Get the split input vector.
4872 SDValue InLoVec, InHiVec;
4873 GetSplitVector(Op: InVec, Lo&: InLoVec, Hi&: InHiVec);
4874
4875 // Truncate them to 1/2 the element size.
4876 //
4877 // This assumes the number of elements is a power of two; any vector that
4878 // isn't should be widened, not split.
4879 EVT HalfElementVT = IsFloat ?
4880 EVT::getFloatingPointVT(BitWidth: InElementSize/2) :
4881 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: InElementSize/2);
4882 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT,
4883 EC: NumElements.divideCoefficientBy(RHS: 2));
4884
4885 SDValue HalfLo;
4886 SDValue HalfHi;
4887 SDValue Chain;
4888 if (N->isStrictFPOpcode()) {
4889 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4890 Ops: {N->getOperand(Num: 0), InLoVec});
4891 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {HalfVT, MVT::Other},
4892 Ops: {N->getOperand(Num: 0), InHiVec});
4893 // Legalize the chain result - switch anything that used the old chain to
4894 // use the new one.
4895 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: HalfLo.getValue(R: 1),
4896 N2: HalfHi.getValue(R: 1));
4897 } else {
4898 HalfLo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InLoVec);
4899 HalfHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: HalfVT, Operand: InHiVec);
4900 }
4901
4902 // Concatenate them to get the full intermediate truncation result.
4903 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: HalfElementVT, EC: NumElements);
4904 SDValue InterVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InterVT, N1: HalfLo,
4905 N2: HalfHi);
4906 // Now finish up by truncating all the way down to the original result
4907 // type. This should normally be something that ends up being legal directly,
4908 // but in theory if a target has very wide vectors and an annoyingly
4909 // restricted set of legal types, this split can chain to build things up.
4910
4911 if (N->isStrictFPOpcode()) {
4912 SDValue Res = DAG.getNode(
4913 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {OutVT, MVT::Other},
4914 Ops: {Chain, InterVec,
4915 DAG.getTargetConstant(Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout()))});
4916 // Relink the chain
4917 ReplaceValueWith(From: SDValue(N, 1), To: SDValue(Res.getNode(), 1));
4918 return Res;
4919 }
4920
4921 return IsFloat
4922 ? DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: OutVT, N1: InterVec,
4923 N2: DAG.getTargetConstant(
4924 Val: 0, DL, VT: TLI.getPointerTy(DL: DAG.getDataLayout())))
4925 : DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OutVT, Operand: InterVec);
4926}
4927
4928SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
4929 unsigned Opc = N->getOpcode();
4930 bool isStrict = Opc == ISD::STRICT_FSETCC || Opc == ISD::STRICT_FSETCCS;
4931 assert(N->getValueType(0).isVector() &&
4932 N->getOperand(isStrict ? 1 : 0).getValueType().isVector() &&
4933 "Operand types must be vectors");
4934 // The result has a legal vector type, but the input needs splitting.
4935 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
4936 SDLoc DL(N);
4937 GetSplitVector(Op: N->getOperand(Num: isStrict ? 1 : 0), Lo&: Lo0, Hi&: Hi0);
4938 GetSplitVector(Op: N->getOperand(Num: isStrict ? 2 : 1), Lo&: Lo1, Hi&: Hi1);
4939
4940 EVT VT = N->getValueType(ResNo: 0);
4941 EVT PartResVT = getSetCCResultType(VT: Lo0.getValueType());
4942
4943 if (Opc == ISD::SETCC) {
4944 LoRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Lo0, N2: Lo1, N3: N->getOperand(Num: 2));
4945 HiRes = DAG.getNode(Opcode: ISD::SETCC, DL, VT: PartResVT, N1: Hi0, N2: Hi1, N3: N->getOperand(Num: 2));
4946 } else {
4947 assert(isStrict && "unexpected node");
4948 LoRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4949 N1: N->getOperand(Num: 0), N2: Lo0, N3: Lo1, N4: N->getOperand(Num: 3));
4950 HiRes = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: PartResVT, VT2: N->getValueType(ResNo: 1)),
4951 N1: N->getOperand(Num: 0), N2: Hi0, N3: Hi1, N4: N->getOperand(Num: 3));
4952 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4953 N1: LoRes.getValue(R: 1), N2: HiRes.getValue(R: 1));
4954 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4955 }
4956
4957 EVT ConcatVT = PartResVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4958 SDValue Con = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: LoRes, N2: HiRes);
4959 if (VT == ConcatVT)
4960 return Con;
4961
4962 EVT OpVT = N->getOperand(Num: 0).getValueType();
4963 ISD::NodeType ExtendCode =
4964 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
4965 return DAG.getExtOrTrunc(Op: Con, DL, VT, Opcode: ExtendCode);
4966}
4967
4968
4969SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
4970 // The result has a legal vector type, but the input needs splitting.
4971 EVT ResVT = N->getValueType(ResNo: 0);
4972 SDValue Lo, Hi;
4973 SDLoc DL(N);
4974 GetSplitVector(Op: N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
4975 EVT InVT = Lo.getValueType();
4976
4977 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
4978 EC: InVT.getVectorElementCount());
4979
4980 if (N->isStrictFPOpcode()) {
4981 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4982 Ops: {N->getOperand(Num: 0), Lo, N->getOperand(Num: 2)});
4983 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, ResultTys: {OutVT, MVT::Other},
4984 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
4985 // Legalize the chain result - switch anything that used the old chain to
4986 // use the new one.
4987 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
4988 N1: Lo.getValue(R: 1), N2: Hi.getValue(R: 1));
4989 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
4990 } else if (N->getOpcode() == ISD::CONVERT_TO_ARBITRARY_FP) {
4991 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1),
4992 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
4993 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1),
4994 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
4995 } else {
4996 Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Lo, N2: N->getOperand(Num: 1));
4997 Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: OutVT, N1: Hi, N2: N->getOperand(Num: 1));
4998 }
4999
5000 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, N1: Lo, N2: Hi);
5001}
5002
5003// Split a vector type in an FP binary operation where the second operand has a
5004// different type from the first.
5005//
5006// The result (and the first input) has a legal vector type, but the second
5007// input needs splitting.
5008SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) {
5009 SDLoc DL(N);
5010
5011 EVT LHSLoVT, LHSHiVT;
5012 std::tie(args&: LHSLoVT, args&: LHSHiVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5013
5014 if (!isTypeLegal(VT: LHSLoVT) || !isTypeLegal(VT: LHSHiVT))
5015 return DAG.UnrollVectorOp(N, ResNE: N->getValueType(ResNo: 0).getVectorNumElements());
5016
5017 SDValue LHSLo, LHSHi;
5018 std::tie(args&: LHSLo, args&: LHSHi) =
5019 DAG.SplitVector(N: N->getOperand(Num: 0), DL, LoVT: LHSLoVT, HiVT: LHSHiVT);
5020
5021 SDValue RHSLo, RHSHi;
5022 std::tie(args&: RHSLo, args&: RHSHi) = DAG.SplitVector(N: N->getOperand(Num: 1), DL);
5023
5024 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSLoVT, N1: LHSLo, N2: RHSLo);
5025 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LHSHiVT, N1: LHSHi, N2: RHSHi);
5026
5027 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
5028}
5029
5030SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) {
5031 LLVMContext &Ctxt = *DAG.getContext();
5032 SDLoc dl(N);
5033
5034 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5035 GetSplitVector(Op: N->getOperand(Num: 0), Lo&: LHSLo, Hi&: LHSHi);
5036 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: RHSLo, Hi&: RHSHi);
5037
5038 EVT ResVT = N->getValueType(ResNo: 0);
5039 ElementCount SplitOpEC = LHSLo.getValueType().getVectorElementCount();
5040 EVT NewResVT =
5041 EVT::getVectorVT(Context&: Ctxt, VT: ResVT.getVectorElementType(), EC: SplitOpEC);
5042
5043 SDValue Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSLo, N2: RHSLo);
5044 SDValue Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: LHSHi, N2: RHSHi);
5045
5046 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5047}
5048
5049SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
5050 EVT ResVT = N->getValueType(ResNo: 0);
5051 SDValue Lo, Hi;
5052 SDLoc dl(N);
5053 GetSplitVector(Op: N->getOperand(Num: 0), Lo, Hi);
5054 EVT InVT = Lo.getValueType();
5055
5056 EVT NewResVT =
5057 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5058 EC: InVT.getVectorElementCount());
5059
5060 Lo = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Lo, N2: N->getOperand(Num: 1));
5061 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NewResVT, N1: Hi, N2: N->getOperand(Num: 1));
5062
5063 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: ResVT, N1: Lo, N2: Hi);
5064}
5065
5066SDValue DAGTypeLegalizer::SplitVecOp_CttzElts(SDNode *N) {
5067 SDLoc DL(N);
5068 EVT ResVT = N->getValueType(ResNo: 0);
5069
5070 SDValue Lo, Hi;
5071 SDValue VecOp = N->getOperand(Num: 0);
5072 GetSplitVector(Op: VecOp, Lo, Hi);
5073
5074 // if CTTZ_ELTS(Lo) != VL => CTTZ_ELTS(Lo).
5075 // else => VL + (CTTZ_ELTS(Hi) or CTTZ_ELTS_ZERO_POISON(Hi)).
5076 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: Lo);
5077 SDValue VL =
5078 DAG.getElementCount(DL, VT: ResVT, EC: Lo.getValueType().getVectorElementCount());
5079 SDValue ResLoNotVL =
5080 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VL, Cond: ISD::SETNE);
5081 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: Hi);
5082 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotVL, LHS: ResLo,
5083 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VL, N2: ResHi));
5084}
5085
5086SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) {
5087 SDLoc DL(N);
5088 EVT ResVT = N->getValueType(ResNo: 0);
5089
5090 SDValue Lo, Hi;
5091 SDValue VecOp = N->getOperand(Num: 0);
5092 GetSplitVector(Op: VecOp, Lo, Hi);
5093
5094 auto [MaskLo, MaskHi] = SplitMask(Mask: N->getOperand(Num: 1));
5095 auto [EVLLo, EVLHi] =
5096 DAG.SplitEVL(N: N->getOperand(Num: 2), VecVT: VecOp.getValueType(), DL);
5097 SDValue VLo = DAG.getZExtOrTrunc(Op: EVLLo, DL, VT: ResVT);
5098
5099 // if VP_CTTZ_ELTS(Lo) != EVLLo => VP_CTTZ_ELTS(Lo).
5100 // else => EVLLo + (VP_CTTZ_ELTS(Hi) or VP_CTTZ_ELTS_ZERO_POISON(Hi)).
5101 SDValue ResLo = DAG.getNode(Opcode: ISD::VP_CTTZ_ELTS, DL, VT: ResVT, N1: Lo, N2: MaskLo, N3: EVLLo);
5102 SDValue ResLoNotEVL =
5103 DAG.getSetCC(DL, VT: getSetCCResultType(VT: ResVT), LHS: ResLo, RHS: VLo, Cond: ISD::SETNE);
5104 SDValue ResHi = DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, N1: Hi, N2: MaskHi, N3: EVLHi);
5105 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotEVL, LHS: ResLo,
5106 RHS: DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: VLo, N2: ResHi));
5107}
5108
5109SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) {
5110 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
5111 SDLoc DL(HG);
5112 SDValue Inc = HG->getInc();
5113 SDValue Ptr = HG->getBasePtr();
5114 SDValue Scale = HG->getScale();
5115 SDValue IntID = HG->getIntID();
5116 EVT MemVT = HG->getMemoryVT();
5117 MachineMemOperand *MMO = HG->getMemOperand();
5118 ISD::MemIndexType IndexType = HG->getIndexType();
5119
5120 SDValue IndexLo, IndexHi, MaskLo, MaskHi;
5121 std::tie(args&: IndexLo, args&: IndexHi) = DAG.SplitVector(N: HG->getIndex(), DL);
5122 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVector(N: HG->getMask(), DL);
5123 SDValue OpsLo[] = {HG->getChain(), Inc, MaskLo, Ptr, IndexLo, Scale, IntID};
5124 SDValue Lo = DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL,
5125 Ops: OpsLo, MMO, IndexType);
5126 SDValue OpsHi[] = {Lo, Inc, MaskHi, Ptr, IndexHi, Scale, IntID};
5127 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops: OpsHi,
5128 MMO, IndexType);
5129}
5130
5131SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
5132 SDLoc DL(N);
5133
5134 if (OpNo == 0) {
5135 EVT LoResVT, HiResVT;
5136 std::tie(args&: LoResVT, args&: HiResVT) = DAG.GetSplitDestVTs(VT: N->getValueType(ResNo: 0));
5137 SDValue SourceLo, SourceHi;
5138 std::tie(args&: SourceLo, args&: SourceHi) = DAG.SplitVectorOperand(N, OpNo: 0);
5139 SDValue MaskLo, MaskHi;
5140 std::tie(args&: MaskLo, args&: MaskHi) = DAG.SplitVectorOperand(N, OpNo: 2);
5141
5142 SDValue MatchLo = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: LoResVT, N1: SourceLo,
5143 N2: N->getOperand(Num: 1), N3: MaskLo, Flags: N->getFlags());
5144 SDValue MatchHi = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: HiResVT, N1: SourceHi,
5145 N2: N->getOperand(Num: 1), N3: MaskHi, Flags: N->getFlags());
5146 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo,
5147 N2: MatchHi);
5148 }
5149
5150 // Note: The Mask (OpNo == 2) should be widened with the result.
5151 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
5152
5153 SDValue NeedleLo, NeedleHi;
5154 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: NeedleLo, Hi&: NeedleHi);
5155
5156 SDValue MatchLo =
5157 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5158 N2: NeedleLo, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5159 SDValue MatchHi =
5160 DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
5161 N2: NeedleHi, N3: N->getOperand(Num: 2), Flags: N->getFlags());
5162 return DAG.getNode(Opcode: ISD::OR, DL, VT: N->getValueType(ResNo: 0), N1: MatchLo, N2: MatchHi);
5163}
5164
5165SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) {
5166 SDValue Acc = N->getOperand(Num: 0);
5167 assert(getTypeAction(Acc.getValueType()) != TargetLowering::TypeSplitVector &&
5168 "Accumulator should already be a legal type, and shouldn't need "
5169 "further splitting");
5170
5171 SDLoc DL(N);
5172 SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi;
5173 GetSplitVector(Op: N->getOperand(Num: 1), Lo&: Input1Lo, Hi&: Input1Hi);
5174 GetSplitVector(Op: N->getOperand(Num: 2), Lo&: Input2Lo, Hi&: Input2Hi);
5175 unsigned Opcode = N->getOpcode();
5176 EVT ResultVT = Acc.getValueType();
5177
5178 SDValue Lo = DAG.getNode(Opcode, DL, VT: ResultVT, N1: Acc, N2: Input1Lo, N3: Input2Lo);
5179 return DAG.getNode(Opcode, DL, VT: ResultVT, N1: Lo, N2: Input1Hi, N3: Input2Hi);
5180}
5181
5182//===----------------------------------------------------------------------===//
5183// Result Vector Widening
5184//===----------------------------------------------------------------------===//
5185
5186void DAGTypeLegalizer::ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
5187 unsigned WidenResNo) {
5188 unsigned NumResults = N->getNumValues();
5189 for (unsigned ResNo = 0; ResNo < NumResults; ResNo++) {
5190 if (ResNo == WidenResNo)
5191 continue;
5192 EVT ResVT = N->getValueType(ResNo);
5193 if (getTypeAction(VT: ResVT) == TargetLowering::TypeWidenVector) {
5194 SetWidenedVector(Op: SDValue(N, ResNo), Result: SDValue(WidenNode, ResNo));
5195 } else {
5196 SDLoc DL(N);
5197 SDValue ResVal =
5198 DAG.getExtractSubvector(DL, VT: ResVT, Vec: SDValue(WidenNode, ResNo), Idx: 0);
5199 ReplaceValueWith(From: SDValue(N, ResNo), To: ResVal);
5200 }
5201 }
5202}
5203
5204void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
5205 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG));
5206
5207 // See if the target wants to custom widen this node.
5208 if (CustomWidenLowerNode(N, VT: N->getValueType(ResNo)))
5209 return;
5210
5211 SDValue Res = SDValue();
5212
5213 auto unrollExpandedOp = [&]() {
5214 // We're going to widen this vector op to a legal type by padding with undef
5215 // elements. If the wide vector op is eventually going to be expanded to
5216 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
5217 // libcalls on the undef elements.
5218 EVT VT = N->getValueType(ResNo: 0);
5219 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
5220 if (!TLI.isOperationLegalOrCustomOrPromote(Op: N->getOpcode(), VT: WideVecVT) &&
5221 TLI.isOperationExpandOrLibCall(Op: N->getOpcode(), VT: VT.getScalarType())) {
5222 Res = DAG.UnrollVectorOp(N, ResNE: WideVecVT.getVectorNumElements());
5223 if (N->getNumValues() > 1)
5224 ReplaceOtherWidenResults(N, WidenNode: Res.getNode(), WidenResNo: ResNo);
5225 return true;
5226 }
5227 return false;
5228 };
5229
5230 switch (N->getOpcode()) {
5231 default:
5232#ifndef NDEBUG
5233 dbgs() << "WidenVectorResult #" << ResNo << ": ";
5234 N->dump(&DAG);
5235 dbgs() << "\n";
5236#endif
5237 report_fatal_error(reason: "Do not know how to widen the result of this operator!");
5238
5239 case ISD::LOOP_DEPENDENCE_RAW_MASK:
5240 case ISD::LOOP_DEPENDENCE_WAR_MASK:
5241 Res = WidenVecRes_LOOP_DEPENDENCE_MASK(N);
5242 break;
5243 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
5244 case ISD::ADDRSPACECAST:
5245 Res = WidenVecRes_ADDRSPACECAST(N);
5246 break;
5247 case ISD::AssertZext: Res = WidenVecRes_AssertZext(N); break;
5248 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
5249 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
5250 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
5251 case ISD::INSERT_SUBVECTOR:
5252 Res = WidenVecRes_INSERT_SUBVECTOR(N);
5253 break;
5254 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
5255 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
5256 case ISD::ATOMIC_LOAD:
5257 Res = WidenVecRes_ATOMIC_LOAD(N: cast<AtomicSDNode>(Val: N));
5258 break;
5259 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
5260 case ISD::STEP_VECTOR:
5261 case ISD::SPLAT_VECTOR:
5262 case ISD::SCALAR_TO_VECTOR:
5263 Res = WidenVecRes_ScalarOp(N);
5264 break;
5265 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
5266 case ISD::VSELECT:
5267 case ISD::SELECT:
5268 case ISD::VP_MERGE:
5269 Res = WidenVecRes_Select(N);
5270 break;
5271 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
5272 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
5273 case ISD::POISON:
5274 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
5275 case ISD::VECTOR_SHUFFLE:
5276 Res = WidenVecRes_VECTOR_SHUFFLE(N: cast<ShuffleVectorSDNode>(Val: N));
5277 break;
5278 case ISD::VP_LOAD:
5279 Res = WidenVecRes_VP_LOAD(N: cast<VPLoadSDNode>(Val: N));
5280 break;
5281 case ISD::VP_LOAD_FF:
5282 Res = WidenVecRes_VP_LOAD_FF(N: cast<VPLoadFFSDNode>(Val: N));
5283 break;
5284 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5285 Res = WidenVecRes_VP_STRIDED_LOAD(N: cast<VPStridedLoadSDNode>(Val: N));
5286 break;
5287 case ISD::VECTOR_COMPRESS:
5288 Res = WidenVecRes_VECTOR_COMPRESS(N);
5289 break;
5290 case ISD::MLOAD:
5291 Res = WidenVecRes_MLOAD(N: cast<MaskedLoadSDNode>(Val: N));
5292 break;
5293 case ISD::MGATHER:
5294 Res = WidenVecRes_MGATHER(N: cast<MaskedGatherSDNode>(Val: N));
5295 break;
5296 case ISD::VP_GATHER:
5297 Res = WidenVecRes_VP_GATHER(N: cast<VPGatherSDNode>(Val: N));
5298 break;
5299 case ISD::VECTOR_REVERSE:
5300 Res = WidenVecRes_VECTOR_REVERSE(N);
5301 break;
5302 case ISD::GET_ACTIVE_LANE_MASK:
5303 Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N);
5304 break;
5305 case ISD::VECTOR_INTERLEAVE:
5306 WidenVecRes_VECTOR_INTERLEAVE(N);
5307 break;
5308 case ISD::VECTOR_MATCH:
5309 Res = WidenVecRes_VECTOR_MATCH(N);
5310 break;
5311 case ISD::VECTOR_DEINTERLEAVE:
5312 WidenVecRes_VECTOR_DEINTERLEAVE(N);
5313 break;
5314
5315 case ISD::ADD:
5316 case ISD::AND:
5317 case ISD::MUL:
5318 case ISD::MULHS:
5319 case ISD::MULHU:
5320 case ISD::ABDS:
5321 case ISD::ABDU:
5322 case ISD::OR:
5323 case ISD::SUB:
5324 case ISD::XOR:
5325 case ISD::SHL:
5326 case ISD::SRA:
5327 case ISD::SRL:
5328 case ISD::CLMUL:
5329 case ISD::CLMULR:
5330 case ISD::CLMULH:
5331 case ISD::PEXT:
5332 case ISD::PDEP:
5333 case ISD::FMINNUM:
5334 case ISD::FMINNUM_IEEE:
5335 case ISD::FMAXNUM:
5336 case ISD::FMAXNUM_IEEE:
5337 case ISD::FMINIMUM:
5338 case ISD::FMAXIMUM:
5339 case ISD::FMINIMUMNUM:
5340 case ISD::FMAXIMUMNUM:
5341 case ISD::SMIN:
5342 case ISD::SMAX:
5343 case ISD::UMIN:
5344 case ISD::UMAX:
5345 case ISD::UADDSAT:
5346 case ISD::SADDSAT:
5347 case ISD::USUBSAT:
5348 case ISD::SSUBSAT:
5349 case ISD::SSHLSAT:
5350 case ISD::USHLSAT:
5351 case ISD::ROTL:
5352 case ISD::ROTR:
5353 case ISD::AVGFLOORS:
5354 case ISD::AVGFLOORU:
5355 case ISD::AVGCEILS:
5356 case ISD::AVGCEILU:
5357 // Vector-predicated binary op widening. Note that -- unlike the
5358 // unpredicated versions -- we don't have to worry about trapping on
5359 // operations like UDIV, FADD, etc., as we pass on the original vector
5360 // length parameter. This means the widened elements containing garbage
5361 // aren't active.
5362 case ISD::VP_SDIV:
5363 case ISD::VP_UDIV:
5364 case ISD::VP_SREM:
5365 case ISD::VP_UREM:
5366 Res = WidenVecRes_Binary(N);
5367 break;
5368
5369 case ISD::MASKED_UDIV:
5370 case ISD::MASKED_SDIV:
5371 case ISD::MASKED_UREM:
5372 case ISD::MASKED_SREM:
5373 Res = WidenVecRes_MaskedBinary(N);
5374 break;
5375
5376 case ISD::SCMP:
5377 case ISD::UCMP:
5378 Res = WidenVecRes_CMP(N);
5379 break;
5380
5381 case ISD::FPOW:
5382 case ISD::FATAN2:
5383 case ISD::FREM:
5384 if (unrollExpandedOp())
5385 break;
5386 // If the target has custom/legal support for the scalar FP intrinsic ops
5387 // (they are probably not destined to become libcalls), then widen those
5388 // like any other binary ops.
5389 [[fallthrough]];
5390
5391 case ISD::FADD:
5392 case ISD::FMUL:
5393 case ISD::FSUB:
5394 case ISD::FDIV:
5395 case ISD::SDIV:
5396 case ISD::UDIV:
5397 case ISD::SREM:
5398 case ISD::UREM:
5399 Res = WidenVecRes_BinaryCanTrap(N);
5400 break;
5401
5402 case ISD::SMULFIX:
5403 case ISD::SMULFIXSAT:
5404 case ISD::UMULFIX:
5405 case ISD::UMULFIXSAT:
5406 // These are binary operations, but with an extra operand that shouldn't
5407 // be widened (the scale).
5408 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
5409 break;
5410
5411#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
5412 case ISD::STRICT_##DAGN:
5413#include "llvm/IR/ConstrainedOps.def"
5414 Res = WidenVecRes_StrictFP(N);
5415 break;
5416
5417 case ISD::UADDO:
5418 case ISD::SADDO:
5419 case ISD::USUBO:
5420 case ISD::SSUBO:
5421 case ISD::UMULO:
5422 case ISD::SMULO:
5423 Res = WidenVecRes_OverflowOp(N, ResNo);
5424 break;
5425
5426 case ISD::FCOPYSIGN:
5427 Res = WidenVecRes_FCOPYSIGN(N);
5428 break;
5429
5430 case ISD::IS_FPCLASS:
5431 case ISD::FPTRUNC_ROUND:
5432 Res = WidenVecRes_UnarySameEltsWithScalarArg(N);
5433 break;
5434
5435 case ISD::FLDEXP:
5436 case ISD::FPOWI:
5437 if (!unrollExpandedOp())
5438 Res = WidenVecRes_ExpOp(N);
5439 break;
5440
5441 case ISD::ANY_EXTEND_VECTOR_INREG:
5442 case ISD::SIGN_EXTEND_VECTOR_INREG:
5443 case ISD::ZERO_EXTEND_VECTOR_INREG:
5444 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
5445 break;
5446
5447 case ISD::ANY_EXTEND:
5448 case ISD::FP_EXTEND:
5449 case ISD::FP_ROUND:
5450 case ISD::FP_TO_SINT:
5451 case ISD::FP_TO_UINT:
5452 case ISD::SIGN_EXTEND:
5453 case ISD::SINT_TO_FP:
5454 case ISD::TRUNCATE:
5455 case ISD::UINT_TO_FP:
5456 case ISD::ZERO_EXTEND:
5457 case ISD::CONVERT_FROM_ARBITRARY_FP:
5458 case ISD::CONVERT_TO_ARBITRARY_FP:
5459 Res = WidenVecRes_Convert(N);
5460 break;
5461
5462 case ISD::FP_TO_SINT_SAT:
5463 case ISD::FP_TO_UINT_SAT:
5464 Res = WidenVecRes_FP_TO_XINT_SAT(N);
5465 break;
5466
5467 case ISD::LRINT:
5468 case ISD::LLRINT:
5469 case ISD::LROUND:
5470 case ISD::LLROUND:
5471 Res = WidenVecRes_XROUND(N);
5472 break;
5473
5474 case ISD::FACOS:
5475 case ISD::FASIN:
5476 case ISD::FATAN:
5477 case ISD::FCEIL:
5478 case ISD::FCOS:
5479 case ISD::FCOSH:
5480 case ISD::FEXP:
5481 case ISD::FEXP2:
5482 case ISD::FEXP10:
5483 case ISD::FFLOOR:
5484 case ISD::FLOG:
5485 case ISD::FLOG10:
5486 case ISD::FLOG2:
5487 case ISD::FNEARBYINT:
5488 case ISD::FRINT:
5489 case ISD::FROUND:
5490 case ISD::FROUNDEVEN:
5491 case ISD::FSIN:
5492 case ISD::FSINH:
5493 case ISD::FSQRT:
5494 case ISD::FTAN:
5495 case ISD::FTANH:
5496 case ISD::FTRUNC:
5497 if (unrollExpandedOp())
5498 break;
5499 // If the target has custom/legal support for the scalar FP intrinsic ops
5500 // (they are probably not destined to become libcalls), then widen those
5501 // like any other unary ops.
5502 [[fallthrough]];
5503
5504 case ISD::ABS:
5505 case ISD::ABS_MIN_POISON:
5506 case ISD::BITREVERSE:
5507 case ISD::BSWAP:
5508 case ISD::CTLZ:
5509 case ISD::CTLZ_ZERO_POISON:
5510 case ISD::CTPOP:
5511 case ISD::CTTZ:
5512 case ISD::CTTZ_ZERO_POISON:
5513 case ISD::FNEG:
5514 case ISD::FABS:
5515 case ISD::FREEZE:
5516 case ISD::ARITH_FENCE:
5517 case ISD::FCANONICALIZE:
5518 case ISD::AssertNoFPClass:
5519 Res = WidenVecRes_Unary(N);
5520 break;
5521 case ISD::FMA:
5522 case ISD::FSHL:
5523 case ISD::FSHR:
5524 Res = WidenVecRes_Ternary(N);
5525 break;
5526 case ISD::FMODF:
5527 case ISD::FFREXP:
5528 case ISD::FSINCOS:
5529 case ISD::FSINCOSPI: {
5530 if (!unrollExpandedOp())
5531 Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
5532 break;
5533 }
5534 }
5535
5536 // If Res is null, the sub-method took care of registering the result.
5537 if (Res.getNode())
5538 SetWidenedVector(Op: SDValue(N, ResNo), Result: Res);
5539}
5540
5541SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
5542 // Ternary op widening.
5543 SDLoc dl(N);
5544 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5545 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5546 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5547 SDValue InOp3 = GetWidenedVector(Op: N->getOperand(Num: 2));
5548 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3);
5549}
5550
5551SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
5552 // Binary op widening.
5553 SDLoc dl(N);
5554 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5555 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5556 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5557 if (N->getNumOperands() == 2)
5558 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2,
5559 Flags: N->getFlags());
5560
5561 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
5562 assert((N->getOpcode() == ISD::VP_UDIV || N->getOpcode() == ISD::VP_SDIV ||
5563 N->getOpcode() == ISD::VP_UREM || N->getOpcode() == ISD::VP_SREM) &&
5564 "Expected VP opcode");
5565
5566 SDValue Mask =
5567 GetWidenedMask(Mask: N->getOperand(Num: 2), EC: WidenVT.getVectorElementCount());
5568 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT,
5569 Ops: {InOp1, InOp2, Mask, N->getOperand(Num: 3)}, Flags: N->getFlags());
5570}
5571
5572SDValue DAGTypeLegalizer::WidenVecRes_MaskedBinary(SDNode *N) {
5573 SDLoc dl(N);
5574 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5575 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5576 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5577 SDValue Mask = N->getOperand(Num: 2);
5578 EVT WideMaskVT = WidenVT.changeVectorElementType(
5579 Context&: *DAG.getContext(), EltVT: Mask.getValueType().getVectorElementType());
5580 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeros=*/FillWithZeroes: true);
5581 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask,
5582 Flags: N->getFlags());
5583}
5584
5585SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) {
5586 LLVMContext &Ctxt = *DAG.getContext();
5587 SDLoc dl(N);
5588
5589 SDValue LHS = N->getOperand(Num: 0);
5590 SDValue RHS = N->getOperand(Num: 1);
5591 EVT OpVT = LHS.getValueType();
5592 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector) {
5593 LHS = GetWidenedVector(Op: LHS);
5594 RHS = GetWidenedVector(Op: RHS);
5595 OpVT = LHS.getValueType();
5596 }
5597
5598 EVT WidenResVT = TLI.getTypeToTransformTo(Context&: Ctxt, VT: N->getValueType(ResNo: 0));
5599 ElementCount WidenResEC = WidenResVT.getVectorElementCount();
5600 if (WidenResEC == OpVT.getVectorElementCount()) {
5601 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenResVT, N1: LHS, N2: RHS);
5602 }
5603
5604 return DAG.UnrollVectorOp(N, ResNE: WidenResVT.getVectorNumElements());
5605}
5606
5607SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
5608 // Binary op widening, but with an extra operand that shouldn't be widened.
5609 SDLoc dl(N);
5610 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5611 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5612 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5613 SDValue InOp3 = N->getOperand(Num: 2);
5614 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: InOp3,
5615 Flags: N->getFlags());
5616}
5617
5618// Given a vector of operations that have been broken up to widen, see
5619// if we can collect them together into the next widest legal VT. This
5620// implementation is trap-safe.
5621static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
5622 SmallVectorImpl<SDValue> &ConcatOps,
5623 unsigned ConcatEnd, EVT VT, EVT MaxVT,
5624 EVT WidenVT) {
5625 // Check to see if we have a single operation with the widen type.
5626 if (ConcatEnd == 1) {
5627 VT = ConcatOps[0].getValueType();
5628 if (VT == WidenVT)
5629 return ConcatOps[0];
5630 }
5631
5632 SDLoc dl(ConcatOps[0]);
5633 EVT WidenEltVT = WidenVT.getVectorElementType();
5634
5635 // while (Some element of ConcatOps is not of type MaxVT) {
5636 // From the end of ConcatOps, collect elements of the same type and put
5637 // them into an op of the next larger supported type
5638 // }
5639 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
5640 int Idx = ConcatEnd - 1;
5641 VT = ConcatOps[Idx--].getValueType();
5642 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
5643 Idx--;
5644
5645 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
5646 EVT NextVT;
5647 do {
5648 NextSize *= 2;
5649 NextVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NextSize);
5650 } while (!TLI.isTypeLegal(VT: NextVT));
5651
5652 if (!VT.isVector()) {
5653 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
5654 SDValue VecOp = DAG.getPOISON(VT: NextVT);
5655 unsigned NumToInsert = ConcatEnd - Idx - 1;
5656 for (unsigned i = 0, OpIdx = Idx + 1; i < NumToInsert; i++, OpIdx++)
5657 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: ConcatOps[OpIdx], Idx: i);
5658 ConcatOps[Idx+1] = VecOp;
5659 ConcatEnd = Idx + 2;
5660 } else {
5661 // Vector type, create a CONCAT_VECTORS of type NextVT
5662 SDValue undefVec = DAG.getPOISON(VT);
5663 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
5664 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
5665 unsigned RealVals = ConcatEnd - Idx - 1;
5666 unsigned SubConcatEnd = 0;
5667 unsigned SubConcatIdx = Idx + 1;
5668 while (SubConcatEnd < RealVals)
5669 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
5670 while (SubConcatEnd < OpsToConcat)
5671 SubConcatOps[SubConcatEnd++] = undefVec;
5672 ConcatOps[SubConcatIdx] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl,
5673 VT: NextVT, Ops: SubConcatOps);
5674 ConcatEnd = SubConcatIdx + 1;
5675 }
5676 }
5677
5678 // Check to see if we have a single operation with the widen type.
5679 if (ConcatEnd == 1) {
5680 VT = ConcatOps[0].getValueType();
5681 if (VT == WidenVT)
5682 return ConcatOps[0];
5683 }
5684
5685 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
5686 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
5687 if (NumOps != ConcatEnd ) {
5688 SDValue UndefVal = DAG.getPOISON(VT: MaxVT);
5689 for (unsigned j = ConcatEnd; j < NumOps; ++j)
5690 ConcatOps[j] = UndefVal;
5691 }
5692 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
5693 Ops: ArrayRef(ConcatOps.data(), NumOps));
5694}
5695
5696SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
5697 // Binary op widening for operations that can trap.
5698 unsigned Opcode = N->getOpcode();
5699 SDLoc dl(N);
5700 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5701 EVT WidenEltVT = WidenVT.getVectorElementType();
5702 EVT VT = WidenVT;
5703 unsigned NumElts = VT.getVectorMinNumElements();
5704 const SDNodeFlags Flags = N->getFlags();
5705 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5706 NumElts = NumElts / 2;
5707 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5708 }
5709
5710 if (NumElts != 1 && !TLI.canOpTrap(Op: N->getOpcode(), VT)) {
5711 // Operation doesn't trap so just widen as normal.
5712 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5713 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5714 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, Flags);
5715 }
5716
5717 // Generate a vp.op if it is custom/legal for the target. This avoids need
5718 // to split and tile the subvectors (below), because the inactive lanes can
5719 // simply be disabled. To avoid possible recursion, only do this if the
5720 // widened mask type is legal.
5721 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode);
5722 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WidenVT)) {
5723 if (EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
5724 EC: WidenVT.getVectorElementCount());
5725 TLI.isTypeLegal(VT: WideMaskVT)) {
5726 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5727 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5728 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
5729 SDValue EVL =
5730 DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
5731 EC: N->getValueType(ResNo: 0).getVectorElementCount());
5732 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Mask, N4: EVL,
5733 Flags);
5734 }
5735 }
5736
5737 // FIXME: Improve support for scalable vectors.
5738 assert(!VT.isScalableVector() && "Scalable vectors not handled yet.");
5739
5740 // No legal vector version so unroll the vector operation and then widen.
5741 if (NumElts == 1)
5742 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
5743
5744 // Since the operation can trap, apply operation on the original vector.
5745 EVT MaxVT = VT;
5746 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
5747 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
5748 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5749
5750 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5751 unsigned ConcatEnd = 0; // Current ConcatOps index.
5752 int Idx = 0; // Current Idx into input vectors.
5753
5754 // NumElts := greatest legal vector size (at most WidenVT)
5755 // while (orig. vector has unhandled elements) {
5756 // take munches of size NumElts from the beginning and add to ConcatOps
5757 // NumElts := next smaller supported vector size or 1
5758 // }
5759 while (CurNumElts != 0) {
5760 while (CurNumElts >= NumElts) {
5761 SDValue EOp1 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp1, Idx);
5762 SDValue EOp2 = DAG.getExtractSubvector(DL: dl, VT, Vec: InOp2, Idx);
5763 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT, N1: EOp1, N2: EOp2, Flags);
5764 Idx += NumElts;
5765 CurNumElts -= NumElts;
5766 }
5767 do {
5768 NumElts = NumElts / 2;
5769 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5770 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5771
5772 if (NumElts == 1) {
5773 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5774 SDValue EOp1 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp1, Idx);
5775 SDValue EOp2 = DAG.getExtractVectorElt(DL: dl, VT: WidenEltVT, Vec: InOp2, Idx);
5776 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, DL: dl, VT: WidenEltVT,
5777 N1: EOp1, N2: EOp2, Flags);
5778 }
5779 CurNumElts = 0;
5780 }
5781 }
5782
5783 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5784}
5785
5786SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
5787 switch (N->getOpcode()) {
5788 case ISD::STRICT_FSETCC:
5789 case ISD::STRICT_FSETCCS:
5790 return WidenVecRes_STRICT_FSETCC(N);
5791 case ISD::STRICT_FP_EXTEND:
5792 case ISD::STRICT_FP_ROUND:
5793 case ISD::STRICT_FP_TO_SINT:
5794 case ISD::STRICT_FP_TO_UINT:
5795 case ISD::STRICT_SINT_TO_FP:
5796 case ISD::STRICT_UINT_TO_FP:
5797 return WidenVecRes_Convert_StrictFP(N);
5798 default:
5799 break;
5800 }
5801
5802 // StrictFP op widening for operations that can trap.
5803 unsigned NumOpers = N->getNumOperands();
5804 unsigned Opcode = N->getOpcode();
5805 SDLoc dl(N);
5806 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
5807 EVT WidenEltVT = WidenVT.getVectorElementType();
5808 EVT VT = WidenVT;
5809 unsigned NumElts = VT.getVectorNumElements();
5810 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
5811 NumElts = NumElts / 2;
5812 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5813 }
5814
5815 // No legal vector version so unroll the vector operation and then widen.
5816 if (NumElts == 1)
5817 return UnrollVectorOp_StrictFP(N, ResNE: WidenVT.getVectorNumElements());
5818
5819 // Since the operation can trap, apply operation on the original vector.
5820 EVT MaxVT = VT;
5821 SmallVector<SDValue, 4> InOps;
5822 unsigned CurNumElts = N->getValueType(ResNo: 0).getVectorNumElements();
5823
5824 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
5825 SmallVector<SDValue, 16> Chains;
5826 unsigned ConcatEnd = 0; // Current ConcatOps index.
5827 int Idx = 0; // Current Idx into input vectors.
5828
5829 // The Chain is the first operand.
5830 InOps.push_back(Elt: N->getOperand(Num: 0));
5831
5832 // Now process the remaining operands.
5833 for (unsigned i = 1; i < NumOpers; ++i) {
5834 SDValue Oper = N->getOperand(Num: i);
5835
5836 EVT OpVT = Oper.getValueType();
5837 if (OpVT.isVector()) {
5838 if (getTypeAction(VT: OpVT) == TargetLowering::TypeWidenVector)
5839 Oper = GetWidenedVector(Op: Oper);
5840 else {
5841 EVT WideOpVT =
5842 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5843 EC: WidenVT.getVectorElementCount());
5844 Oper = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WideOpVT,
5845 N1: DAG.getPOISON(VT: WideOpVT), N2: Oper,
5846 N3: DAG.getVectorIdxConstant(Val: 0, DL: dl));
5847 }
5848 }
5849
5850 InOps.push_back(Elt: Oper);
5851 }
5852
5853 // NumElts := greatest legal vector size (at most WidenVT)
5854 // while (orig. vector has unhandled elements) {
5855 // take munches of size NumElts from the beginning and add to ConcatOps
5856 // NumElts := next smaller supported vector size or 1
5857 // }
5858 while (CurNumElts != 0) {
5859 while (CurNumElts >= NumElts) {
5860 SmallVector<SDValue, 4> EOps;
5861
5862 for (unsigned i = 0; i < NumOpers; ++i) {
5863 SDValue Op = InOps[i];
5864
5865 EVT OpVT = Op.getValueType();
5866 if (OpVT.isVector()) {
5867 EVT OpExtractVT =
5868 EVT::getVectorVT(Context&: *DAG.getContext(), VT: OpVT.getVectorElementType(),
5869 EC: VT.getVectorElementCount());
5870 Op = DAG.getExtractSubvector(DL: dl, VT: OpExtractVT, Vec: Op, Idx);
5871 }
5872
5873 EOps.push_back(Elt: Op);
5874 }
5875
5876 EVT OperVT[] = {VT, MVT::Other};
5877 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: OperVT, Ops: EOps);
5878 ConcatOps[ConcatEnd++] = Oper;
5879 Chains.push_back(Elt: Oper.getValue(R: 1));
5880 Idx += NumElts;
5881 CurNumElts -= NumElts;
5882 }
5883 do {
5884 NumElts = NumElts / 2;
5885 VT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WidenEltVT, NumElements: NumElts);
5886 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
5887
5888 if (NumElts == 1) {
5889 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
5890 SmallVector<SDValue, 4> EOps;
5891
5892 for (unsigned i = 0; i < NumOpers; ++i) {
5893 SDValue Op = InOps[i];
5894
5895 EVT OpVT = Op.getValueType();
5896 if (OpVT.isVector())
5897 Op = DAG.getExtractVectorElt(DL: dl, VT: OpVT.getVectorElementType(), Vec: Op,
5898 Idx);
5899
5900 EOps.push_back(Elt: Op);
5901 }
5902
5903 EVT WidenVT[] = {WidenEltVT, MVT::Other};
5904 SDValue Oper = DAG.getNode(Opcode, DL: dl, ResultTys: WidenVT, Ops: EOps);
5905 ConcatOps[ConcatEnd++] = Oper;
5906 Chains.push_back(Elt: Oper.getValue(R: 1));
5907 }
5908 CurNumElts = 0;
5909 }
5910 }
5911
5912 // Build a factor node to remember all the Ops that have been created.
5913 SDValue NewChain;
5914 if (Chains.size() == 1)
5915 NewChain = Chains[0];
5916 else
5917 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
5918 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
5919
5920 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
5921}
5922
5923SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
5924 SDLoc DL(N);
5925 EVT ResVT = N->getValueType(ResNo: 0);
5926 EVT OvVT = N->getValueType(ResNo: 1);
5927 EVT WideResVT, WideOvVT;
5928 SDValue WideLHS, WideRHS;
5929
5930 // TODO: This might result in a widen/split loop.
5931 if (ResNo == 0) {
5932 WideResVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: ResVT);
5933 WideOvVT = EVT::getVectorVT(
5934 Context&: *DAG.getContext(), VT: OvVT.getVectorElementType(),
5935 NumElements: WideResVT.getVectorNumElements());
5936
5937 WideLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
5938 WideRHS = GetWidenedVector(Op: N->getOperand(Num: 1));
5939 } else {
5940 WideOvVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OvVT);
5941 WideResVT = EVT::getVectorVT(
5942 Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
5943 NumElements: WideOvVT.getVectorNumElements());
5944
5945 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5946 SDValue Poison = DAG.getPOISON(VT: WideResVT);
5947
5948 WideLHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5949 N2: N->getOperand(Num: 0), N3: Zero);
5950 WideRHS = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideResVT, N1: Poison,
5951 N2: N->getOperand(Num: 1), N3: Zero);
5952 }
5953
5954 SDVTList WideVTs = DAG.getVTList(VT1: WideResVT, VT2: WideOvVT);
5955 SDNode *WideNode = DAG.getNode(
5956 Opcode: N->getOpcode(), DL, VTList: WideVTs, N1: WideLHS, N2: WideRHS).getNode();
5957
5958 // Replace the other vector result not being explicitly widened here.
5959 unsigned OtherNo = 1 - ResNo;
5960 EVT OtherVT = N->getValueType(ResNo: OtherNo);
5961 if (getTypeAction(VT: OtherVT) == TargetLowering::TypeWidenVector) {
5962 SetWidenedVector(Op: SDValue(N, OtherNo), Result: SDValue(WideNode, OtherNo));
5963 } else {
5964 SDValue Zero = DAG.getVectorIdxConstant(Val: 0, DL);
5965 SDValue OtherVal = DAG.getNode(
5966 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: OtherVT, N1: SDValue(WideNode, OtherNo), N2: Zero);
5967 ReplaceValueWith(From: SDValue(N, OtherNo), To: OtherVal);
5968 }
5969
5970 return SDValue(WideNode, ResNo);
5971}
5972
5973SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
5974 LLVMContext &Ctx = *DAG.getContext();
5975 SDValue InOp = N->getOperand(Num: 0);
5976 SDLoc DL(N);
5977
5978 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo: 0));
5979 ElementCount WidenEC = WidenVT.getVectorElementCount();
5980
5981 EVT InVT = InOp.getValueType();
5982
5983 unsigned Opcode = N->getOpcode();
5984 const SDNodeFlags Flags = N->getFlags();
5985
5986 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
5987 // not equal that of WidenVT.
5988 if (N->getOpcode() == ISD::ZERO_EXTEND &&
5989 getTypeAction(VT: InVT) == TargetLowering::TypePromoteInteger &&
5990 TLI.getTypeToTransformTo(Context&: Ctx, VT: InVT).getScalarSizeInBits() !=
5991 WidenVT.getScalarSizeInBits()) {
5992 InOp = ZExtPromotedInteger(Op: InOp);
5993 InVT = InOp.getValueType();
5994 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
5995 Opcode = ISD::TRUNCATE;
5996 }
5997
5998 EVT InEltVT = InVT.getVectorElementType();
5999 EVT InWidenVT = EVT::getVectorVT(Context&: Ctx, VT: InEltVT, EC: WidenEC);
6000 ElementCount InVTEC = InVT.getVectorElementCount();
6001
6002 // Helper to build node with all scalar trailing operands.
6003 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
6004 if (N->getNumOperands() == 1)
6005 return DAG.getNode(Opcode, DL, VT, Operand: Op, Flags);
6006 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
6007 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
6008 N4: N->getOperand(Num: 3), Flags);
6009 return DAG.getNode(Opcode, DL, VT, N1: Op, N2: N->getOperand(Num: 1), Flags);
6010 };
6011
6012 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6013 InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6014 InVT = InOp.getValueType();
6015 InVTEC = InVT.getVectorElementCount();
6016 if (InVTEC == WidenEC)
6017 return MakeConvertNode(WidenVT, InOp);
6018 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
6019 // If both input and result vector types are of same width, extend
6020 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
6021 // accepts fewer elements in the result than in the input.
6022 if (Opcode == ISD::ANY_EXTEND)
6023 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6024 if (Opcode == ISD::SIGN_EXTEND)
6025 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6026 if (Opcode == ISD::ZERO_EXTEND)
6027 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT: WidenVT, Operand: InOp);
6028 }
6029
6030 // For TRUNCATE, try to widen using the legal EC of the input type instead
6031 // if the legalisation action for that intermediate type is not widening.
6032 // E.g. for trunc nxv1i64 -> nxv1i8 where
6033 // - nxv1i64 input gets widened to nxv2i64
6034 // - nxv1i8 output gets widened to nxv16i8
6035 // Then one can try widening the result to nxv2i8 (instead of going all the
6036 // way to nxv16i8) if this later allows type promotion.
6037 EVT MidResVT =
6038 EVT::getVectorVT(Context&: Ctx, VT: WidenVT.getVectorElementType(), EC: InVTEC);
6039 if (N->getOpcode() == ISD::TRUNCATE &&
6040 getTypeAction(VT: MidResVT) == TargetLowering::TypePromoteInteger) {
6041 SDValue MidRes = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MidResVT, Operand: InOp, Flags);
6042 return DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: MidRes, Idx: 0);
6043 }
6044 }
6045
6046 if (TLI.isTypeLegal(VT: InWidenVT)) {
6047 // Because the result and the input are different vector types, widening
6048 // the result could create a legal type but widening the input might make
6049 // it an illegal type that might lead to repeatedly splitting the input
6050 // and then widening it. To avoid this, we widen the input only if
6051 // it results in a legal type.
6052 if (WidenEC.isKnownMultipleOf(RHS: InVTEC.getKnownMinValue())) {
6053 // Widen the input and call convert on the widened input vector.
6054 unsigned NumConcat =
6055 WidenEC.getKnownMinValue() / InVTEC.getKnownMinValue();
6056 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getPOISON(VT: InVT));
6057 Ops[0] = InOp;
6058 SDValue InVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: InWidenVT, Ops);
6059 return MakeConvertNode(WidenVT, InVec);
6060 }
6061
6062 if (InVTEC.isKnownMultipleOf(RHS: WidenEC.getKnownMinValue())) {
6063 SDValue InVal = DAG.getExtractSubvector(DL, VT: InWidenVT, Vec: InOp, Idx: 0);
6064 // Extract the input and convert the shorten input vector.
6065 return MakeConvertNode(WidenVT, InVal);
6066 }
6067 }
6068
6069 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6070 EVT EltVT = WidenVT.getVectorElementType();
6071 SmallVector<SDValue, 16> Ops(WidenEC.getFixedValue(), DAG.getPOISON(VT: EltVT));
6072 // Use the original element count so we don't do more scalar opts than
6073 // necessary.
6074 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6075 for (unsigned i=0; i < MinElts; ++i) {
6076 SDValue Val = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6077 Ops[i] = MakeConvertNode(EltVT, Val);
6078 }
6079
6080 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6081}
6082
6083SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
6084 SDLoc dl(N);
6085 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6086 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6087
6088 SDValue Src = N->getOperand(Num: 0);
6089 EVT SrcVT = Src.getValueType();
6090
6091 // Also widen the input.
6092 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6093 Src = GetWidenedVector(Op: Src);
6094 SrcVT = Src.getValueType();
6095 }
6096
6097 // Input and output not widened to the same size, give up.
6098 if (WidenNumElts != SrcVT.getVectorElementCount())
6099 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6100
6101 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, N1: Src, N2: N->getOperand(Num: 1));
6102}
6103
6104SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) {
6105 SDLoc dl(N);
6106 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6107 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
6108
6109 SDValue Src = N->getOperand(Num: 0);
6110 EVT SrcVT = Src.getValueType();
6111
6112 // Also widen the input.
6113 if (getTypeAction(VT: SrcVT) == TargetLowering::TypeWidenVector) {
6114 Src = GetWidenedVector(Op: Src);
6115 SrcVT = Src.getValueType();
6116 }
6117
6118 // Input and output not widened to the same size, give up.
6119 if (WidenNumElts != SrcVT.getVectorElementCount())
6120 return DAG.UnrollVectorOp(N, ResNE: WidenNumElts.getKnownMinValue());
6121
6122 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WidenVT, Operand: Src);
6123}
6124
6125SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
6126 SDValue InOp = N->getOperand(Num: 1);
6127 SDLoc DL(N);
6128 SmallVector<SDValue, 4> NewOps(N->ops());
6129
6130 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6131 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6132
6133 EVT InVT = InOp.getValueType();
6134 EVT InEltVT = InVT.getVectorElementType();
6135
6136 unsigned Opcode = N->getOpcode();
6137
6138 // FIXME: Optimizations need to be implemented here.
6139
6140 // Otherwise unroll into some nasty scalar code and rebuild the vector.
6141 EVT EltVT = WidenVT.getVectorElementType();
6142 std::array<EVT, 2> EltVTs = {._M_elems: {EltVT, MVT::Other}};
6143 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getPOISON(VT: EltVT));
6144 SmallVector<SDValue, 32> OpChains;
6145 // Use the original element count so we don't do more scalar opts than
6146 // necessary.
6147 unsigned MinElts = N->getValueType(ResNo: 0).getVectorNumElements();
6148 for (unsigned i=0; i < MinElts; ++i) {
6149 NewOps[1] = DAG.getExtractVectorElt(DL, VT: InEltVT, Vec: InOp, Idx: i);
6150 Ops[i] = DAG.getNode(Opcode, DL, ResultTys: EltVTs, Ops: NewOps);
6151 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
6152 }
6153 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OpChains);
6154 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6155
6156 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6157}
6158
6159SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
6160 unsigned Opcode = N->getOpcode();
6161 SDValue InOp = N->getOperand(Num: 0);
6162 SDLoc DL(N);
6163
6164 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6165 EVT WidenSVT = WidenVT.getVectorElementType();
6166 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6167
6168 EVT InVT = InOp.getValueType();
6169 EVT InSVT = InVT.getVectorElementType();
6170 unsigned InVTNumElts = InVT.getVectorNumElements();
6171
6172 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6173 InOp = GetWidenedVector(Op: InOp);
6174 InVT = InOp.getValueType();
6175 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
6176 switch (Opcode) {
6177 case ISD::ANY_EXTEND_VECTOR_INREG:
6178 case ISD::SIGN_EXTEND_VECTOR_INREG:
6179 case ISD::ZERO_EXTEND_VECTOR_INREG:
6180 return DAG.getNode(Opcode, DL, VT: WidenVT, Operand: InOp);
6181 }
6182 }
6183 }
6184
6185 // Unroll, extend the scalars and rebuild the vector.
6186 SmallVector<SDValue, 16> Ops;
6187 for (unsigned i = 0, e = std::min(a: InVTNumElts, b: WidenNumElts); i != e; ++i) {
6188 SDValue Val = DAG.getExtractVectorElt(DL, VT: InSVT, Vec: InOp, Idx: i);
6189 switch (Opcode) {
6190 case ISD::ANY_EXTEND_VECTOR_INREG:
6191 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: WidenSVT, Operand: Val);
6192 break;
6193 case ISD::SIGN_EXTEND_VECTOR_INREG:
6194 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: WidenSVT, Operand: Val);
6195 break;
6196 case ISD::ZERO_EXTEND_VECTOR_INREG:
6197 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenSVT, Operand: Val);
6198 break;
6199 default:
6200 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
6201 }
6202 Ops.push_back(Elt: Val);
6203 }
6204
6205 while (Ops.size() != WidenNumElts)
6206 Ops.push_back(Elt: DAG.getPOISON(VT: WidenSVT));
6207
6208 return DAG.getBuildVector(VT: WidenVT, DL, Ops);
6209}
6210
6211SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
6212 // If this is an FCOPYSIGN with same input types, we can treat it as a
6213 // normal (can trap) binary op.
6214 if (N->getOperand(Num: 0).getValueType() == N->getOperand(Num: 1).getValueType())
6215 return WidenVecRes_BinaryCanTrap(N);
6216
6217 // If the types are different, fall back to unrolling.
6218 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6219 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6220}
6221
6222/// Result and first source operand are different scalar types, but must have
6223/// the same number of elements. There is an additional control argument which
6224/// should be passed through unchanged.
6225SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) {
6226 SDValue FpValue = N->getOperand(Num: 0);
6227 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6228 if (getTypeAction(VT: FpValue.getValueType()) != TargetLowering::TypeWidenVector)
6229 return DAG.UnrollVectorOp(N, ResNE: WidenVT.getVectorNumElements());
6230 SDValue Arg = GetWidenedVector(Op: FpValue);
6231 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Ops: {Arg, N->getOperand(Num: 1)},
6232 Flags: N->getFlags());
6233}
6234
6235SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) {
6236 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6237 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6238 SDValue RHS = N->getOperand(Num: 1);
6239 EVT ExpVT = RHS.getValueType();
6240 SDValue ExpOp = RHS;
6241 if (ExpVT.isVector()) {
6242 EVT WideExpVT = WidenVT.changeVectorElementType(
6243 Context&: *DAG.getContext(), EltVT: ExpVT.getVectorElementType());
6244 ExpOp = ModifyToType(InOp: RHS, NVT: WideExpVT);
6245 }
6246
6247 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: ExpOp);
6248}
6249
6250SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
6251 // Unary op widening.
6252 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6253 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6254 if (N->getNumOperands() == 1)
6255 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: InOp, Flags: N->getFlags());
6256 assert(N->getOpcode() == ISD::AssertNoFPClass && "unexpected opcode");
6257 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, N1: InOp, N2: N->getOperand(Num: 1),
6258 Flags: N->getFlags());
6259}
6260
6261SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
6262 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6263 EVT ExtVT = EVT::getVectorVT(
6264 Context&: *DAG.getContext(),
6265 VT: cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getVectorElementType(),
6266 EC: WidenVT.getVectorElementCount());
6267 SDValue WidenLHS = GetWidenedVector(Op: N->getOperand(Num: 0));
6268 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N),
6269 VT: WidenVT, N1: WidenLHS, N2: DAG.getValueType(ExtVT));
6270}
6271
6272SDValue DAGTypeLegalizer::WidenVecRes_UnaryOpWithTwoResults(SDNode *N,
6273 unsigned ResNo) {
6274 EVT VT0 = N->getValueType(ResNo: 0);
6275 EVT VT1 = N->getValueType(ResNo: 1);
6276
6277 assert(VT0.isVector() && VT1.isVector() &&
6278 VT0.getVectorElementCount() == VT1.getVectorElementCount() &&
6279 "expected both results to be vectors of matching element count");
6280
6281 LLVMContext &Ctx = *DAG.getContext();
6282 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6283
6284 EVT WidenVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: N->getValueType(ResNo));
6285 ElementCount WidenEC = WidenVT.getVectorElementCount();
6286
6287 EVT WidenVT0 = EVT::getVectorVT(Context&: Ctx, VT: VT0.getVectorElementType(), EC: WidenEC);
6288 EVT WidenVT1 = EVT::getVectorVT(Context&: Ctx, VT: VT1.getVectorElementType(), EC: WidenEC);
6289
6290 SDNode *WidenNode =
6291 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {WidenVT0, WidenVT1}, Ops: InOp)
6292 .getNode();
6293
6294 ReplaceOtherWidenResults(N, WidenNode, WidenResNo: ResNo);
6295 return SDValue(WidenNode, ResNo);
6296}
6297
6298SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
6299 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
6300 return GetWidenedVector(Op: WidenVec);
6301}
6302
6303SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) {
6304 SDLoc DL(N);
6305 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6306 ElementCount WidenEC = WidenVT.getVectorElementCount();
6307 auto *AddrSpaceCastN = cast<AddrSpaceCastSDNode>(Val: N);
6308
6309 // The source has the same number of elements as the result, so widen it to
6310 // match WidenVT. It only lives in the widened-vector map if it is itself
6311 // widened; otherwise pad it up to the widened element count.
6312 SDValue InOp = N->getOperand(Num: 0);
6313 EVT InVT = InOp.getValueType();
6314 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
6315 InOp = GetWidenedVector(Op: InOp);
6316 } else {
6317 EVT InWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6318 VT: InVT.getVectorElementType(), EC: WidenEC);
6319 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: InWidenVT), SubVec: InOp, Idx: 0);
6320 }
6321
6322 return DAG.getAddrSpaceCast(dl: DL, VT: WidenVT, Ptr: InOp,
6323 SrcAS: AddrSpaceCastN->getSrcAddressSpace(),
6324 DestAS: AddrSpaceCastN->getDestAddressSpace());
6325}
6326
6327SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
6328 SDValue InOp = N->getOperand(Num: 0);
6329 EVT InVT = InOp.getValueType();
6330 EVT VT = N->getValueType(ResNo: 0);
6331 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6332 SDLoc dl(N);
6333
6334 switch (getTypeAction(VT: InVT)) {
6335 case TargetLowering::TypeLegal:
6336 break;
6337 case TargetLowering::TypeScalarizeScalableVector:
6338 report_fatal_error(reason: "Scalarization of scalable vectors is not supported.");
6339 case TargetLowering::TypePromoteInteger: {
6340 // If the incoming type is a vector that is being promoted, then
6341 // we know that the elements are arranged differently and that we
6342 // must perform the conversion using a stack slot.
6343 if (InVT.isVector())
6344 break;
6345
6346 // If the InOp is promoted to the same size, convert it. Otherwise,
6347 // fall out of the switch and widen the promoted input.
6348 SDValue NInOp = GetPromotedInteger(Op: InOp);
6349 EVT NInVT = NInOp.getValueType();
6350 if (WidenVT.bitsEq(VT: NInVT)) {
6351 // For big endian targets we need to shift the input integer or the
6352 // interesting bits will end up at the wrong place.
6353 if (DAG.getDataLayout().isBigEndian()) {
6354 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
6355 NInOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NInVT, N1: NInOp,
6356 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: NInVT, DL: dl));
6357 }
6358 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NInOp);
6359 }
6360 InOp = NInOp;
6361 InVT = NInVT;
6362 break;
6363 }
6364 case TargetLowering::TypeSoftenFloat:
6365 case TargetLowering::TypeSoftPromoteHalf:
6366 case TargetLowering::TypeExpandInteger:
6367 case TargetLowering::TypeExpandFloat:
6368 case TargetLowering::TypeScalarizeVector:
6369 case TargetLowering::TypeSplitVector:
6370 break;
6371 case TargetLowering::TypeWidenVector:
6372 // If the InOp is widened to the same size, convert it. Otherwise, fall
6373 // out of the switch and widen the widened input.
6374 InOp = GetWidenedVector(Op: InOp);
6375 InVT = InOp.getValueType();
6376 if (WidenVT.bitsEq(VT: InVT))
6377 // The input widens to the same size. Convert to the widen value.
6378 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: InOp);
6379 break;
6380 }
6381
6382 unsigned WidenSize = WidenVT.getSizeInBits();
6383 unsigned InSize = InVT.getSizeInBits();
6384 unsigned InScalarSize = InVT.getScalarSizeInBits();
6385 // x86mmx is not an acceptable vector element type, so don't try.
6386 if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
6387 // Determine new input vector type. The new input vector type will use
6388 // the same element type (if its a vector) or use the input type as a
6389 // vector. It is the same size as the type to widen to.
6390 EVT NewInVT;
6391 unsigned NewNumParts = WidenSize / InSize;
6392 if (InVT.isVector()) {
6393 EVT InEltVT = InVT.getVectorElementType();
6394 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InEltVT,
6395 NumElements: WidenSize / InEltVT.getSizeInBits());
6396 } else {
6397 // For big endian systems, using the promoted input scalar type
6398 // to produce the scalar_to_vector would put the desired bits into
6399 // the least significant byte(s) of the wider element zero. This
6400 // will mean that the users of the result vector are using incorrect
6401 // bits. Use the original input type instead. Although either input
6402 // type can be used on little endian systems, for consistency we
6403 // use the original type there as well.
6404 EVT OrigInVT = N->getOperand(Num: 0).getValueType();
6405 NewNumParts = WidenSize / OrigInVT.getSizeInBits();
6406 NewInVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OrigInVT, NumElements: NewNumParts);
6407 }
6408
6409 if (TLI.isTypeLegal(VT: NewInVT)) {
6410 SDValue NewVec;
6411 if (InVT.isVector()) {
6412 // Because the result and the input are different vector types, widening
6413 // the result could create a legal type but widening the input might
6414 // make it an illegal type that might lead to repeatedly splitting the
6415 // input and then widening it. To avoid this, we widen the input only if
6416 // it results in a legal type.
6417 if (WidenSize % InSize == 0) {
6418 SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getPOISON(VT: InVT));
6419 Ops[0] = InOp;
6420
6421 NewVec = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewInVT, Ops);
6422 } else {
6423 SmallVector<SDValue, 16> Ops;
6424 DAG.ExtractVectorElements(Op: InOp, Args&: Ops);
6425 Ops.append(NumInputs: WidenSize / InScalarSize - Ops.size(),
6426 Elt: DAG.getPOISON(VT: InVT.getVectorElementType()));
6427
6428 NewVec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: NewInVT, Ops);
6429 }
6430 } else {
6431 NewVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewInVT, Operand: InOp);
6432 }
6433 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: NewVec);
6434 }
6435 }
6436
6437 return CreateStackStoreLoad(Op: InOp, DestVT: WidenVT);
6438}
6439
6440SDValue DAGTypeLegalizer::WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
6441 return DAG.getNode(
6442 Opcode: N->getOpcode(), DL: SDLoc(N),
6443 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)),
6444 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
6445}
6446
6447SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
6448 SDLoc dl(N);
6449 // Build a vector with poison for the new nodes.
6450 EVT VT = N->getValueType(ResNo: 0);
6451
6452 // Integer BUILD_VECTOR operands may be larger than the node's vector element
6453 // type. The POISONs need to have the same type as the existing operands.
6454 EVT EltVT = N->getOperand(Num: 0).getValueType();
6455 unsigned NumElts = VT.getVectorNumElements();
6456
6457 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6458 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6459
6460 SmallVector<SDValue, 16> NewOps(N->ops());
6461 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
6462 NewOps.append(NumInputs: WidenNumElts - NumElts, Elt: DAG.getPOISON(VT: EltVT));
6463
6464 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: NewOps);
6465}
6466
6467SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
6468 EVT InVT = N->getOperand(Num: 0).getValueType();
6469 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6470 SDLoc dl(N);
6471 unsigned NumOperands = N->getNumOperands();
6472
6473 bool InputWidened = false; // Indicates we need to widen the input.
6474 if (getTypeAction(VT: InVT) != TargetLowering::TypeWidenVector) {
6475 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6476 unsigned NumInElts = InVT.getVectorMinNumElements();
6477 if (WidenNumElts % NumInElts == 0) {
6478 // Add undef vectors to widen to correct length.
6479 unsigned NumConcat = WidenNumElts / NumInElts;
6480 SDValue UndefVal = DAG.getPOISON(VT: InVT);
6481 SmallVector<SDValue, 16> Ops(NumConcat);
6482 for (unsigned i=0; i < NumOperands; ++i)
6483 Ops[i] = N->getOperand(Num: i);
6484 for (unsigned i = NumOperands; i != NumConcat; ++i)
6485 Ops[i] = UndefVal;
6486 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops);
6487 }
6488 } else {
6489 InputWidened = true;
6490 if (WidenVT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
6491 // The inputs and the result are widen to the same value.
6492 unsigned i;
6493 for (i=1; i < NumOperands; ++i)
6494 if (!N->getOperand(Num: i).isUndef())
6495 break;
6496
6497 if (i == NumOperands)
6498 // Everything but the first operand is an UNDEF so just return the
6499 // widened first operand.
6500 return GetWidenedVector(Op: N->getOperand(Num: 0));
6501
6502 if (NumOperands == 2) {
6503 assert(!WidenVT.isScalableVector() &&
6504 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
6505 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6506 unsigned NumInElts = InVT.getVectorNumElements();
6507
6508 // Replace concat of two operands with a shuffle.
6509 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
6510 for (unsigned i = 0; i < NumInElts; ++i) {
6511 MaskOps[i] = i;
6512 MaskOps[i + NumInElts] = i + WidenNumElts;
6513 }
6514 return DAG.getVectorShuffle(VT: WidenVT, dl,
6515 N1: GetWidenedVector(Op: N->getOperand(Num: 0)),
6516 N2: GetWidenedVector(Op: N->getOperand(Num: 1)),
6517 Mask: MaskOps);
6518 }
6519 }
6520 }
6521
6522 if (WidenVT.isScalableVector()) {
6523 SDValue WideVec = DAG.getPOISON(VT: WidenVT);
6524 unsigned NumInElts = InVT.getVectorMinNumElements();
6525 for (unsigned I = 0; I < NumOperands; ++I)
6526 WideVec =
6527 DAG.getInsertSubvector(DL: dl, Vec: WideVec, SubVec: N->getOperand(Num: I), Idx: I * NumInElts);
6528 return WideVec;
6529 }
6530
6531 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6532 unsigned NumInElts = InVT.getVectorNumElements();
6533
6534 // Fall back to use extracts and build vector.
6535 EVT EltVT = WidenVT.getVectorElementType();
6536 SmallVector<SDValue, 16> Ops(WidenNumElts);
6537 unsigned Idx = 0;
6538 for (unsigned i=0; i < NumOperands; ++i) {
6539 SDValue InOp = N->getOperand(Num: i);
6540 if (InputWidened)
6541 InOp = GetWidenedVector(Op: InOp);
6542 for (unsigned j = 0; j < NumInElts; ++j)
6543 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
6544 }
6545 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6546 for (; Idx < WidenNumElts; ++Idx)
6547 Ops[Idx] = UndefVal;
6548 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6549}
6550
6551SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) {
6552 EVT VT = N->getValueType(ResNo: 0);
6553 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6554 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
6555 SDValue InOp2 = N->getOperand(Num: 1);
6556 SDValue Idx = N->getOperand(Num: 2);
6557 SDLoc dl(N);
6558 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp1, N2: InOp2, N3: Idx);
6559}
6560
6561SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
6562 EVT VT = N->getValueType(ResNo: 0);
6563 EVT EltVT = VT.getVectorElementType();
6564 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6565 SDValue InOp = N->getOperand(Num: 0);
6566 SDValue Idx = N->getOperand(Num: 1);
6567 SDLoc dl(N);
6568
6569 auto InOpTypeAction = getTypeAction(VT: InOp.getValueType());
6570 if (InOpTypeAction == TargetLowering::TypeWidenVector)
6571 InOp = GetWidenedVector(Op: InOp);
6572
6573 EVT InVT = InOp.getValueType();
6574
6575 // Check if we can just return the input vector after widening.
6576 uint64_t IdxVal = Idx->getAsZExtVal();
6577 if (IdxVal == 0 && InVT == WidenVT)
6578 return InOp;
6579
6580 // Check if we can extract from the vector.
6581 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
6582 unsigned InNumElts = InVT.getVectorMinNumElements();
6583 unsigned VTNumElts = VT.getVectorMinNumElements();
6584 assert(IdxVal % VTNumElts == 0 &&
6585 "Expected Idx to be a multiple of subvector minimum vector length");
6586 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
6587 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: WidenVT, N1: InOp, N2: Idx);
6588
6589 if (VT.isScalableVector()) {
6590 // Try to split the operation up into smaller extracts and concat the
6591 // results together, e.g.
6592 // nxv6i64 extract_subvector(nxv12i64, 6)
6593 // <->
6594 // nxv8i64 concat(
6595 // nxv2i64 extract_subvector(nxv16i64, 6)
6596 // nxv2i64 extract_subvector(nxv16i64, 8)
6597 // nxv2i64 extract_subvector(nxv16i64, 10)
6598 // undef)
6599 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
6600 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
6601 "down type's element count");
6602 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
6603 EC: ElementCount::getScalable(MinVal: GCD));
6604 // Avoid recursion around e.g. nxv1i8.
6605 if (getTypeAction(VT: PartVT) != TargetLowering::TypeWidenVector) {
6606 SmallVector<SDValue> Parts;
6607 unsigned I = 0;
6608 for (; I < VTNumElts / GCD; ++I)
6609 Parts.push_back(
6610 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: IdxVal + I * GCD));
6611 for (; I < WidenNumElts / GCD; ++I)
6612 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
6613
6614 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
6615 }
6616
6617 // Fallback to extracting through memory.
6618
6619 Align Alignment = DAG.getReducedAlign(VT: InVT, /*UseABI=*/false);
6620 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: InVT.getStoreSize(), Alignment);
6621 MachineFunction &MF = DAG.getMachineFunction();
6622 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
6623 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
6624
6625 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
6626 PtrInfo, F: MachineMemOperand::MOStore,
6627 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6628 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
6629 PtrInfo, F: MachineMemOperand::MOLoad,
6630 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
6631
6632 // Write out the input vector.
6633 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: InOp, Ptr: StackPtr, MMO: StoreMMO);
6634
6635 // Build a mask to match the length of the non-widened result.
6636 SDValue Mask =
6637 DAG.getMaskFromElementCount(DL: dl, VT: WidenVT, Len: VT.getVectorElementCount());
6638
6639 // Read back the sub-vector setting the remaining lanes to poison.
6640 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: InVT, SubVecVT: VT, Index: Idx);
6641 return DAG.getMaskedLoad(
6642 VT: WidenVT, dl, Chain: Ch, Base: StackPtr, Offset: DAG.getPOISON(VT: StackPtr.getValueType()), Mask,
6643 Src0: DAG.getPOISON(VT: WidenVT), MemVT: VT, MMO: LoadMMO, AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
6644 }
6645
6646 // We could try widening the input to the right length but for now, extract
6647 // the original elements, fill the rest with undefs and build a vector.
6648 SmallVector<SDValue, 16> Ops(WidenNumElts);
6649 unsigned i;
6650 for (i = 0; i < VTNumElts; ++i)
6651 Ops[i] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: IdxVal + i);
6652
6653 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
6654 for (; i < WidenNumElts; ++i)
6655 Ops[i] = UndefVal;
6656 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
6657}
6658
6659SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) {
6660 SDValue InOp = ModifyToType(
6661 InOp: N->getOperand(Num: 0),
6662 NVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0)), FillWithZeroes: true);
6663 return DAG.getNode(Opcode: ISD::AssertZext, DL: SDLoc(N), VT: InOp.getValueType(), N1: InOp,
6664 N2: N->getOperand(Num: 1));
6665}
6666
6667SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
6668 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
6669 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
6670 VT: InOp.getValueType(), N1: InOp,
6671 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2));
6672}
6673
6674/// Either return the same load or provide appropriate casts
6675/// from the load and return that.
6676static SDValue coerceLoadedValue(SDValue LdOp, EVT FirstVT, EVT WidenVT,
6677 TypeSize LdWidth, TypeSize FirstVTWidth,
6678 SDLoc dl, SelectionDAG &DAG) {
6679 assert(TypeSize::isKnownLE(LdWidth, FirstVTWidth) &&
6680 "Load width must be less than or equal to first value type width");
6681 TypeSize WidenWidth = WidenVT.getSizeInBits();
6682 if (!FirstVT.isVector()) {
6683 unsigned NumElts =
6684 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6685 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6686 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT, Operand: LdOp);
6687 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: WidenVT, Operand: VecOp);
6688 }
6689 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6690 return LdOp;
6691}
6692
6693/// Inverse of coerceLoadedValue: pull a FirstVT-sized scalar/vector out of the
6694/// widened value so it can be issued in a single atomic store.
6695static SDValue coerceStoredValue(SDValue StVal, EVT FirstVT, EVT WidenVT,
6696 TypeSize FirstVTWidth, const SDLoc &dl,
6697 SelectionDAG &DAG) {
6698 TypeSize WidenWidth = WidenVT.getSizeInBits();
6699 if (!FirstVT.isVector()) {
6700 unsigned NumElts =
6701 WidenWidth.getFixedValue() / FirstVTWidth.getFixedValue();
6702 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: FirstVT, NumElements: NumElts);
6703 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: StVal);
6704 return DAG.getExtractVectorElt(DL: dl, VT: FirstVT, Vec: VecOp, Idx: 0);
6705 }
6706 assert(FirstVT == WidenVT && "First value type must equal widen value type");
6707 return StVal;
6708}
6709
6710static std::optional<EVT> findMemType(SelectionDAG &DAG,
6711 const TargetLowering &TLI, unsigned Width,
6712 EVT WidenVT, unsigned Align,
6713 unsigned WidenEx);
6714
6715SDValue DAGTypeLegalizer::WidenVecRes_ATOMIC_LOAD(AtomicSDNode *LD) {
6716 EVT WidenVT =
6717 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
6718 EVT LdVT = LD->getMemoryVT();
6719 SDLoc dl(LD);
6720
6721 // Load information
6722 SDValue Chain = LD->getChain();
6723 SDValue BasePtr = LD->getBasePtr();
6724
6725 TypeSize LdWidth = LdVT.getSizeInBits();
6726 TypeSize WidenWidth = WidenVT.getSizeInBits();
6727 TypeSize WidthDiff = WidenWidth - LdWidth;
6728
6729 // Find the vector type that can load from.
6730 std::optional<EVT> FirstVT =
6731 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, /*LdAlign=*/Align: 0,
6732 WidenEx: WidthDiff.getKnownMinValue());
6733
6734 if (!FirstVT)
6735 return SDValue();
6736
6737 SmallVector<EVT, 8> MemVTs;
6738 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
6739
6740 SDValue LdOp = DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl, MemVT: *FirstVT, VT: *FirstVT,
6741 Chain, Ptr: BasePtr, MMO: LD->getMemOperand());
6742
6743 // Load the element with one instruction.
6744 SDValue Result = coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth,
6745 FirstVTWidth, dl, DAG);
6746
6747 // Modified the chain - switch anything that used the old chain to use
6748 // the new one.
6749 ReplaceValueWith(From: SDValue(LD, 1), To: LdOp.getValue(R: 1));
6750 return Result;
6751}
6752
6753SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
6754 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
6755 ISD::LoadExtType ExtType = LD->getExtensionType();
6756
6757 // A vector must always be stored in memory as-is, i.e. without any padding
6758 // between the elements, since various code depend on it, e.g. in the
6759 // handling of a bitcast of a vector type to int, which may be done with a
6760 // vector store followed by an integer load. A vector that does not have
6761 // elements that are byte-sized must therefore be stored as an integer
6762 // built out of the extracted vector elements.
6763 if (!LD->getMemoryVT().isByteSized()) {
6764 SDValue Value, NewChain;
6765 std::tie(args&: Value, args&: NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
6766 ReplaceValueWith(From: SDValue(LD, 0), To: Value);
6767 ReplaceValueWith(From: SDValue(LD, 1), To: NewChain);
6768 return SDValue();
6769 }
6770
6771 // Generate a vector-predicated load if it is custom/legal on the target. To
6772 // avoid possible recursion, only do this if the widened mask type is legal.
6773 // FIXME: Not all targets may support EVL in VP_LOAD. These will have been
6774 // removed from the IR by the ExpandVectorPredication pass but we're
6775 // reintroducing them here.
6776 EVT VT = LD->getValueType(ResNo: 0);
6777 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6778 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
6779
6780 if (ExtType == ISD::NON_EXTLOAD &&
6781 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WideVT) &&
6782 TLI.isTypeLegal(VT: WideMaskVT)) {
6783 SDLoc DL(N);
6784 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
6785 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
6786 EC: VT.getVectorElementCount());
6787 SDValue NewLoad =
6788 DAG.getLoadVP(AM: LD->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WideVT, dl: DL,
6789 Chain: LD->getChain(), Ptr: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6790 EVL, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
6791
6792 // Modified the chain - switch anything that used the old chain to use
6793 // the new one.
6794 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6795
6796 return NewLoad;
6797 }
6798
6799 SDValue Result;
6800 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
6801 if (ExtType != ISD::NON_EXTLOAD)
6802 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
6803 else
6804 Result = GenWidenVectorLoads(LdChain, LD);
6805
6806 if (Result) {
6807 // If we generate a single load, we can use that for the chain. Otherwise,
6808 // build a factor node to remember the multiple loads are independent and
6809 // chain to that.
6810 SDValue NewChain;
6811 if (LdChain.size() == 1)
6812 NewChain = LdChain[0];
6813 else
6814 NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other, Ops: LdChain);
6815
6816 // Modified the chain - switch anything that used the old chain to use
6817 // the new one.
6818 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
6819
6820 return Result;
6821 }
6822
6823 if (VT.isVector()) {
6824 // If all else fails replace the load with a wide masked load.
6825 SDLoc DL(N);
6826 SDValue Mask =
6827 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: VT.getVectorElementCount());
6828
6829 SDValue NewLoad = DAG.getMaskedLoad(
6830 VT: WideVT, dl: DL, Chain: LD->getChain(), Base: LD->getBasePtr(), Offset: LD->getOffset(), Mask,
6831 Src0: DAG.getPOISON(VT: WideVT), MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(),
6832 AM: LD->getAddressingMode(), LD->getExtensionType());
6833
6834 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6835 return NewLoad;
6836 }
6837
6838 report_fatal_error(reason: "Unable to widen vector load");
6839}
6840
6841SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) {
6842 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6843 SDValue Mask = N->getMask();
6844 SDValue EVL = N->getVectorLength();
6845 ISD::LoadExtType ExtType = N->getExtensionType();
6846 SDLoc dl(N);
6847
6848 // The mask should be widened as well
6849 assert(getTypeAction(Mask.getValueType()) ==
6850 TargetLowering::TypeWidenVector &&
6851 "Unable to widen binary VP op");
6852 Mask = GetWidenedVector(Op: Mask);
6853 assert(Mask.getValueType().getVectorElementCount() ==
6854 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6855 .getVectorElementCount() &&
6856 "Unable to widen vector load");
6857
6858 SDValue Res =
6859 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType, VT: WidenVT, dl, Chain: N->getChain(),
6860 Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6861 MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), IsExpanding: N->isExpandingLoad());
6862 // Legalize the chain result - switch anything that used the old chain to
6863 // use the new one.
6864 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6865 return Res;
6866}
6867
6868SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N) {
6869 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6870 SDValue Mask = N->getMask();
6871 SDValue EVL = N->getVectorLength();
6872 SDLoc dl(N);
6873
6874 // The mask should be widened as well
6875 assert(getTypeAction(Mask.getValueType()) ==
6876 TargetLowering::TypeWidenVector &&
6877 "Unable to widen binary VP op");
6878 Mask = GetWidenedVector(Op: Mask);
6879 assert(Mask.getValueType().getVectorElementCount() ==
6880 TLI.getTypeToTransformTo(*DAG.getContext(), Mask.getValueType())
6881 .getVectorElementCount() &&
6882 "Unable to widen vector load");
6883
6884 SDValue Res = DAG.getLoadFFVP(VT: WidenVT, DL: dl, Chain: N->getChain(), Ptr: N->getBasePtr(),
6885 Mask, EVL, MMO: N->getMemOperand());
6886 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6887 ReplaceValueWith(From: SDValue(N, 2), To: Res.getValue(R: 2));
6888 return Res;
6889}
6890
6891SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) {
6892 SDLoc DL(N);
6893
6894 // The mask should be widened as well
6895 SDValue Mask = N->getMask();
6896 assert(getTypeAction(Mask.getValueType()) ==
6897 TargetLowering::TypeWidenVector &&
6898 "Unable to widen VP strided load");
6899 Mask = GetWidenedVector(Op: Mask);
6900
6901 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6902 assert(Mask.getValueType().getVectorElementCount() ==
6903 WidenVT.getVectorElementCount() &&
6904 "Data and mask vectors should have the same number of elements");
6905
6906 SDValue Res = DAG.getStridedLoadVP(
6907 AM: N->getAddressingMode(), ExtType: N->getExtensionType(), VT: WidenVT, DL, Chain: N->getChain(),
6908 Ptr: N->getBasePtr(), Offset: N->getOffset(), Stride: N->getStride(), Mask,
6909 EVL: N->getVectorLength(), MemVT: N->getMemoryVT(), MMO: N->getMemOperand(),
6910 IsExpanding: N->isExpandingLoad());
6911
6912 // Legalize the chain result - switch anything that used the old chain to
6913 // use the new one.
6914 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6915 return Res;
6916}
6917
6918SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) {
6919 SDValue Vec = N->getOperand(Num: 0);
6920 SDValue Mask = N->getOperand(Num: 1);
6921 SDValue Passthru = N->getOperand(Num: 2);
6922 EVT WideVecVT =
6923 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Vec.getValueType());
6924 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
6925 VT: Mask.getValueType().getVectorElementType(),
6926 EC: WideVecVT.getVectorElementCount());
6927
6928 SDValue WideVec = ModifyToType(InOp: Vec, NVT: WideVecVT);
6929 SDValue WideMask = ModifyToType(InOp: Mask, NVT: WideMaskVT, /*FillWithZeroes=*/true);
6930 SDValue WidePassthru = ModifyToType(InOp: Passthru, NVT: WideVecVT);
6931 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL: SDLoc(N), VT: WideVecVT, N1: WideVec,
6932 N2: WideMask, N3: WidePassthru);
6933}
6934
6935SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
6936 EVT VT = N->getValueType(ResNo: 0);
6937 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6938 SDValue Mask = N->getMask();
6939 EVT MaskVT = Mask.getValueType();
6940 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
6941 ISD::LoadExtType ExtType = N->getExtensionType();
6942 SDLoc dl(N);
6943
6944 EVT WideMaskVT =
6945 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
6946 EC: WidenVT.getVectorElementCount());
6947
6948 if (ExtType == ISD::NON_EXTLOAD && !N->isExpandingLoad() &&
6949 TLI.isOperationLegalOrCustom(Op: ISD::VP_LOAD, VT: WidenVT) &&
6950 TLI.isTypeLegal(VT: WideMaskVT) &&
6951 // If there is a passthru, we shouldn't use vp.load. However,
6952 // type legalizer will struggle on masked.load with
6953 // scalable vectors, so for scalable vectors, we still use vp.load
6954 // but manually merge the load result with the passthru using vp.select.
6955 (N->getPassThru()->isUndef() || VT.isScalableVector())) {
6956 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
6957 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
6958 EC: VT.getVectorElementCount());
6959 SDValue NewLoad =
6960 DAG.getLoadVP(AM: N->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: WidenVT, dl,
6961 Chain: N->getChain(), Ptr: N->getBasePtr(), Offset: N->getOffset(), Mask, EVL,
6962 MemVT: N->getMemoryVT(), MMO: N->getMemOperand());
6963 SDValue NewVal = NewLoad;
6964
6965 // Manually merge with vselect
6966 if (!N->getPassThru()->isUndef()) {
6967 assert(WidenVT.isScalableVector());
6968 NewVal = DAG.getNode(Opcode: ISD::VSELECT, DL: dl, VT: WidenVT, N1: Mask, N2: NewVal, N3: PassThru);
6969 // The lanes past EVL are poison.
6970 NewVal = DAG.getNode(Opcode: ISD::VP_MERGE, DL: dl, VT: WidenVT,
6971 N1: DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT), N2: NewVal,
6972 N3: DAG.getPOISON(VT: WidenVT), N4: EVL);
6973 }
6974
6975 // Modified the chain - switch anything that used the old chain to use
6976 // the new one.
6977 ReplaceValueWith(From: SDValue(N, 1), To: NewLoad.getValue(R: 1));
6978
6979 return NewVal;
6980 }
6981
6982 // The mask should be widened as well
6983 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
6984
6985 SDValue Res = DAG.getMaskedLoad(
6986 VT: WidenVT, dl, Chain: N->getChain(), Base: N->getBasePtr(), Offset: N->getOffset(), Mask,
6987 Src0: PassThru, MemVT: N->getMemoryVT(), MMO: N->getMemOperand(), AM: N->getAddressingMode(),
6988 ExtType, IsExpanding: N->isExpandingLoad());
6989 // Legalize the chain result - switch anything that used the old chain to
6990 // use the new one.
6991 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
6992 return Res;
6993}
6994
6995SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
6996
6997 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
6998 SDValue Mask = N->getMask();
6999 EVT MaskVT = Mask.getValueType();
7000 SDValue PassThru = GetWidenedVector(Op: N->getPassThru());
7001 SDValue Scale = N->getScale();
7002 ElementCount WideEC = WideVT.getVectorElementCount();
7003 SDLoc dl(N);
7004
7005 // The mask should be widened as well
7006 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7007 VT: MaskVT.getVectorElementType(), EC: WideEC);
7008 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
7009
7010 // Widen the Index operand
7011 SDValue Index = N->getIndex();
7012 EVT WideIndexVT = EVT::getVectorVT(
7013 Context&: *DAG.getContext(), VT: Index.getValueType().getScalarType(), EC: WideEC);
7014 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
7015 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
7016 Scale };
7017
7018 // Widen the MemoryType
7019 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7020 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7021 SDValue Res = DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other),
7022 MemVT: WideMemVT, dl, Ops, MMO: N->getMemOperand(),
7023 IndexType: N->getIndexType(), ExtTy: N->getExtensionType());
7024
7025 // Legalize the chain result - switch anything that used the old chain to
7026 // use the new one.
7027 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7028 return Res;
7029}
7030
7031SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
7032 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7033 SDValue Mask = N->getMask();
7034 SDValue Scale = N->getScale();
7035 ElementCount WideEC = WideVT.getVectorElementCount();
7036 SDLoc dl(N);
7037
7038 SDValue Index = GetWidenedVector(Op: N->getIndex());
7039 EVT WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7040 VT: N->getMemoryVT().getScalarType(), EC: WideEC);
7041 Mask = GetWidenedMask(Mask, EC: WideEC);
7042
7043 SDValue Ops[] = {N->getChain(), N->getBasePtr(), Index, Scale,
7044 Mask, N->getVectorLength()};
7045 SDValue Res = DAG.getGatherVP(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), VT: WideMemVT,
7046 dl, Ops, MMO: N->getMemOperand(), IndexType: N->getIndexType());
7047
7048 // Legalize the chain result - switch anything that used the old chain to
7049 // use the new one.
7050 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7051 return Res;
7052}
7053
7054SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
7055 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7056 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: WidenVT, Operand: N->getOperand(Num: 0));
7057}
7058
7059// Return true is this is a SETCC node or a strict version of it.
7060static inline bool isSETCCOp(unsigned Opcode) {
7061 switch (Opcode) {
7062 case ISD::SETCC:
7063 case ISD::STRICT_FSETCC:
7064 case ISD::STRICT_FSETCCS:
7065 return true;
7066 }
7067 return false;
7068}
7069
7070// Return true if this is a node that could have two SETCCs as operands.
7071static inline bool isLogicalMaskOp(unsigned Opcode) {
7072 switch (Opcode) {
7073 case ISD::AND:
7074 case ISD::OR:
7075 case ISD::XOR:
7076 return true;
7077 }
7078 return false;
7079}
7080
7081// If N is a SETCC or a strict variant of it, return the type
7082// of the compare operands.
7083static inline EVT getSETCCOperandType(SDValue N) {
7084 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
7085 return N->getOperand(Num: OpNo).getValueType();
7086}
7087
7088// This is used just for the assert in convertMask(). Check that this either
7089// a SETCC or a previously handled SETCC by convertMask().
7090#ifndef NDEBUG
7091static inline bool isSETCCorConvertedSETCC(SDValue N) {
7092 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
7093 N = N.getOperand(0);
7094 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
7095 for (unsigned i = 1; i < N->getNumOperands(); ++i)
7096 if (!N->getOperand(i)->isUndef())
7097 return false;
7098 N = N.getOperand(0);
7099 }
7100
7101 if (N.getOpcode() == ISD::TRUNCATE)
7102 N = N.getOperand(0);
7103 else if (N.getOpcode() == ISD::SIGN_EXTEND)
7104 N = N.getOperand(0);
7105
7106 if (isLogicalMaskOp(N.getOpcode()))
7107 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
7108 isSETCCorConvertedSETCC(N.getOperand(1));
7109
7110 return (isSETCCOp(N.getOpcode()) ||
7111 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
7112}
7113#endif
7114
7115// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
7116// to ToMaskVT if needed with vector extension or truncation.
7117SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
7118 EVT ToMaskVT) {
7119 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
7120 // FIXME: This code seems to be too restrictive, we might consider
7121 // generalizing it or dropping it.
7122 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
7123
7124 // Make a new Mask node, with a legal result VT.
7125 SDValue Mask;
7126 SmallVector<SDValue, 4> Ops;
7127 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
7128 Ops.push_back(Elt: InMask->getOperand(Num: i));
7129 if (InMask->isStrictFPOpcode()) {
7130 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask),
7131 ResultTys: { MaskVT, MVT::Other }, Ops);
7132 ReplaceValueWith(From: InMask.getValue(R: 1), To: Mask.getValue(R: 1));
7133 }
7134 else
7135 Mask = DAG.getNode(Opcode: InMask->getOpcode(), DL: SDLoc(InMask), VT: MaskVT, Ops,
7136 Flags: InMask->getFlags());
7137
7138 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
7139 // extend or truncate is needed.
7140 LLVMContext &Ctx = *DAG.getContext();
7141 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
7142 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
7143 if (MaskScalarBits < ToMaskScalBits) {
7144 EVT ExtVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7145 NumElements: MaskVT.getVectorNumElements());
7146 Mask = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Mask), VT: ExtVT, Operand: Mask);
7147 } else if (MaskScalarBits > ToMaskScalBits) {
7148 EVT TruncVT = EVT::getVectorVT(Context&: Ctx, VT: ToMaskVT.getVectorElementType(),
7149 NumElements: MaskVT.getVectorNumElements());
7150 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Mask), VT: TruncVT, Operand: Mask);
7151 }
7152
7153 assert(Mask->getValueType(0).getScalarSizeInBits() ==
7154 ToMaskVT.getScalarSizeInBits() &&
7155 "Mask should have the right element size by now.");
7156
7157 // Adjust Mask to the right number of elements.
7158 unsigned CurrMaskNumEls = Mask->getValueType(ResNo: 0).getVectorNumElements();
7159 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
7160 Mask = DAG.getExtractSubvector(DL: SDLoc(Mask), VT: ToMaskVT, Vec: Mask, Idx: 0);
7161 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
7162 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
7163 EVT SubVT = Mask->getValueType(ResNo: 0);
7164 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getPOISON(VT: SubVT));
7165 SubOps[0] = Mask;
7166 Mask = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Mask), VT: ToMaskVT, Ops: SubOps);
7167 }
7168
7169 assert((Mask->getValueType(0) == ToMaskVT) &&
7170 "A mask of ToMaskVT should have been produced by now.");
7171
7172 return Mask;
7173}
7174
7175// This method tries to handle some special cases for the vselect mask
7176// and if needed adjusting the mask vector type to match that of the VSELECT.
7177// Without it, many cases end up with scalarization of the SETCC, with many
7178// unnecessary instructions.
7179SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
7180 LLVMContext &Ctx = *DAG.getContext();
7181 SDValue Cond = N->getOperand(Num: 0);
7182
7183 if (N->getOpcode() != ISD::VSELECT)
7184 return SDValue();
7185
7186 if (!isSETCCOp(Opcode: Cond->getOpcode()) && !isLogicalMaskOp(Opcode: Cond->getOpcode()))
7187 return SDValue();
7188
7189 // If this is a splitted VSELECT that was previously already handled, do
7190 // nothing.
7191 EVT CondVT = Cond->getValueType(ResNo: 0);
7192 if (CondVT.getScalarSizeInBits() != 1)
7193 return SDValue();
7194
7195 EVT VSelVT = N->getValueType(ResNo: 0);
7196
7197 // This method can't handle scalable vector types.
7198 // FIXME: This support could be added in the future.
7199 if (VSelVT.isScalableVector())
7200 return SDValue();
7201
7202 // Only handle vector types which are a power of 2.
7203 if (!isPowerOf2_64(Value: VSelVT.getSizeInBits()))
7204 return SDValue();
7205
7206 // Don't touch if this will be scalarized.
7207 EVT FinalVT = VSelVT;
7208 while (getTypeAction(VT: FinalVT) == TargetLowering::TypeSplitVector)
7209 FinalVT = FinalVT.getHalfNumVectorElementsVT(Context&: Ctx);
7210
7211 if (FinalVT.getVectorNumElements() == 1)
7212 return SDValue();
7213
7214 // If there is support for an i1 vector mask, don't touch.
7215 if (isSETCCOp(Opcode: Cond.getOpcode())) {
7216 EVT SetCCOpVT = getSETCCOperandType(N: Cond);
7217 while (TLI.getTypeAction(Context&: Ctx, VT: SetCCOpVT) != TargetLowering::TypeLegal)
7218 SetCCOpVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: SetCCOpVT);
7219 EVT SetCCResVT = getSetCCResultType(VT: SetCCOpVT);
7220 if (SetCCResVT.getScalarSizeInBits() == 1)
7221 return SDValue();
7222 } else if (CondVT.getScalarType() == MVT::i1) {
7223 // If there is support for an i1 vector mask (or only scalar i1 conditions),
7224 // don't touch.
7225 while (TLI.getTypeAction(Context&: Ctx, VT: CondVT) != TargetLowering::TypeLegal)
7226 CondVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: CondVT);
7227
7228 if (CondVT.getScalarType() == MVT::i1)
7229 return SDValue();
7230 }
7231
7232 // Widen the vselect result type if needed.
7233 if (getTypeAction(VT: VSelVT) == TargetLowering::TypeWidenVector)
7234 VSelVT = TLI.getTypeToTransformTo(Context&: Ctx, VT: VSelVT);
7235
7236 // The mask of the VSELECT should have integer elements.
7237 EVT ToMaskVT = VSelVT;
7238 if (!ToMaskVT.getScalarType().isInteger())
7239 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
7240
7241 SDValue Mask;
7242 if (isSETCCOp(Opcode: Cond->getOpcode())) {
7243 EVT MaskVT = getSetCCResultType(VT: getSETCCOperandType(N: Cond));
7244 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7245 } else if (isLogicalMaskOp(Opcode: Cond->getOpcode()) &&
7246 isSETCCOp(Opcode: Cond->getOperand(Num: 0).getOpcode()) &&
7247 isSETCCOp(Opcode: Cond->getOperand(Num: 1).getOpcode())) {
7248 // Cond is (AND/OR/XOR (SETCC, SETCC))
7249 SDValue SETCC0 = Cond->getOperand(Num: 0);
7250 SDValue SETCC1 = Cond->getOperand(Num: 1);
7251 EVT VT0 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC0));
7252 EVT VT1 = getSetCCResultType(VT: getSETCCOperandType(N: SETCC1));
7253 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
7254 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
7255 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
7256 EVT MaskVT;
7257 // If the two SETCCs have different VTs, either extend/truncate one of
7258 // them to the other "towards" ToMaskVT, or truncate one and extend the
7259 // other to ToMaskVT.
7260 if (ScalarBits0 != ScalarBits1) {
7261 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
7262 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
7263 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
7264 MaskVT = WideVT;
7265 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
7266 MaskVT = NarrowVT;
7267 else
7268 MaskVT = ToMaskVT;
7269 } else
7270 // If the two SETCCs have the same VT, don't change it.
7271 MaskVT = VT0;
7272
7273 // Make new SETCCs and logical nodes.
7274 SETCC0 = convertMask(InMask: SETCC0, MaskVT: VT0, ToMaskVT: MaskVT);
7275 SETCC1 = convertMask(InMask: SETCC1, MaskVT: VT1, ToMaskVT: MaskVT);
7276 Cond = DAG.getNode(Opcode: Cond->getOpcode(), DL: SDLoc(Cond), VT: MaskVT, N1: SETCC0, N2: SETCC1);
7277
7278 // Convert the logical op for VSELECT if needed.
7279 Mask = convertMask(InMask: Cond, MaskVT, ToMaskVT);
7280 } else
7281 return SDValue();
7282
7283 return Mask;
7284}
7285
7286SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) {
7287 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7288 ElementCount WidenEC = WidenVT.getVectorElementCount();
7289
7290 SDValue Cond1 = N->getOperand(Num: 0);
7291 EVT CondVT = Cond1.getValueType();
7292 unsigned Opcode = N->getOpcode();
7293 if (CondVT.isVector()) {
7294 if (SDValue WideCond = WidenVSELECTMask(N)) {
7295 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7296 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7297 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7298 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: WideCond, N2: InOp1, N3: InOp2);
7299 }
7300
7301 EVT CondEltVT = CondVT.getVectorElementType();
7302 EVT CondWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: CondEltVT, EC: WidenEC);
7303 if (getTypeAction(VT: CondVT) == TargetLowering::TypeWidenVector)
7304 Cond1 = GetWidenedVector(Op: Cond1);
7305
7306 // If we have to split the condition there is no point in widening the
7307 // select. This would result in an cycle of widening the select ->
7308 // widening the condition operand -> splitting the condition operand ->
7309 // splitting the select -> widening the select. Instead split this select
7310 // further and widen the resulting type.
7311 if (getTypeAction(VT: CondVT) == TargetLowering::TypeSplitVector) {
7312 SDValue SplitSelect = SplitVecOp_VSELECT(N, OpNo: 0);
7313 SDValue Res = ModifyToType(InOp: SplitSelect, NVT: WidenVT);
7314 return Res;
7315 }
7316
7317 if (Cond1.getValueType() != CondWidenVT)
7318 Cond1 = ModifyToType(InOp: Cond1, NVT: CondWidenVT);
7319 }
7320
7321 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
7322 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 2));
7323 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
7324 if (Opcode == ISD::VP_MERGE)
7325 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2,
7326 N4: N->getOperand(Num: 3));
7327 return DAG.getNode(Opcode, DL: SDLoc(N), VT: WidenVT, N1: Cond1, N2: InOp1, N3: InOp2);
7328}
7329
7330SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
7331 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 2));
7332 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 3));
7333 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
7334 VT: InOp1.getValueType(), N1: N->getOperand(Num: 0),
7335 N2: N->getOperand(Num: 1), N3: InOp1, N4: InOp2, N5: N->getOperand(Num: 4));
7336}
7337
7338SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
7339 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7340 return DAG.getUNDEF(VT: WidenVT);
7341}
7342
7343SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
7344 EVT VT = N->getValueType(ResNo: 0);
7345 SDLoc dl(N);
7346
7347 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7348 unsigned NumElts = VT.getVectorNumElements();
7349 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7350
7351 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 0));
7352 SDValue InOp2 = GetWidenedVector(Op: N->getOperand(Num: 1));
7353
7354 // Adjust mask based on new input vector length.
7355 SmallVector<int, 16> NewMask(WidenNumElts, -1);
7356 for (unsigned i = 0; i != NumElts; ++i) {
7357 int Idx = N->getMaskElt(Idx: i);
7358 if (Idx < (int)NumElts)
7359 NewMask[i] = Idx;
7360 else
7361 NewMask[i] = Idx - NumElts + WidenNumElts;
7362 }
7363 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: InOp1, N2: InOp2, Mask: NewMask);
7364}
7365
7366SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) {
7367 EVT VT = N->getValueType(ResNo: 0);
7368 EVT EltVT = VT.getVectorElementType();
7369 SDLoc dl(N);
7370
7371 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7372 SDValue OpValue = GetWidenedVector(Op: N->getOperand(Num: 0));
7373 assert(WidenVT == OpValue.getValueType() && "Unexpected widened vector type");
7374
7375 SDValue ReverseVal = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL: dl, VT: WidenVT, Operand: OpValue);
7376 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
7377 unsigned VTNumElts = VT.getVectorMinNumElements();
7378 unsigned IdxVal = WidenNumElts - VTNumElts;
7379
7380 if (VT.isScalableVector()) {
7381 // Try to split the 'Widen ReverseVal' into smaller extracts and concat the
7382 // results together, e.g.(nxv6i64 -> nxv8i64)
7383 // nxv8i64 vector_reverse
7384 // <->
7385 // nxv8i64 concat(
7386 // nxv2i64 extract_subvector(nxv8i64, 2)
7387 // nxv2i64 extract_subvector(nxv8i64, 4)
7388 // nxv2i64 extract_subvector(nxv8i64, 6)
7389 // nxv2i64 undef)
7390
7391 unsigned GCD = std::gcd(m: VTNumElts, n: WidenNumElts);
7392 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7393 EC: ElementCount::getScalable(MinVal: GCD));
7394 assert((IdxVal % GCD) == 0 && "Expected Idx to be a multiple of the broken "
7395 "down type's element count");
7396 SmallVector<SDValue> Parts;
7397 unsigned i = 0;
7398 for (; i < VTNumElts / GCD; ++i)
7399 Parts.push_back(
7400 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: ReverseVal, Idx: IdxVal + i * GCD));
7401 for (; i < WidenNumElts / GCD; ++i)
7402 Parts.push_back(Elt: DAG.getPOISON(VT: PartVT));
7403
7404 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: Parts);
7405 }
7406
7407 // Use VECTOR_SHUFFLE to combine new vector from 'ReverseVal' for
7408 // fixed-vectors.
7409 SmallVector<int, 16> Mask(WidenNumElts, -1);
7410 std::iota(first: Mask.begin(), last: Mask.begin() + VTNumElts, value: IdxVal);
7411
7412 return DAG.getVectorShuffle(VT: WidenVT, dl, N1: ReverseVal, N2: DAG.getPOISON(VT: WidenVT),
7413 Mask);
7414}
7415
7416SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
7417 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7418 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL: SDLoc(N), VT: NVT, Ops: N->ops());
7419}
7420
7421void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) {
7422 EVT VT = N->getValueType(ResNo: 0);
7423 EVT EltVT = VT.getVectorElementType();
7424 ElementCount OrigEC = VT.getVectorElementCount();
7425 unsigned Factor = N->getNumOperands();
7426 SDLoc DL(N);
7427
7428 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7429 ElementCount WidenEC = WidenVT.getVectorElementCount();
7430
7431 SmallVector<SDValue, 8> WidenOps(Factor);
7432 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7433 WidenOps[Idx] = GetWidenedVector(Op: N->getOperand(Num: Idx));
7434
7435 SmallVector<EVT, 8> WidenVTs(Factor, WidenVT);
7436 SDValue Interleaved =
7437 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: WidenVTs, Ops: WidenOps);
7438
7439 EVT PackedWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7440 EC: WidenEC.multiplyCoefficientBy(RHS: Factor));
7441 SmallVector<SDValue, 8> Slices(Factor);
7442 for (unsigned Idx = 0; Idx != Factor; ++Idx)
7443 Slices[Idx] = Interleaved.getValue(R: Idx);
7444
7445 SDValue Packed = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PackedWidenVT, Ops: Slices);
7446
7447 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7448 SDValue Narrow = DAG.getExtractSubvector(
7449 DL, VT, Vec: Packed, Idx: OrigEC.multiplyCoefficientBy(RHS: Idx).getKnownMinValue());
7450 SDValue Wide =
7451 DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: WidenVT), SubVec: Narrow, /*Idx=*/0U);
7452 SetWidenedVector(Op: SDValue(N, Idx), Result: Wide);
7453 }
7454}
7455
7456SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_MATCH(SDNode *N) {
7457 SDLoc DL(N);
7458 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7459 EVT SourceVT = N->getOperand(Num: 0).getValueType();
7460 EVT WideSourceVT =
7461 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SourceVT.getVectorElementType(),
7462 EC: WidenVT.getVectorElementCount());
7463
7464 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
7465 SubVec: N->getOperand(Num: 0), Idx: 0);
7466 SDValue WideMask = DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT),
7467 SubVec: N->getOperand(Num: 2), Idx: 0);
7468 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
7469 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
7470}
7471
7472void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) {
7473 EVT VT = N->getValueType(ResNo: 0);
7474 EVT EltVT = VT.getVectorElementType();
7475 ElementCount OrigEC = VT.getVectorElementCount();
7476 unsigned Factor = N->getNumOperands();
7477 SDLoc DL(N);
7478
7479 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7480 ElementCount WidenEC = WidenVT.getVectorElementCount();
7481 // We cannot just use the widened operands directly: since they might be
7482 // individually widened, using them directly will result in de-interleaving
7483 // the "padded" lanes that sit in the middle of the vector. Instead, we should
7484 // not concat the widened operands but the original ones to effectively
7485 // generate a "packed" concated and widened vector, before extracting new
7486 // operand vectors with the widened type.
7487 EVT PackedWidenVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7488 EC: WidenEC.multiplyCoefficientBy(RHS: Factor));
7489 EVT ConcatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7490 EC: OrigEC.multiplyCoefficientBy(RHS: Factor));
7491 SDValue ConcatOp = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops: N->ops());
7492 SDValue PackedWidenVec = DAG.getInsertSubvector(
7493 DL, Vec: DAG.getUNDEF(VT: PackedWidenVT), SubVec: ConcatOp, /*Idx=*/0U);
7494
7495 // Extract the new widened operand vectors.
7496 SmallVector<SDValue, 8> NewOps(Factor, SDValue());
7497 for (unsigned Idx = 0U; Idx < Factor; ++Idx) {
7498 NewOps[Idx] = DAG.getExtractSubvector(
7499 DL, VT: WidenVT, Vec: PackedWidenVec,
7500 Idx: WidenEC.multiplyCoefficientBy(RHS: Idx).getKnownMinValue());
7501 }
7502
7503 SmallVector<EVT, 8> NewVTs(Factor, WidenVT);
7504 SDValue NewRes = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: NewVTs, Ops: NewOps);
7505 // Set the widened results manually.
7506 for (unsigned Idx = 0U; Idx < Factor; ++Idx)
7507 SetWidenedVector(Op: SDValue(N, Idx), Result: NewRes.getValue(R: Idx));
7508}
7509
7510SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
7511 assert(N->getValueType(0).isVector() &&
7512 N->getOperand(0).getValueType().isVector() &&
7513 "Operands must be vectors");
7514 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
7515 ElementCount WidenEC = WidenVT.getVectorElementCount();
7516
7517 SDValue InOp1 = N->getOperand(Num: 0);
7518 EVT InVT = InOp1.getValueType();
7519 assert(InVT.isVector() && "can not widen non-vector type");
7520 EVT WidenInVT =
7521 EVT::getVectorVT(Context&: *DAG.getContext(), VT: InVT.getVectorElementType(), EC: WidenEC);
7522
7523 // The input and output types often differ here, and it could be that while
7524 // we'd prefer to widen the result type, the input operands have been split.
7525 // In this case, we also need to split the result of this node as well.
7526 if (getTypeAction(VT: InVT) == TargetLowering::TypeSplitVector) {
7527 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
7528 SDValue Res = ModifyToType(InOp: SplitVSetCC, NVT: WidenVT);
7529 return Res;
7530 }
7531
7532 // If the inputs also widen, handle them directly. Otherwise widen by hand.
7533 SDValue InOp2 = N->getOperand(Num: 1);
7534 if (getTypeAction(VT: InVT) == TargetLowering::TypeWidenVector) {
7535 InOp1 = GetWidenedVector(Op: InOp1);
7536 InOp2 = GetWidenedVector(Op: InOp2);
7537 } else {
7538 SDValue Poison = DAG.getPOISON(VT: WidenInVT);
7539 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
7540 InOp1 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7541 N2: InOp1, N3: ZeroIdx);
7542 InOp2 = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT: WidenInVT, N1: Poison,
7543 N2: InOp2, N3: ZeroIdx);
7544 }
7545
7546 // Assume that the input and output will be widen appropriately. If not,
7547 // we will have to unroll it at some point.
7548 assert(InOp1.getValueType() == WidenInVT &&
7549 InOp2.getValueType() == WidenInVT &&
7550 "Input not widened to expected type!");
7551 (void)WidenInVT;
7552 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: WidenVT, N1: InOp1, N2: InOp2,
7553 N3: N->getOperand(Num: 2));
7554}
7555
7556SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
7557 assert(N->getValueType(0).isVector() &&
7558 N->getOperand(1).getValueType().isVector() &&
7559 "Operands must be vectors");
7560 EVT VT = N->getValueType(ResNo: 0);
7561 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7562 unsigned WidenNumElts = WidenVT.getVectorNumElements();
7563 unsigned NumElts = VT.getVectorNumElements();
7564 EVT EltVT = VT.getVectorElementType();
7565
7566 SDLoc dl(N);
7567 SDValue Chain = N->getOperand(Num: 0);
7568 SDValue LHS = N->getOperand(Num: 1);
7569 SDValue RHS = N->getOperand(Num: 2);
7570 SDValue CC = N->getOperand(Num: 3);
7571 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
7572
7573 // Fully unroll and reassemble.
7574 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getPOISON(VT: EltVT));
7575 SmallVector<SDValue, 8> Chains(NumElts);
7576 for (unsigned i = 0; i != NumElts; ++i) {
7577 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
7578 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
7579
7580 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
7581 Ops: {Chain, LHSElem, RHSElem, CC});
7582 Chains[i] = Scalars[i].getValue(R: 1);
7583 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
7584 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
7585 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
7586 }
7587
7588 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
7589 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7590
7591 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops: Scalars);
7592}
7593
7594//===----------------------------------------------------------------------===//
7595// Widen Vector Operand
7596//===----------------------------------------------------------------------===//
7597bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
7598 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG));
7599 SDValue Res = SDValue();
7600
7601 // See if the target wants to custom widen this node.
7602 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
7603 return false;
7604
7605 switch (N->getOpcode()) {
7606 default:
7607#ifndef NDEBUG
7608 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
7609 N->dump(&DAG);
7610 dbgs() << "\n";
7611#endif
7612 report_fatal_error(reason: "Do not know how to widen this operator's operand!");
7613
7614 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
7615 case ISD::FAKE_USE:
7616 Res = WidenVecOp_FAKE_USE(N);
7617 break;
7618 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
7619 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
7620 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
7621 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
7622 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
7623 case ISD::ATOMIC_STORE:
7624 Res = WidenVecOp_ATOMIC_STORE(ST: cast<AtomicSDNode>(Val: N));
7625 break;
7626 case ISD::VP_STORE: Res = WidenVecOp_VP_STORE(N, OpNo); break;
7627 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7628 Res = WidenVecOp_VP_STRIDED_STORE(N, OpNo);
7629 break;
7630 case ISD::ANY_EXTEND_VECTOR_INREG:
7631 case ISD::SIGN_EXTEND_VECTOR_INREG:
7632 case ISD::ZERO_EXTEND_VECTOR_INREG:
7633 Res = WidenVecOp_EXTEND_VECTOR_INREG(N);
7634 break;
7635 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
7636 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
7637 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
7638 case ISD::VP_SCATTER: Res = WidenVecOp_VP_SCATTER(N, OpNo); break;
7639 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
7640 case ISD::STRICT_FSETCC:
7641 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
7642 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
7643 case ISD::FLDEXP:
7644 case ISD::FCOPYSIGN:
7645 case ISD::LROUND:
7646 case ISD::LLROUND:
7647 case ISD::LRINT:
7648 case ISD::LLRINT:
7649 Res = WidenVecOp_UnrollVectorOp(N);
7650 break;
7651 case ISD::IS_FPCLASS: Res = WidenVecOp_IS_FPCLASS(N); break;
7652
7653 case ISD::ANY_EXTEND:
7654 case ISD::SIGN_EXTEND:
7655 case ISD::ZERO_EXTEND:
7656 Res = WidenVecOp_EXTEND(N);
7657 break;
7658
7659 case ISD::SCMP:
7660 case ISD::UCMP:
7661 Res = WidenVecOp_CMP(N);
7662 break;
7663
7664 case ISD::FP_EXTEND:
7665 case ISD::STRICT_FP_EXTEND:
7666 case ISD::FP_ROUND:
7667 case ISD::STRICT_FP_ROUND:
7668 case ISD::FP_TO_SINT:
7669 case ISD::STRICT_FP_TO_SINT:
7670 case ISD::FP_TO_UINT:
7671 case ISD::STRICT_FP_TO_UINT:
7672 case ISD::SINT_TO_FP:
7673 case ISD::STRICT_SINT_TO_FP:
7674 case ISD::UINT_TO_FP:
7675 case ISD::STRICT_UINT_TO_FP:
7676 case ISD::TRUNCATE:
7677 case ISD::CONVERT_FROM_ARBITRARY_FP:
7678 case ISD::CONVERT_TO_ARBITRARY_FP:
7679 Res = WidenVecOp_Convert(N);
7680 break;
7681
7682 case ISD::FP_TO_SINT_SAT:
7683 case ISD::FP_TO_UINT_SAT:
7684 Res = WidenVecOp_FP_TO_XINT_SAT(N);
7685 break;
7686
7687 case ISD::VECREDUCE_FADD:
7688 case ISD::VECREDUCE_FMUL:
7689 case ISD::VECREDUCE_ADD:
7690 case ISD::VECREDUCE_MUL:
7691 case ISD::VECREDUCE_AND:
7692 case ISD::VECREDUCE_OR:
7693 case ISD::VECREDUCE_XOR:
7694 case ISD::VECREDUCE_SMAX:
7695 case ISD::VECREDUCE_SMIN:
7696 case ISD::VECREDUCE_UMAX:
7697 case ISD::VECREDUCE_UMIN:
7698 case ISD::VECREDUCE_FMAX:
7699 case ISD::VECREDUCE_FMIN:
7700 case ISD::VECREDUCE_FMAXIMUM:
7701 case ISD::VECREDUCE_FMINIMUM:
7702 case ISD::VECREDUCE_FMAXIMUMNUM:
7703 case ISD::VECREDUCE_FMINIMUMNUM:
7704 Res = WidenVecOp_VECREDUCE(N);
7705 break;
7706 case ISD::VECREDUCE_SEQ_FADD:
7707 case ISD::VECREDUCE_SEQ_FMUL:
7708 Res = WidenVecOp_VECREDUCE_SEQ(N);
7709 break;
7710 case ISD::VP_REDUCE_FADD:
7711 case ISD::VP_REDUCE_SEQ_FADD:
7712 case ISD::VP_REDUCE_FMUL:
7713 case ISD::VP_REDUCE_SEQ_FMUL:
7714 case ISD::VP_REDUCE_ADD:
7715 case ISD::VP_REDUCE_MUL:
7716 case ISD::VP_REDUCE_AND:
7717 case ISD::VP_REDUCE_OR:
7718 case ISD::VP_REDUCE_XOR:
7719 case ISD::VP_REDUCE_SMAX:
7720 case ISD::VP_REDUCE_SMIN:
7721 case ISD::VP_REDUCE_UMAX:
7722 case ISD::VP_REDUCE_UMIN:
7723 case ISD::VP_REDUCE_FMAX:
7724 case ISD::VP_REDUCE_FMIN:
7725 case ISD::VP_REDUCE_FMAXIMUM:
7726 case ISD::VP_REDUCE_FMINIMUM:
7727 Res = WidenVecOp_VP_REDUCE(N);
7728 break;
7729 case ISD::CTTZ_ELTS:
7730 case ISD::CTTZ_ELTS_ZERO_POISON:
7731 Res = WidenVecOp_CttzElements(N);
7732 break;
7733 case ISD::VP_CTTZ_ELTS:
7734 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
7735 Res = WidenVecOp_VP_CttzElements(N);
7736 break;
7737 case ISD::VECTOR_FIND_LAST_ACTIVE:
7738 Res = WidenVecOp_VECTOR_FIND_LAST_ACTIVE(N);
7739 break;
7740 case ISD::VECTOR_MATCH:
7741 Res = WidenVecOp_VECTOR_MATCH(N, OpNo);
7742 break;
7743 }
7744
7745 // If Res is null, the sub-method took care of registering the result.
7746 if (!Res.getNode()) return false;
7747
7748 // If the result is N, the sub-method updated N in place. Tell the legalizer
7749 // core about this.
7750 if (Res.getNode() == N)
7751 return true;
7752
7753
7754 if (N->isStrictFPOpcode())
7755 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
7756 "Invalid operand expansion");
7757 else
7758 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
7759 "Invalid operand expansion");
7760
7761 ReplaceValueWith(From: SDValue(N, 0), To: Res);
7762 return false;
7763}
7764
7765SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
7766 SDLoc DL(N);
7767 EVT VT = N->getValueType(ResNo: 0);
7768
7769 SDValue InOp = N->getOperand(Num: 0);
7770 assert(getTypeAction(InOp.getValueType()) ==
7771 TargetLowering::TypeWidenVector &&
7772 "Unexpected type action");
7773 InOp = GetWidenedVector(Op: InOp);
7774 assert(VT.getVectorNumElements() <
7775 InOp.getValueType().getVectorNumElements() &&
7776 "Input wasn't widened!");
7777
7778 // We may need to further widen the operand until it has the same total
7779 // vector size as the result.
7780 EVT InVT = InOp.getValueType();
7781 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
7782 EVT InEltVT = InVT.getVectorElementType();
7783 for (EVT FixedVT : MVT::vector_valuetypes()) {
7784 EVT FixedEltVT = FixedVT.getVectorElementType();
7785 if (TLI.isTypeLegal(VT: FixedVT) &&
7786 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
7787 FixedEltVT == InEltVT) {
7788 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
7789 "Not enough elements in the fixed type for the operand!");
7790 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
7791 "We can't have the same type as we started with!");
7792 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
7793 InOp = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: FixedVT), SubVec: InOp, Idx: 0);
7794 else
7795 InOp = DAG.getExtractSubvector(DL, VT: FixedVT, Vec: InOp, Idx: 0);
7796 break;
7797 }
7798 }
7799 InVT = InOp.getValueType();
7800 if (InVT.getSizeInBits() != VT.getSizeInBits())
7801 // We couldn't find a legal vector type that was a widening of the input
7802 // and could be extended in-register to the result type, so we have to
7803 // scalarize.
7804 return WidenVecOp_Convert(N);
7805 }
7806
7807 // Use special DAG nodes to represent the operation of extending the
7808 // low lanes.
7809 switch (N->getOpcode()) {
7810 default:
7811 llvm_unreachable("Extend legalization on extend operation!");
7812 case ISD::ANY_EXTEND:
7813 return DAG.getNode(Opcode: ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7814 case ISD::SIGN_EXTEND:
7815 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7816 case ISD::ZERO_EXTEND:
7817 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: InOp);
7818 }
7819}
7820
7821SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) {
7822 SDLoc dl(N);
7823
7824 EVT OpVT = N->getOperand(Num: 0).getValueType();
7825 EVT ResVT = N->getValueType(ResNo: 0);
7826 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 0));
7827 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 1));
7828
7829 // 1. EXTRACT_SUBVECTOR
7830 // 2. SIGN_EXTEND/ZERO_EXTEND
7831 // 3. CMP
7832 LHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: LHS, Idx: 0);
7833 RHS = DAG.getExtractSubvector(DL: dl, VT: OpVT, Vec: RHS, Idx: 0);
7834
7835 // At this point the result type is guaranteed to be valid, so we can use it
7836 // as the operand type by extending it appropriately
7837 ISD::NodeType ExtendOpcode =
7838 N->getOpcode() == ISD::SCMP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7839 LHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: LHS);
7840 RHS = DAG.getNode(Opcode: ExtendOpcode, DL: dl, VT: ResVT, Operand: RHS);
7841
7842 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: ResVT, N1: LHS, N2: RHS);
7843}
7844
7845SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) {
7846 // The result (and first input) is legal, but the second input is illegal.
7847 // We can't do much to fix that, so just unroll and let the extracts off of
7848 // the second input be widened as needed later.
7849 return DAG.UnrollVectorOp(N);
7850}
7851
7852SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) {
7853 SDLoc DL(N);
7854 EVT ResultVT = N->getValueType(ResNo: 0);
7855 SDValue Test = N->getOperand(Num: 1);
7856 SDValue WideArg = GetWidenedVector(Op: N->getOperand(Num: 0));
7857
7858 // Process this node similarly to SETCC.
7859 EVT WideResultVT = getSetCCResultType(VT: WideArg.getValueType());
7860 if (ResultVT.getScalarType() == MVT::i1)
7861 WideResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
7862 NumElements: WideResultVT.getVectorNumElements());
7863
7864 SDValue WideNode = DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: WideResultVT,
7865 Ops: {WideArg, Test}, Flags: N->getFlags());
7866
7867 // Extract the needed results from the result vector.
7868 EVT ResVT =
7869 EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideResultVT.getVectorElementType(),
7870 NumElements: ResultVT.getVectorNumElements());
7871 SDValue CC = DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideNode, Idx: 0);
7872
7873 EVT OpVT = N->getOperand(Num: 0).getValueType();
7874 ISD::NodeType ExtendCode =
7875 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
7876 return DAG.getNode(Opcode: ExtendCode, DL, VT: ResultVT, Operand: CC);
7877}
7878
7879SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
7880 // Since the result is legal and the input is illegal.
7881 EVT VT = N->getValueType(ResNo: 0);
7882 EVT EltVT = VT.getVectorElementType();
7883 SDLoc dl(N);
7884 SDValue InOp = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0);
7885 assert(getTypeAction(InOp.getValueType()) ==
7886 TargetLowering::TypeWidenVector &&
7887 "Unexpected type action");
7888 InOp = GetWidenedVector(Op: InOp);
7889 EVT InVT = InOp.getValueType();
7890 unsigned Opcode = N->getOpcode();
7891
7892 // Helper to build a convert node with all scalar trailing operands.
7893 auto MakeConvertNode = [&](EVT VT, SDValue Op) -> SDValue {
7894 if (Opcode == ISD::CONVERT_TO_ARBITRARY_FP)
7895 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
7896 N4: N->getOperand(Num: 3));
7897 if (Opcode == ISD::FP_ROUND || Opcode == ISD::CONVERT_FROM_ARBITRARY_FP)
7898 return DAG.getNode(Opcode, DL: dl, VT, N1: Op, N2: N->getOperand(Num: 1));
7899 return DAG.getNode(Opcode, DL: dl, VT, Operand: Op);
7900 };
7901
7902 // See if a widened result type would be legal, if so widen the node.
7903 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
7904 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
7905 EC: InVT.getVectorElementCount());
7906 if (TLI.isTypeLegal(VT: WideVT) && !N->isStrictFPOpcode()) {
7907 SDValue Res;
7908 if (N->isStrictFPOpcode()) {
7909 if (Opcode == ISD::STRICT_FP_ROUND)
7910 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7911 Ops: { N->getOperand(Num: 0), InOp, N->getOperand(Num: 2) });
7912 else
7913 Res = DAG.getNode(Opcode, DL: dl, ResultTys: { WideVT, MVT::Other },
7914 Ops: { N->getOperand(Num: 0), InOp });
7915 // Legalize the chain result - switch anything that used the old chain to
7916 // use the new one.
7917 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
7918 } else {
7919 Res = MakeConvertNode(WideVT, InOp);
7920 }
7921 return DAG.getExtractSubvector(DL: dl, VT, Vec: Res, Idx: 0);
7922 }
7923
7924 EVT InEltVT = InVT.getVectorElementType();
7925
7926 // Unroll the convert into some scalar code and create a nasty build vector.
7927 unsigned NumElts = VT.getVectorNumElements();
7928 SmallVector<SDValue, 16> Ops(NumElts);
7929 if (N->isStrictFPOpcode()) {
7930 SmallVector<SDValue, 4> NewOps(N->ops());
7931 SmallVector<SDValue, 32> OpChains;
7932 for (unsigned i=0; i < NumElts; ++i) {
7933 NewOps[1] = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7934 Ops[i] = DAG.getNode(Opcode, DL: dl, ResultTys: { EltVT, MVT::Other }, Ops: NewOps);
7935 OpChains.push_back(Elt: Ops[i].getValue(R: 1));
7936 }
7937 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OpChains);
7938 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
7939 } else {
7940 for (unsigned i = 0; i < NumElts; ++i) {
7941 SDValue Elt = DAG.getExtractVectorElt(DL: dl, VT: InEltVT, Vec: InOp, Idx: i);
7942 Ops[i] = MakeConvertNode(EltVT, Elt);
7943 }
7944 }
7945
7946 return DAG.getBuildVector(VT, DL: dl, Ops);
7947}
7948
7949SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
7950 EVT DstVT = N->getValueType(ResNo: 0);
7951 SDValue Src = GetWidenedVector(Op: N->getOperand(Num: 0));
7952 EVT SrcVT = Src.getValueType();
7953 ElementCount WideNumElts = SrcVT.getVectorElementCount();
7954 SDLoc dl(N);
7955
7956 // See if a widened result type would be legal, if so widen the node.
7957 EVT WideDstVT = EVT::getVectorVT(Context&: *DAG.getContext(),
7958 VT: DstVT.getVectorElementType(), EC: WideNumElts);
7959 if (TLI.isTypeLegal(VT: WideDstVT)) {
7960 SDValue Res =
7961 DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: WideDstVT, N1: Src, N2: N->getOperand(Num: 1));
7962 return DAG.getNode(
7963 Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: DstVT, N1: Res,
7964 N2: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getVectorIdxTy(DL: DAG.getDataLayout())));
7965 }
7966
7967 // Give up and unroll.
7968 return DAG.UnrollVectorOp(N);
7969}
7970
7971SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
7972 EVT VT = N->getValueType(ResNo: 0);
7973 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
7974 EVT InWidenVT = InOp.getValueType();
7975 SDLoc dl(N);
7976
7977 // Check if we can convert between two legal vector types and extract.
7978 TypeSize InWidenSize = InWidenVT.getSizeInBits();
7979 TypeSize Size = VT.getSizeInBits();
7980 // x86mmx is not an acceptable vector element type, so don't try.
7981 if (!VT.isVector() && VT != MVT::x86mmx &&
7982 InWidenSize.hasKnownScalarFactor(RHS: Size)) {
7983 unsigned NewNumElts = InWidenSize.getKnownScalarFactor(RHS: Size);
7984 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NewNumElts);
7985 if (TLI.isTypeLegal(VT: NewVT)) {
7986 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
7987 return DAG.getExtractVectorElt(DL: dl, VT, Vec: BitOp, Idx: 0);
7988 }
7989 }
7990
7991 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
7992 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
7993 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
7994 // having to copy via memory.
7995 if (VT.isVector()) {
7996 EVT EltVT = VT.getVectorElementType();
7997 unsigned EltSize = EltVT.getFixedSizeInBits();
7998 if (InWidenSize.isKnownMultipleOf(RHS: EltSize)) {
7999 ElementCount NewNumElts =
8000 (InWidenVT.getVectorElementCount() * InWidenVT.getScalarSizeInBits())
8001 .divideCoefficientBy(RHS: EltSize);
8002 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, EC: NewNumElts);
8003 if (TLI.isTypeLegal(VT: NewVT)) {
8004 SDValue BitOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: InOp);
8005 return DAG.getExtractSubvector(DL: dl, VT, Vec: BitOp, Idx: 0);
8006 }
8007 }
8008 }
8009
8010 return CreateStackStoreLoad(Op: InOp, DestVT: VT);
8011}
8012
8013// Vectors with sizes that are not powers of 2 need to be widened to the
8014// next largest power of 2. For example, we may get a vector of 3 32-bit
8015// integers or of 6 16-bit integers, both of which have to be widened to a
8016// 128-bit vector.
8017SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) {
8018 SDValue WidenedOp = GetWidenedVector(Op: N->getOperand(Num: 1));
8019 return DAG.getNode(Opcode: ISD::FAKE_USE, DL: SDLoc(), VT: MVT::Other, N1: N->getOperand(Num: 0),
8020 N2: WidenedOp);
8021}
8022
8023SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
8024 EVT VT = N->getValueType(ResNo: 0);
8025 EVT EltVT = VT.getVectorElementType();
8026 EVT InVT = N->getOperand(Num: 0).getValueType();
8027 SDLoc dl(N);
8028
8029 // If the widen width for this operand is the same as the width of the concat
8030 // and all but the first operand is undef, just use the widened operand.
8031 unsigned NumOperands = N->getNumOperands();
8032 if (VT == TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: InVT)) {
8033 unsigned i;
8034 for (i = 1; i < NumOperands; ++i)
8035 if (!N->getOperand(Num: i).isUndef())
8036 break;
8037
8038 if (i == NumOperands)
8039 return GetWidenedVector(Op: N->getOperand(Num: 0));
8040 }
8041
8042 // Otherwise, fall back to a nasty build vector.
8043 unsigned NumElts = VT.getVectorNumElements();
8044 SmallVector<SDValue, 16> Ops(NumElts);
8045
8046 unsigned NumInElts = InVT.getVectorNumElements();
8047
8048 unsigned Idx = 0;
8049 for (unsigned i=0; i < NumOperands; ++i) {
8050 SDValue InOp = N->getOperand(Num: i);
8051 assert(getTypeAction(InOp.getValueType()) ==
8052 TargetLowering::TypeWidenVector &&
8053 "Unexpected type action");
8054 InOp = GetWidenedVector(Op: InOp);
8055 for (unsigned j = 0; j < NumInElts; ++j)
8056 Ops[Idx++] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx: j);
8057 }
8058 return DAG.getBuildVector(VT, DL: dl, Ops);
8059}
8060
8061SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
8062 EVT VT = N->getValueType(ResNo: 0);
8063 SDValue SubVec = N->getOperand(Num: 1);
8064 SDValue InVec = N->getOperand(Num: 0);
8065
8066 EVT OrigVT = SubVec.getValueType();
8067 SubVec = GetWidenedVector(Op: SubVec);
8068 EVT SubVT = SubVec.getValueType();
8069
8070 // Whether or not all the elements of the widened SubVec will be inserted into
8071 // valid indices of VT.
8072 bool IndicesValid = false;
8073 // If we statically know that VT can fit SubVT, the indices are valid.
8074 if (VT.knownBitsGE(VT: SubVT))
8075 IndicesValid = true;
8076 else if (VT.isScalableVector() && SubVT.isFixedLengthVector()) {
8077 // Otherwise, if we're inserting a fixed vector into a scalable vector and
8078 // we know the minimum vscale we can work out if it's valid ourselves.
8079 Attribute Attr = DAG.getMachineFunction().getFunction().getFnAttribute(
8080 Kind: Attribute::VScaleRange);
8081 if (Attr.isValid()) {
8082 unsigned VScaleMin = Attr.getVScaleRangeMin();
8083 if (VT.getSizeInBits().getKnownMinValue() * VScaleMin >=
8084 SubVT.getFixedSizeInBits())
8085 IndicesValid = true;
8086 }
8087 }
8088
8089 if (!IndicesValid)
8090 report_fatal_error(
8091 reason: "Don't know how to widen the operands for INSERT_SUBVECTOR");
8092
8093 SDLoc DL(N);
8094
8095 // We need to make sure that the indices are still valid, otherwise we might
8096 // widen what was previously well-defined to something undefined.
8097 if (InVec.isUndef() && N->getConstantOperandVal(Num: 2) == 0)
8098 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: InVec, N2: SubVec,
8099 N3: N->getOperand(Num: 2));
8100
8101 if (OrigVT.isScalableVector()) {
8102 // When the widened types match, overwriting the start of a vector is
8103 // effectively a merge operation that can be implement as a vselect.
8104 if (SubVT == VT && N->getConstantOperandVal(Num: 2) == 0) {
8105 SDValue Mask =
8106 DAG.getMaskFromElementCount(DL, VT, Len: OrigVT.getVectorElementCount());
8107 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Mask, N2: SubVec, N3: InVec);
8108 }
8109
8110 // Fallback to inserting through memory.
8111 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8112 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment);
8113 MachineFunction &MF = DAG.getMachineFunction();
8114 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
8115 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
8116
8117 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
8118 PtrInfo, F: MachineMemOperand::MOStore,
8119 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8120 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
8121 PtrInfo, F: MachineMemOperand::MOLoad,
8122 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: Alignment);
8123
8124 // Write out the vector being inserting into.
8125 SDValue Ch =
8126 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr, MMO: StoreMMO);
8127
8128 // Build a mask to match the length of the sub-vector.
8129 SDValue Mask =
8130 DAG.getMaskFromElementCount(DL, VT: SubVT, Len: OrigVT.getVectorElementCount());
8131
8132 // Overwrite the sub-vector at the required offset.
8133 SDValue SubVecPtr =
8134 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT: VT, SubVecVT: OrigVT, Index: N->getOperand(Num: 2));
8135 Ch = DAG.getMaskedStore(Chain: Ch, dl: DL, Val: SubVec, Base: SubVecPtr,
8136 Offset: DAG.getPOISON(VT: SubVecPtr.getValueType()), Mask, MemVT: VT,
8137 MMO: StoreMMO, AM: ISD::UNINDEXED, IsTruncating: ISD::NON_EXTLOAD);
8138
8139 // Read back the result.
8140 return DAG.getLoad(VT, dl: DL, Chain: Ch, Ptr: StackPtr, MMO: LoadMMO);
8141 }
8142
8143 // If the operands can't be widened legally, just replace the INSERT_SUBVECTOR
8144 // with a series of INSERT_VECTOR_ELT
8145 unsigned Idx = N->getConstantOperandVal(Num: 2);
8146
8147 SDValue InsertElt = InVec;
8148 for (unsigned I = 0, E = OrigVT.getVectorNumElements(); I != E; ++I) {
8149 SDValue ExtractElt =
8150 DAG.getExtractVectorElt(DL, VT: VT.getVectorElementType(), Vec: SubVec, Idx: I);
8151 InsertElt = DAG.getInsertVectorElt(DL, Vec: InsertElt, Elt: ExtractElt, Idx: I + Idx);
8152 }
8153
8154 return InsertElt;
8155}
8156
8157SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
8158 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8159 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
8160 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8161}
8162
8163SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
8164 SDValue InOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8165 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
8166 VT: N->getValueType(ResNo: 0), N1: InOp, N2: N->getOperand(Num: 1));
8167}
8168
8169SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
8170 SDLoc DL(N);
8171 EVT ResVT = N->getValueType(ResNo: 0);
8172
8173 // Widen the input as requested by the legalizer.
8174 SDValue WideInOp = GetWidenedVector(Op: N->getOperand(Num: 0));
8175 EVT WideInVT = WideInOp.getValueType();
8176
8177 // Simple case: if widened input is still smaller than or equal to result,
8178 // just use it directly.
8179 if (WideInVT.getSizeInBits() <= ResVT.getSizeInBits())
8180 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: WideInOp);
8181
8182 // EXTEND_VECTOR_INREG requires input bits <= result bits.
8183 // If widening makes the input larger than the original result, widen the
8184 // result to match, then extract back down.
8185 EVT ResEltVT = ResVT.getVectorElementType();
8186 unsigned EltBits = ResEltVT.getSizeInBits();
8187 assert((WideInVT.getSizeInBits() % EltBits) == 0 &&
8188 "Widened input size must be a multiple of result element size");
8189
8190 unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
8191 EVT WideResVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResEltVT, NumElements: WideNumElts);
8192
8193 SDValue WideRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: WideResVT, Operand: WideInOp);
8194 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideRes, Idx: 0);
8195}
8196
8197SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
8198 // We have to widen the value, but we want only to store the original
8199 // vector type.
8200 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
8201
8202 if (!ST->getMemoryVT().getScalarType().isByteSized())
8203 return TLI.scalarizeVectorStore(ST, DAG);
8204
8205 if (ST->isTruncatingStore())
8206 return TLI.scalarizeVectorStore(ST, DAG);
8207
8208 // Generate a vector-predicated store if it is custom/legal on the target.
8209 // To avoid possible recursion, only do this if the widened mask type is
8210 // legal.
8211 // FIXME: Not all targets may support EVL in VP_STORE. These will have been
8212 // removed from the IR by the ExpandVectorPredication pass but we're
8213 // reintroducing them here.
8214 SDValue StVal = ST->getValue();
8215 EVT StVT = StVal.getValueType();
8216 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT);
8217 EVT WideMaskVT = getSetCCResultType(VT: WideVT);
8218
8219 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8220 TLI.isTypeLegal(VT: WideMaskVT)) {
8221 // Widen the value.
8222 SDLoc DL(N);
8223 StVal = GetWidenedVector(Op: StVal);
8224 SDValue Mask = DAG.getAllOnesConstant(DL, VT: WideMaskVT);
8225 SDValue EVL = DAG.getElementCount(DL, VT: TLI.getVPExplicitVectorLengthTy(),
8226 EC: StVT.getVectorElementCount());
8227 return DAG.getStoreVP(Chain: ST->getChain(), dl: DL, Val: StVal, Ptr: ST->getBasePtr(),
8228 Offset: ST->getOffset(), Mask, EVL, MemVT: StVT, MMO: ST->getMemOperand(),
8229 AM: ST->getAddressingMode());
8230 }
8231
8232 SmallVector<SDValue, 16> StChain;
8233 if (GenWidenVectorStores(StChain, ST)) {
8234 if (StChain.size() == 1)
8235 return StChain[0];
8236
8237 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(ST), VT: MVT::Other, Ops: StChain);
8238 }
8239
8240 if (StVT.isVector()) {
8241 // If all else fails replace the store with a wide masked store.
8242 SDLoc DL(N);
8243 SDValue WideStVal = GetWidenedVector(Op: StVal);
8244 SDValue Mask =
8245 DAG.getMaskFromElementCount(DL, VT: WideVT, Len: StVT.getVectorElementCount());
8246
8247 return DAG.getMaskedStore(Chain: ST->getChain(), dl: DL, Val: WideStVal, Base: ST->getBasePtr(),
8248 Offset: ST->getOffset(), Mask, MemVT: ST->getMemoryVT(),
8249 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
8250 IsTruncating: ST->isTruncatingStore());
8251 }
8252
8253 report_fatal_error(reason: "Unable to widen vector store");
8254}
8255
8256SDValue DAGTypeLegalizer::WidenVecOp_ATOMIC_STORE(AtomicSDNode *ST) {
8257 EVT StVT = ST->getMemoryVT();
8258 SDLoc dl(ST);
8259
8260 SDValue StVal = GetWidenedVector(Op: ST->getVal());
8261 EVT WidenVT = StVal.getValueType();
8262
8263 TypeSize StWidth = StVT.getSizeInBits();
8264 TypeSize WidenWidth = WidenVT.getSizeInBits();
8265 TypeSize WidthDiff = WidenWidth - StWidth;
8266
8267 // Find the vector type that can store the original memory width in one
8268 // atomic operation. Pass StAlign=0 (like atomic loads); a real align would
8269 // let findMemType widen the access past the value (e.g. <2 x i8> at align 4
8270 // implies a 4-byte movl, writing undef bytes past its object).
8271 std::optional<EVT> FirstVT =
8272 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT, /*StAlign=*/Align: 0,
8273 WidenEx: WidthDiff.getKnownMinValue());
8274 if (!FirstVT)
8275 return SDValue();
8276
8277 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8278
8279 SDValue StOp =
8280 coerceStoredValue(StVal, FirstVT: *FirstVT, WidenVT, FirstVTWidth, dl, DAG);
8281
8282 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: *FirstVT, Chain: ST->getChain(), Ptr: StOp,
8283 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
8284}
8285
8286SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
8287 assert((OpNo == 1 || OpNo == 3) &&
8288 "Can widen only data or mask operand of vp_store");
8289 VPStoreSDNode *ST = cast<VPStoreSDNode>(Val: N);
8290 SDValue Mask = ST->getMask();
8291 SDValue StVal = ST->getValue();
8292 SDLoc dl(N);
8293
8294 if (OpNo == 1) {
8295 // Widen the value.
8296 StVal = GetWidenedVector(Op: StVal);
8297
8298 // We only handle the case where the mask needs widening to an
8299 // identically-sized type as the vector inputs.
8300 assert(getTypeAction(Mask.getValueType()) ==
8301 TargetLowering::TypeWidenVector &&
8302 "Unable to widen VP store");
8303 Mask = GetWidenedVector(Op: Mask);
8304 } else {
8305 Mask = GetWidenedVector(Op: Mask);
8306
8307 // We only handle the case where the stored value needs widening to an
8308 // identically-sized type as the mask.
8309 assert(getTypeAction(StVal.getValueType()) ==
8310 TargetLowering::TypeWidenVector &&
8311 "Unable to widen VP store");
8312 StVal = GetWidenedVector(Op: StVal);
8313 }
8314
8315 assert(Mask.getValueType().getVectorElementCount() ==
8316 StVal.getValueType().getVectorElementCount() &&
8317 "Mask and data vectors should have the same number of elements");
8318 return DAG.getStoreVP(Chain: ST->getChain(), dl, Val: StVal, Ptr: ST->getBasePtr(),
8319 Offset: ST->getOffset(), Mask, EVL: ST->getVectorLength(),
8320 MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
8321 AM: ST->getAddressingMode(), IsTruncating: ST->isTruncatingStore(),
8322 IsCompressing: ST->isCompressingStore());
8323}
8324
8325SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N,
8326 unsigned OpNo) {
8327 assert((OpNo == 1 || OpNo == 4) &&
8328 "Can widen only data or mask operand of vp_strided_store");
8329 VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
8330 SDValue Mask = SST->getMask();
8331 SDValue StVal = SST->getValue();
8332 SDLoc DL(N);
8333
8334 if (OpNo == 1)
8335 assert(getTypeAction(Mask.getValueType()) ==
8336 TargetLowering::TypeWidenVector &&
8337 "Unable to widen VP strided store");
8338 else
8339 assert(getTypeAction(StVal.getValueType()) ==
8340 TargetLowering::TypeWidenVector &&
8341 "Unable to widen VP strided store");
8342
8343 StVal = GetWidenedVector(Op: StVal);
8344 Mask = GetWidenedVector(Op: Mask);
8345
8346 assert(StVal.getValueType().getVectorElementCount() ==
8347 Mask.getValueType().getVectorElementCount() &&
8348 "Data and mask vectors should have the same number of elements");
8349
8350 return DAG.getStridedStoreVP(
8351 Chain: SST->getChain(), DL, Val: StVal, Ptr: SST->getBasePtr(), Offset: SST->getOffset(),
8352 Stride: SST->getStride(), Mask, EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
8353 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(), IsTruncating: SST->isTruncatingStore(),
8354 IsCompressing: SST->isCompressingStore());
8355}
8356
8357SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
8358 assert((OpNo == 1 || OpNo == 4) &&
8359 "Can widen only data or mask operand of mstore");
8360 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
8361 SDValue Mask = MST->getMask();
8362 EVT MaskVT = Mask.getValueType();
8363 SDValue StVal = MST->getValue();
8364 EVT VT = StVal.getValueType();
8365 SDLoc dl(N);
8366
8367 EVT WideVT, WideMaskVT;
8368 if (OpNo == 1) {
8369 // Widen the value.
8370 StVal = GetWidenedVector(Op: StVal);
8371
8372 WideVT = StVal.getValueType();
8373 WideMaskVT =
8374 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MaskVT.getVectorElementType(),
8375 EC: WideVT.getVectorElementCount());
8376 } else {
8377 WideMaskVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MaskVT);
8378
8379 EVT ValueVT = StVal.getValueType();
8380 WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ValueVT.getVectorElementType(),
8381 EC: WideMaskVT.getVectorElementCount());
8382 }
8383
8384 if (TLI.isOperationLegalOrCustom(Op: ISD::VP_STORE, VT: WideVT) &&
8385 TLI.isTypeLegal(VT: WideMaskVT) && !MST->isCompressingStore()) {
8386 Mask = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideMaskVT), SubVec: Mask, Idx: 0);
8387 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8388 EC: VT.getVectorElementCount());
8389 return DAG.getStoreVP(Chain: MST->getChain(), dl, Val: StVal, Ptr: MST->getBasePtr(),
8390 Offset: MST->getOffset(), Mask, EVL, MemVT: MST->getMemoryVT(),
8391 MMO: MST->getMemOperand(), AM: MST->getAddressingMode());
8392 }
8393
8394 if (OpNo == 1) {
8395 // The mask should be widened as well.
8396 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8397 } else {
8398 // Widen the mask.
8399 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8400
8401 StVal = ModifyToType(InOp: StVal, NVT: WideVT);
8402 }
8403
8404 assert(Mask.getValueType().getVectorElementCount() ==
8405 StVal.getValueType().getVectorElementCount() &&
8406 "Mask and data vectors should have the same number of elements");
8407 return DAG.getMaskedStore(Chain: MST->getChain(), dl, Val: StVal, Base: MST->getBasePtr(),
8408 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
8409 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
8410 IsTruncating: false, IsCompressing: MST->isCompressingStore());
8411}
8412
8413SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
8414 assert(OpNo == 4 && "Can widen only the index of mgather");
8415 auto *MG = cast<MaskedGatherSDNode>(Val: N);
8416 SDValue DataOp = MG->getPassThru();
8417 SDValue Mask = MG->getMask();
8418 SDValue Scale = MG->getScale();
8419
8420 // Just widen the index. It's allowed to have extra elements.
8421 SDValue Index = GetWidenedVector(Op: MG->getIndex());
8422
8423 SDLoc dl(N);
8424 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
8425 Scale};
8426 SDValue Res = DAG.getMaskedGather(VTs: MG->getVTList(), MemVT: MG->getMemoryVT(), dl, Ops,
8427 MMO: MG->getMemOperand(), IndexType: MG->getIndexType(),
8428 ExtTy: MG->getExtensionType());
8429 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
8430 ReplaceValueWith(From: SDValue(N, 0), To: Res.getValue(R: 0));
8431 return SDValue();
8432}
8433
8434SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
8435 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
8436 SDValue DataOp = MSC->getValue();
8437 SDValue Mask = MSC->getMask();
8438 SDValue Index = MSC->getIndex();
8439 SDValue Scale = MSC->getScale();
8440 EVT WideMemVT = MSC->getMemoryVT();
8441
8442 if (OpNo == 1) {
8443 DataOp = GetWidenedVector(Op: DataOp);
8444 ElementCount WideEC = DataOp.getValueType().getVectorElementCount();
8445
8446 // Widen index.
8447 EVT IndexVT = Index.getValueType();
8448 EVT WideIndexVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8449 VT: IndexVT.getVectorElementType(), EC: WideEC);
8450 Index = ModifyToType(InOp: Index, NVT: WideIndexVT);
8451
8452 // The mask should be widened as well.
8453 EVT MaskVT = Mask.getValueType();
8454 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8455 VT: MaskVT.getVectorElementType(), EC: WideEC);
8456 Mask = ModifyToType(InOp: Mask, NVT: WideMaskVT, FillWithZeroes: true);
8457
8458 // Widen the MemoryType
8459 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8460 VT: MSC->getMemoryVT().getScalarType(), EC: WideEC);
8461 } else if (OpNo == 4) {
8462 // Just widen the index. It's allowed to have extra elements.
8463 Index = GetWidenedVector(Op: Index);
8464 } else
8465 llvm_unreachable("Can't widen this operand of mscatter");
8466
8467 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
8468 Scale};
8469 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: WideMemVT, dl: SDLoc(N),
8470 Ops, MMO: MSC->getMemOperand(), IndexType: MSC->getIndexType(),
8471 IsTruncating: MSC->isTruncatingStore());
8472}
8473
8474SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) {
8475 VPScatterSDNode *VPSC = cast<VPScatterSDNode>(Val: N);
8476 SDValue DataOp = VPSC->getValue();
8477 SDValue Mask = VPSC->getMask();
8478 SDValue Index = VPSC->getIndex();
8479 SDValue Scale = VPSC->getScale();
8480 EVT WideMemVT = VPSC->getMemoryVT();
8481
8482 if (OpNo == 1) {
8483 DataOp = GetWidenedVector(Op: DataOp);
8484 Index = GetWidenedVector(Op: Index);
8485 const auto WideEC = DataOp.getValueType().getVectorElementCount();
8486 Mask = GetWidenedMask(Mask, EC: WideEC);
8487 WideMemVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8488 VT: VPSC->getMemoryVT().getScalarType(), EC: WideEC);
8489 } else if (OpNo == 3) {
8490 // Just widen the index. It's allowed to have extra elements.
8491 Index = GetWidenedVector(Op: Index);
8492 } else
8493 llvm_unreachable("Can't widen this operand of VP_SCATTER");
8494
8495 SDValue Ops[] = {
8496 VPSC->getChain(), DataOp, VPSC->getBasePtr(), Index, Scale, Mask,
8497 VPSC->getVectorLength()};
8498 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: WideMemVT, dl: SDLoc(N), Ops,
8499 MMO: VPSC->getMemOperand(), IndexType: VPSC->getIndexType());
8500}
8501
8502SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
8503 SDValue InOp0 = GetWidenedVector(Op: N->getOperand(Num: 0));
8504 SDValue InOp1 = GetWidenedVector(Op: N->getOperand(Num: 1));
8505 SDLoc dl(N);
8506 EVT VT = N->getValueType(ResNo: 0);
8507
8508 // WARNING: In this code we widen the compare instruction with garbage.
8509 // This garbage may contain denormal floats which may be slow. Is this a real
8510 // concern ? Should we zero the unused lanes if this is a float compare ?
8511
8512 // Get a new SETCC node to compare the newly widened operands.
8513 // Only some of the compared elements are legal.
8514 EVT SVT = getSetCCResultType(VT: InOp0.getValueType());
8515 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
8516 if (VT.getScalarType() == MVT::i1)
8517 SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8518 EC: SVT.getVectorElementCount());
8519
8520 SDValue WideSETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N),
8521 VT: SVT, N1: InOp0, N2: InOp1, N3: N->getOperand(Num: 2));
8522
8523 // Extract the needed results from the result vector.
8524 EVT ResVT = EVT::getVectorVT(Context&: *DAG.getContext(),
8525 VT: SVT.getVectorElementType(),
8526 EC: VT.getVectorElementCount());
8527 SDValue CC = DAG.getExtractSubvector(DL: dl, VT: ResVT, Vec: WideSETCC, Idx: 0);
8528
8529 EVT OpVT = N->getOperand(Num: 0).getValueType();
8530 ISD::NodeType ExtendCode =
8531 TargetLowering::getExtendForContent(Content: TLI.getBooleanContents(Type: OpVT));
8532 return DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: CC);
8533}
8534
8535SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
8536 SDValue Chain = N->getOperand(Num: 0);
8537 SDValue LHS = GetWidenedVector(Op: N->getOperand(Num: 1));
8538 SDValue RHS = GetWidenedVector(Op: N->getOperand(Num: 2));
8539 SDValue CC = N->getOperand(Num: 3);
8540 SDLoc dl(N);
8541
8542 EVT VT = N->getValueType(ResNo: 0);
8543 EVT EltVT = VT.getVectorElementType();
8544 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
8545 unsigned NumElts = VT.getVectorNumElements();
8546
8547 // Unroll into a build vector.
8548 SmallVector<SDValue, 8> Scalars(NumElts);
8549 SmallVector<SDValue, 8> Chains(NumElts);
8550
8551 for (unsigned i = 0; i != NumElts; ++i) {
8552 SDValue LHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: LHS, Idx: i);
8553 SDValue RHSElem = DAG.getExtractVectorElt(DL: dl, VT: TmpEltVT, Vec: RHS, Idx: i);
8554
8555 Scalars[i] = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {MVT::i1, MVT::Other},
8556 Ops: {Chain, LHSElem, RHSElem, CC});
8557 Chains[i] = Scalars[i].getValue(R: 1);
8558 Scalars[i] = DAG.getSelect(DL: dl, VT: EltVT, Cond: Scalars[i],
8559 LHS: DAG.getBoolConstant(V: true, DL: dl, VT: EltVT, OpVT: VT),
8560 RHS: DAG.getBoolConstant(V: false, DL: dl, VT: EltVT, OpVT: VT));
8561 }
8562
8563 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
8564 ReplaceValueWith(From: SDValue(N, 1), To: NewChain);
8565
8566 return DAG.getBuildVector(VT, DL: dl, Ops: Scalars);
8567}
8568
8569static unsigned getExtendForIntVecReduction(unsigned Opc) {
8570 switch (Opc) {
8571 default:
8572 llvm_unreachable("Expected integer vector reduction");
8573 case ISD::VECREDUCE_ADD:
8574 case ISD::VECREDUCE_MUL:
8575 case ISD::VECREDUCE_AND:
8576 case ISD::VECREDUCE_OR:
8577 case ISD::VECREDUCE_XOR:
8578 return ISD::ANY_EXTEND;
8579 case ISD::VECREDUCE_SMAX:
8580 case ISD::VECREDUCE_SMIN:
8581 return ISD::SIGN_EXTEND;
8582 case ISD::VECREDUCE_UMAX:
8583 case ISD::VECREDUCE_UMIN:
8584 return ISD::ZERO_EXTEND;
8585 }
8586}
8587
8588SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
8589 SDLoc dl(N);
8590 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 0));
8591 EVT VT = N->getValueType(ResNo: 0);
8592 EVT OrigVT = N->getOperand(Num: 0).getValueType();
8593 EVT WideVT = Op.getValueType();
8594 EVT ElemVT = OrigVT.getVectorElementType();
8595 SDNodeFlags Flags = N->getFlags();
8596
8597 unsigned Opc = N->getOpcode();
8598 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8599 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8600 assert(NeutralElem && "Neutral element must exist");
8601
8602 // Pad the vector with the neutral element.
8603 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8604 unsigned WideElts = WideVT.getVectorMinNumElements();
8605
8606 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8607 // needing to pad the source vector, because the inactive lanes can simply be
8608 // disabled and not contribute to the result.
8609 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8610 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8611 SDValue Start = NeutralElem;
8612 if (VT.isInteger())
8613 Start = DAG.getNode(Opcode: getExtendForIntVecReduction(Opc), DL: dl, VT, Operand: Start);
8614 assert(Start.getValueType() == VT);
8615 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8616 EC: WideVT.getVectorElementCount());
8617 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8618 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8619 EC: OrigVT.getVectorElementCount());
8620 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {Start, Op, Mask, EVL}, Flags);
8621 }
8622
8623 if (WideVT.isScalableVector()) {
8624 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8625 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8626 EC: ElementCount::getScalable(MinVal: GCD));
8627 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8628 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8629 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8630 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8631 }
8632
8633 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8634 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8635
8636 return DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Op, Flags);
8637}
8638
8639SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
8640 SDLoc dl(N);
8641 SDValue AccOp = N->getOperand(Num: 0);
8642 SDValue VecOp = N->getOperand(Num: 1);
8643 SDValue Op = GetWidenedVector(Op: VecOp);
8644
8645 EVT VT = N->getValueType(ResNo: 0);
8646 EVT OrigVT = VecOp.getValueType();
8647 EVT WideVT = Op.getValueType();
8648 EVT ElemVT = OrigVT.getVectorElementType();
8649 SDNodeFlags Flags = N->getFlags();
8650
8651 unsigned Opc = N->getOpcode();
8652 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Opc);
8653 SDValue NeutralElem = DAG.getIdentityElement(Opcode: BaseOpc, DL: dl, VT: ElemVT, Flags);
8654
8655 // Pad the vector with the neutral element.
8656 unsigned OrigElts = OrigVT.getVectorMinNumElements();
8657 unsigned WideElts = WideVT.getVectorMinNumElements();
8658
8659 // Generate a vp.reduce_op if it is custom/legal for the target. This avoids
8660 // needing to pad the source vector, because the inactive lanes can simply be
8661 // disabled and not contribute to the result.
8662 if (auto VPOpcode = ISD::getVPForBaseOpcode(Opcode: Opc);
8663 VPOpcode && TLI.isOperationLegalOrCustom(Op: *VPOpcode, VT: WideVT)) {
8664 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
8665 EC: WideVT.getVectorElementCount());
8666 SDValue Mask = DAG.getAllOnesConstant(DL: dl, VT: WideMaskVT);
8667 SDValue EVL = DAG.getElementCount(DL: dl, VT: TLI.getVPExplicitVectorLengthTy(),
8668 EC: OrigVT.getVectorElementCount());
8669 return DAG.getNode(Opcode: *VPOpcode, DL: dl, VT, Ops: {AccOp, Op, Mask, EVL}, Flags);
8670 }
8671
8672 if (WideVT.isScalableVector()) {
8673 unsigned GCD = std::gcd(m: OrigElts, n: WideElts);
8674 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElemVT,
8675 EC: ElementCount::getScalable(MinVal: GCD));
8676 SDValue SplatNeutral = DAG.getSplatVector(VT: SplatVT, DL: dl, Op: NeutralElem);
8677 for (unsigned Idx = OrigElts; Idx < WideElts; Idx = Idx + GCD)
8678 Op = DAG.getInsertSubvector(DL: dl, Vec: Op, SubVec: SplatNeutral, Idx);
8679 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8680 }
8681
8682 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
8683 Op = DAG.getInsertVectorElt(DL: dl, Vec: Op, Elt: NeutralElem, Idx);
8684
8685 return DAG.getNode(Opcode: Opc, DL: dl, VT, N1: AccOp, N2: Op, Flags);
8686}
8687
8688SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) {
8689 assert(N->isVPOpcode() && "Expected VP opcode");
8690
8691 SDLoc dl(N);
8692 SDValue Op = GetWidenedVector(Op: N->getOperand(Num: 1));
8693 SDValue Mask = GetWidenedMask(Mask: N->getOperand(Num: 2),
8694 EC: Op.getValueType().getVectorElementCount());
8695
8696 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0),
8697 Ops: {N->getOperand(Num: 0), Op, Mask, N->getOperand(Num: 3)},
8698 Flags: N->getFlags());
8699}
8700
8701SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
8702 // This only gets called in the case that the left and right inputs and
8703 // result are of a legal odd vector type, and the condition is illegal i1 of
8704 // the same odd width that needs widening.
8705 EVT VT = N->getValueType(ResNo: 0);
8706 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
8707
8708 SDValue Cond = GetWidenedVector(Op: N->getOperand(Num: 0));
8709 SDValue LeftIn = DAG.WidenVector(N: N->getOperand(Num: 1), DL: SDLoc(N));
8710 SDValue RightIn = DAG.WidenVector(N: N->getOperand(Num: 2), DL: SDLoc(N));
8711 SDLoc DL(N);
8712
8713 SDValue Select = DAG.getNode(Opcode: N->getOpcode(), DL, VT: LeftIn.getValueType(), N1: Cond,
8714 N2: LeftIn, N3: RightIn);
8715 return DAG.getExtractSubvector(DL, VT, Vec: Select, Idx: 0);
8716}
8717
8718SDValue DAGTypeLegalizer::WidenVecOp_CttzElements(SDNode *N) {
8719 SDLoc DL(N);
8720 SDValue Source = N->getOperand(Num: 0);
8721 EVT SourceVT = Source.getValueType();
8722 EVT WideVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
8723
8724 SDValue WideSource;
8725 if (N->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON) {
8726 WideSource = GetWidenedVector(Op: Source);
8727 } else {
8728 // Pad the widened portion with all-ones so the extra lanes appear as
8729 // active (non-zero) elements and do not contribute trailing zeros.
8730 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: WideVT);
8731 if (WideVT.isFixedLengthVector() &&
8732 getTypeAction(VT: WideVT) == TargetLowering::TypeSplitVector) {
8733 WideSource = GetWidenedVector(Op: Source);
8734 unsigned WideElts = WideVT.getVectorNumElements();
8735 SmallVector<int> Mask(WideElts);
8736 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
8737 for (unsigned I = SourceVT.getVectorNumElements(); I != WideElts; ++I)
8738 Mask[I] += WideElts;
8739 WideSource = DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: WideSource, N2: AllOnes, Mask);
8740 } else {
8741 WideSource = DAG.getInsertSubvector(DL, Vec: AllOnes, SubVec: Source, Idx: 0);
8742 }
8743 }
8744
8745 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0), Operand: WideSource,
8746 Flags: N->getFlags());
8747}
8748
8749SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) {
8750 SDLoc DL(N);
8751 SDValue Source = GetWidenedVector(Op: N->getOperand(Num: 0));
8752 EVT SrcVT = Source.getValueType();
8753 SDValue Mask =
8754 GetWidenedMask(Mask: N->getOperand(Num: 1), EC: SrcVT.getVectorElementCount());
8755
8756 return DAG.getNode(Opcode: N->getOpcode(), DL, VT: N->getValueType(ResNo: 0),
8757 Ops: {Source, Mask, N->getOperand(Num: 2)}, Flags: N->getFlags());
8758}
8759
8760SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
8761 SDLoc DL(N);
8762 SDValue Mask = N->getOperand(Num: 0);
8763 EVT OrigMaskVT = Mask.getValueType();
8764 SDValue WideMask = GetWidenedVector(Op: Mask);
8765 EVT WideMaskVT = WideMask.getValueType();
8766
8767 // Pad the mask with zeros to ensure inactive lanes don't affect the result.
8768 unsigned OrigElts = OrigMaskVT.getVectorNumElements();
8769 unsigned WideElts = WideMaskVT.getVectorNumElements();
8770 if (OrigElts != WideElts) {
8771 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
8772 WideMask = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideMaskVT, N1: ZeroMask,
8773 N2: Mask, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8774 }
8775
8776 return DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: N->getValueType(ResNo: 0),
8777 Operand: WideMask);
8778}
8779
8780SDValue DAGTypeLegalizer::WidenVecOp_VECTOR_MATCH(SDNode *N, unsigned OpNo) {
8781 if (OpNo == 0) {
8782 SDLoc DL(N);
8783 EVT ResVT = N->getValueType(ResNo: 0);
8784 EVT SourceVT = N->getOperand(Num: 0).getValueType();
8785 EVT WideSourceVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SourceVT);
8786 EVT WidenVT =
8787 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT.getVectorElementType(),
8788 EC: WideSourceVT.getVectorElementCount());
8789
8790 SDValue WideSource = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: WideSourceVT),
8791 SubVec: N->getOperand(Num: 0), Idx: 0);
8792 SDValue WideMask = DAG.getInsertSubvector(
8793 DL, Vec: DAG.getConstant(Val: 0, DL, VT: WidenVT), SubVec: N->getOperand(Num: 2), Idx: 0);
8794 SDValue WideMatch = DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: WidenVT, N1: WideSource,
8795 N2: N->getOperand(Num: 1), N3: WideMask, Flags: N->getFlags());
8796 return DAG.getExtractSubvector(DL, VT: ResVT, Vec: WideMatch, Idx: 0);
8797 }
8798
8799 // Note: The Mask (OpNo == 2) should be widened with the result.
8800 assert(OpNo == 1 && "Unexpected VECTOR_MATCH operand");
8801
8802 SDLoc DL(N);
8803 SDValue Needle = N->getOperand(Num: 1);
8804 EVT NeedleVT = Needle.getValueType();
8805 if (NeedleVT.getVectorNumElements() == 1)
8806 return TLI.expandVectorMatch(N, DAG);
8807
8808 EVT WidenNeedleVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: NeedleVT);
8809
8810 SDValue Fill =
8811 DAG.getExtractVectorElt(DL, VT: NeedleVT.getVectorElementType(), Vec: Needle, Idx: 0);
8812 SDValue WideNeedle = DAG.getSplatVector(VT: WidenNeedleVT, DL, Op: Fill);
8813 WideNeedle = DAG.getInsertSubvector(DL, Vec: WideNeedle, SubVec: Needle, Idx: 0);
8814
8815 return DAG.getNode(Opcode: ISD::VECTOR_MATCH, DL, VT: N->getValueType(ResNo: 0),
8816 N1: N->getOperand(Num: 0), N2: WideNeedle, N3: N->getOperand(Num: 2),
8817 Flags: N->getFlags());
8818}
8819
8820//===----------------------------------------------------------------------===//
8821// Vector Widening Utilities
8822//===----------------------------------------------------------------------===//
8823
8824// Utility function to find the type to chop up a widen vector for load/store
8825// TLI: Target lowering used to determine legal types.
8826// Width: Width left need to load/store.
8827// WidenVT: The widen vector type to load to/store from
8828// Align: If 0, don't allow use of a wider type
8829// WidenEx: If Align is not 0, the amount additional we can load/store from.
8830
8831static std::optional<EVT> findMemType(SelectionDAG &DAG,
8832 const TargetLowering &TLI, unsigned Width,
8833 EVT WidenVT, unsigned Align = 0,
8834 unsigned WidenEx = 0) {
8835 EVT WidenEltVT = WidenVT.getVectorElementType();
8836 const bool Scalable = WidenVT.isScalableVector();
8837 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinValue();
8838 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
8839 unsigned AlignInBits = Align*8;
8840
8841 EVT RetVT = WidenEltVT;
8842 // Don't bother looking for an integer type if the vector is scalable, skip
8843 // to vector types.
8844 if (!Scalable) {
8845 // If we have one element to load/store, return it.
8846 if (Width == WidenEltWidth)
8847 return RetVT;
8848
8849 // See if there is larger legal integer than the element type to load/store.
8850 for (EVT MemVT : reverse(C: MVT::integer_valuetypes())) {
8851 unsigned MemVTWidth = MemVT.getSizeInBits();
8852 if (MemVT.getSizeInBits() <= WidenEltWidth)
8853 break;
8854 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8855 if ((Action == TargetLowering::TypeLegal ||
8856 Action == TargetLowering::TypePromoteInteger) &&
8857 (WidenWidth % MemVTWidth) == 0 &&
8858 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8859 (MemVTWidth <= Width ||
8860 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8861 if (MemVTWidth == WidenWidth)
8862 return MemVT;
8863 RetVT = MemVT;
8864 break;
8865 }
8866 }
8867 }
8868
8869 // See if there is a larger vector type to load/store that has the same vector
8870 // element type and is evenly divisible with the WidenVT.
8871 for (EVT MemVT : reverse(C: MVT::vector_valuetypes())) {
8872 // Skip vector MVTs which don't match the scalable property of WidenVT.
8873 if (Scalable != MemVT.isScalableVector())
8874 continue;
8875 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinValue();
8876 auto Action = TLI.getTypeAction(Context&: *DAG.getContext(), VT: MemVT);
8877 if ((Action == TargetLowering::TypeLegal ||
8878 Action == TargetLowering::TypePromoteInteger) &&
8879 WidenEltVT == MemVT.getVectorElementType() &&
8880 (WidenWidth % MemVTWidth) == 0 &&
8881 isPowerOf2_32(Value: WidenWidth / MemVTWidth) &&
8882 (MemVTWidth <= Width ||
8883 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
8884 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
8885 return MemVT;
8886 }
8887 }
8888
8889 // Using element-wise loads and stores for widening operations is not
8890 // supported for scalable vectors
8891 if (Scalable)
8892 return std::nullopt;
8893
8894 return RetVT;
8895}
8896
8897// Builds a vector type from scalar loads
8898// VecTy: Resulting Vector type
8899// LDOps: Load operators to build a vector type
8900// [Start,End) the list of loads to use.
8901static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
8902 SmallVectorImpl<SDValue> &LdOps,
8903 unsigned Start, unsigned End) {
8904 SDLoc dl(LdOps[Start]);
8905 EVT LdTy = LdOps[Start].getValueType();
8906 unsigned Width = VecTy.getSizeInBits();
8907 unsigned NumElts = Width / LdTy.getSizeInBits();
8908 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: LdTy, NumElements: NumElts);
8909
8910 unsigned Idx = 1;
8911 SDValue VecOp = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NewVecVT,Operand: LdOps[Start]);
8912
8913 for (unsigned i = Start + 1; i != End; ++i) {
8914 EVT NewLdTy = LdOps[i].getValueType();
8915 if (NewLdTy != LdTy) {
8916 NumElts = Width / NewLdTy.getSizeInBits();
8917 NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewLdTy, NumElements: NumElts);
8918 VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: VecOp);
8919 // Readjust position and vector position based on new load type.
8920 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
8921 LdTy = NewLdTy;
8922 }
8923 VecOp = DAG.getInsertVectorElt(DL: dl, Vec: VecOp, Elt: LdOps[i], Idx: Idx++);
8924 }
8925 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: VecOp);
8926}
8927
8928SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
8929 LoadSDNode *LD) {
8930 // The strategy assumes that we can efficiently load power-of-two widths.
8931 // The routine chops the vector into the largest vector loads with the same
8932 // element type or scalar loads and then recombines it to the widen vector
8933 // type.
8934 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
8935 EVT LdVT = LD->getMemoryVT();
8936 SDLoc dl(LD);
8937 assert(LdVT.isVector() && WidenVT.isVector());
8938 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
8939 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
8940
8941 // Load information
8942 SDValue Chain = LD->getChain();
8943 SDValue BasePtr = LD->getBasePtr();
8944 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
8945 AAMDNodes AAInfo = LD->getAAInfo();
8946
8947 TypeSize LdWidth = LdVT.getSizeInBits();
8948 TypeSize WidenWidth = WidenVT.getSizeInBits();
8949 TypeSize WidthDiff = WidenWidth - LdWidth;
8950 // Allow wider loads if they are sufficiently aligned to avoid memory faults
8951 // and if the original load is simple.
8952 unsigned LdAlign =
8953 (!LD->isSimple() || LdVT.isScalableVector()) ? 0 : LD->getAlign().value();
8954
8955 // Find the vector type that can load from.
8956 std::optional<EVT> FirstVT =
8957 findMemType(DAG, TLI, Width: LdWidth.getKnownMinValue(), WidenVT, Align: LdAlign,
8958 WidenEx: WidthDiff.getKnownMinValue());
8959
8960 if (!FirstVT)
8961 return SDValue();
8962
8963 SmallVector<EVT, 8> MemVTs;
8964 TypeSize FirstVTWidth = FirstVT->getSizeInBits();
8965
8966 // Unless we're able to load in one instruction we must work out how to load
8967 // the remainder.
8968 if (!TypeSize::isKnownLE(LHS: LdWidth, RHS: FirstVTWidth)) {
8969 std::optional<EVT> NewVT = FirstVT;
8970 TypeSize RemainingWidth = LdWidth;
8971 TypeSize NewVTWidth = FirstVTWidth;
8972 do {
8973 RemainingWidth -= NewVTWidth;
8974 if (TypeSize::isKnownLT(LHS: RemainingWidth, RHS: NewVTWidth)) {
8975 // The current type we are using is too large. Find a better size.
8976 NewVT = findMemType(DAG, TLI, Width: RemainingWidth.getKnownMinValue(),
8977 WidenVT, Align: LdAlign, WidenEx: WidthDiff.getKnownMinValue());
8978 if (!NewVT)
8979 return SDValue();
8980 NewVTWidth = NewVT->getSizeInBits();
8981 }
8982 MemVTs.push_back(Elt: *NewVT);
8983 } while (TypeSize::isKnownGT(LHS: RemainingWidth, RHS: NewVTWidth));
8984 }
8985
8986 SDValue LdOp = DAG.getLoad(VT: *FirstVT, dl, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
8987 Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
8988 LdChain.push_back(Elt: LdOp.getValue(R: 1));
8989
8990 // Check if we can load the element with one instruction.
8991 if (MemVTs.empty())
8992 return coerceLoadedValue(LdOp, FirstVT: *FirstVT, WidenVT, LdWidth, FirstVTWidth, dl,
8993 DAG);
8994
8995 // Load vector by using multiple loads from largest vector to scalar.
8996 SmallVector<SDValue, 16> LdOps;
8997 LdOps.push_back(Elt: LdOp);
8998
8999 uint64_t ScaledOffset = 0;
9000 MachinePointerInfo MPI = LD->getPointerInfo();
9001
9002 // First incremement past the first load.
9003 IncrementPointer(N: cast<LoadSDNode>(Val&: LdOp), MemVT: *FirstVT, MPI, Ptr&: BasePtr,
9004 ScaledOffset: &ScaledOffset);
9005
9006 for (EVT MemVT : MemVTs) {
9007 Align NewAlign = ScaledOffset == 0
9008 ? LD->getBaseAlign()
9009 : commonAlignment(A: LD->getAlign(), Offset: ScaledOffset);
9010 SDValue L =
9011 DAG.getLoad(VT: MemVT, dl, Chain, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign, MMOFlags, Metadata: AAInfo);
9012
9013 LdOps.push_back(Elt: L);
9014 LdChain.push_back(Elt: L.getValue(R: 1));
9015 IncrementPointer(N: cast<LoadSDNode>(Val&: L), MemVT, MPI, Ptr&: BasePtr, ScaledOffset: &ScaledOffset);
9016 }
9017
9018 // Build the vector from the load operations.
9019 unsigned End = LdOps.size();
9020 if (!LdOps[0].getValueType().isVector())
9021 // All the loads are scalar loads.
9022 return BuildVectorFromScalar(DAG, VecTy: WidenVT, LdOps, Start: 0, End);
9023
9024 // If the load contains vectors, build the vector using concat vector.
9025 // All of the vectors used to load are power-of-2, and the scalar loads can be
9026 // combined to make a power-of-2 vector.
9027 SmallVector<SDValue, 16> ConcatOps(End);
9028 int i = End - 1;
9029 int Idx = End;
9030 EVT LdTy = LdOps[i].getValueType();
9031 // First, combine the scalar loads to a vector.
9032 if (!LdTy.isVector()) {
9033 for (--i; i >= 0; --i) {
9034 LdTy = LdOps[i].getValueType();
9035 if (LdTy.isVector())
9036 break;
9037 }
9038 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, VecTy: LdTy, LdOps, Start: i + 1, End);
9039 }
9040
9041 ConcatOps[--Idx] = LdOps[i];
9042 for (--i; i >= 0; --i) {
9043 EVT NewLdTy = LdOps[i].getValueType();
9044 if (NewLdTy != LdTy) {
9045 // Create a larger vector.
9046 TypeSize LdTySize = LdTy.getSizeInBits();
9047 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
9048 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
9049 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinValue()));
9050 unsigned NumOps =
9051 NewLdTySize.getKnownMinValue() / LdTySize.getKnownMinValue();
9052 SmallVector<SDValue, 16> WidenOps(NumOps);
9053 unsigned j = 0;
9054 for (; j != End-Idx; ++j)
9055 WidenOps[j] = ConcatOps[Idx+j];
9056 for (; j != NumOps; ++j)
9057 WidenOps[j] = DAG.getPOISON(VT: LdTy);
9058
9059 ConcatOps[End-1] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NewLdTy,
9060 Ops: WidenOps);
9061 Idx = End - 1;
9062 LdTy = NewLdTy;
9063 }
9064 ConcatOps[--Idx] = LdOps[i];
9065 }
9066
9067 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
9068 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT,
9069 Ops: ArrayRef(&ConcatOps[Idx], End - Idx));
9070
9071 // We need to fill the rest with undefs to build the vector.
9072 unsigned NumOps =
9073 WidenWidth.getKnownMinValue() / LdTy.getSizeInBits().getKnownMinValue();
9074 SmallVector<SDValue, 16> WidenOps(NumOps);
9075 SDValue UndefVal = DAG.getPOISON(VT: LdTy);
9076 {
9077 unsigned i = 0;
9078 for (; i != End-Idx; ++i)
9079 WidenOps[i] = ConcatOps[Idx+i];
9080 for (; i != NumOps; ++i)
9081 WidenOps[i] = UndefVal;
9082 }
9083 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WidenVT, Ops: WidenOps);
9084}
9085
9086SDValue
9087DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
9088 LoadSDNode *LD,
9089 ISD::LoadExtType ExtType) {
9090 // For extension loads, it may not be more efficient to chop up the vector
9091 // and then extend it. Instead, we unroll the load and build a new vector.
9092 EVT WidenVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),VT: LD->getValueType(ResNo: 0));
9093 EVT LdVT = LD->getMemoryVT();
9094 SDLoc dl(LD);
9095 assert(LdVT.isVector() && WidenVT.isVector());
9096 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
9097
9098 // Load information
9099 SDValue Chain = LD->getChain();
9100 SDValue BasePtr = LD->getBasePtr();
9101 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
9102 AAMDNodes AAInfo = LD->getAAInfo();
9103
9104 if (LdVT.isScalableVector())
9105 return SDValue();
9106
9107 EVT EltVT = WidenVT.getVectorElementType();
9108 EVT LdEltVT = LdVT.getVectorElementType();
9109 unsigned NumElts = LdVT.getVectorNumElements();
9110
9111 // Load each element and widen.
9112 unsigned WidenNumElts = WidenVT.getVectorNumElements();
9113 SmallVector<SDValue, 16> Ops(WidenNumElts);
9114 unsigned Increment = LdEltVT.getSizeInBits() / 8;
9115 Ops[0] =
9116 DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: BasePtr, PtrInfo: LD->getPointerInfo(),
9117 MemVT: LdEltVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
9118 LdChain.push_back(Elt: Ops[0].getValue(R: 1));
9119 unsigned i = 0, Offset = Increment;
9120 for (i=1; i < NumElts; ++i, Offset += Increment) {
9121 SDValue NewBasePtr =
9122 DAG.getObjectPtrOffset(SL: dl, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
9123 Ops[i] = DAG.getExtLoad(ExtType, dl, VT: EltVT, Chain, Ptr: NewBasePtr,
9124 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT: LdEltVT,
9125 Alignment: LD->getBaseAlign(), MMOFlags, Metadata: AAInfo);
9126 LdChain.push_back(Elt: Ops[i].getValue(R: 1));
9127 }
9128
9129 // Fill the rest with undefs.
9130 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9131 for (; i != WidenNumElts; ++i)
9132 Ops[i] = UndefVal;
9133
9134 return DAG.getBuildVector(VT: WidenVT, DL: dl, Ops);
9135}
9136
9137bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
9138 StoreSDNode *ST) {
9139 // The strategy assumes that we can efficiently store power-of-two widths.
9140 // The routine chops the vector into the largest vector stores with the same
9141 // element type or scalar stores.
9142 SDValue Chain = ST->getChain();
9143 SDValue BasePtr = ST->getBasePtr();
9144 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
9145 AAMDNodes AAInfo = ST->getAAInfo();
9146 SDValue ValOp = GetWidenedVector(Op: ST->getValue());
9147 SDLoc dl(ST);
9148
9149 EVT StVT = ST->getMemoryVT();
9150 TypeSize StWidth = StVT.getSizeInBits();
9151 EVT ValVT = ValOp.getValueType();
9152 TypeSize ValWidth = ValVT.getSizeInBits();
9153 EVT ValEltVT = ValVT.getVectorElementType();
9154 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
9155 assert(StVT.getVectorElementType() == ValEltVT);
9156 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
9157 "Mismatch between store and value types");
9158
9159 int Idx = 0; // current index to store
9160
9161 MachinePointerInfo MPI = ST->getPointerInfo();
9162 uint64_t ScaledOffset = 0;
9163
9164 // A breakdown of how to widen this vector store. Each element of the vector
9165 // is a memory VT combined with the number of times it is to be stored to,
9166 // e,g., v5i32 -> {{v2i32,2},{i32,1}}
9167 SmallVector<std::pair<EVT, unsigned>, 4> MemVTs;
9168
9169 while (StWidth.isNonZero()) {
9170 // Find the largest vector type we can store with.
9171 std::optional<EVT> NewVT =
9172 findMemType(DAG, TLI, Width: StWidth.getKnownMinValue(), WidenVT: ValVT);
9173 if (!NewVT)
9174 return false;
9175 MemVTs.push_back(Elt: {*NewVT, 0});
9176 TypeSize NewVTWidth = NewVT->getSizeInBits();
9177
9178 do {
9179 StWidth -= NewVTWidth;
9180 MemVTs.back().second++;
9181 } while (StWidth.isNonZero() && TypeSize::isKnownGE(LHS: StWidth, RHS: NewVTWidth));
9182 }
9183
9184 for (const auto &Pair : MemVTs) {
9185 EVT NewVT = Pair.first;
9186 unsigned Count = Pair.second;
9187 TypeSize NewVTWidth = NewVT.getSizeInBits();
9188
9189 if (NewVT.isVector()) {
9190 unsigned NumVTElts = NewVT.getVectorMinNumElements();
9191 do {
9192 Align NewAlign = ScaledOffset == 0
9193 ? ST->getBaseAlign()
9194 : commonAlignment(A: ST->getAlign(), Offset: ScaledOffset);
9195 SDValue EOp = DAG.getExtractSubvector(DL: dl, VT: NewVT, Vec: ValOp, Idx);
9196 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI, Alignment: NewAlign,
9197 MMOFlags, Metadata: AAInfo);
9198 StChain.push_back(Elt: PartStore);
9199
9200 Idx += NumVTElts;
9201 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr,
9202 ScaledOffset: &ScaledOffset);
9203 } while (--Count);
9204 } else {
9205 // Cast the vector to the scalar type we can store.
9206 unsigned NumElts = ValWidth.getFixedValue() / NewVTWidth.getFixedValue();
9207 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewVT, NumElements: NumElts);
9208 SDValue VecOp = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVecVT, Operand: ValOp);
9209 // Readjust index position based on new vector type.
9210 Idx = Idx * ValEltWidth / NewVTWidth.getFixedValue();
9211 do {
9212 SDValue EOp = DAG.getExtractVectorElt(DL: dl, VT: NewVT, Vec: VecOp, Idx: Idx++);
9213 SDValue PartStore = DAG.getStore(Chain, dl, Val: EOp, Ptr: BasePtr, PtrInfo: MPI,
9214 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
9215 StChain.push_back(Elt: PartStore);
9216
9217 IncrementPointer(N: cast<StoreSDNode>(Val&: PartStore), MemVT: NewVT, MPI, Ptr&: BasePtr);
9218 } while (--Count);
9219 // Restore index back to be relative to the original widen element type.
9220 Idx = Idx * NewVTWidth.getFixedValue() / ValEltWidth;
9221 }
9222 }
9223
9224 return true;
9225}
9226
9227/// Modifies a vector input (widen or narrows) to a vector of NVT. The
9228/// input vector must have the same element type as NVT.
9229/// FillWithZeroes specifies that the vector should be widened with zeroes.
9230SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
9231 bool FillWithZeroes) {
9232 // Note that InOp might have been widened so it might already have
9233 // the right width or it might need be narrowed.
9234 EVT InVT = InOp.getValueType();
9235 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
9236 "input and widen element type must match");
9237 assert(InVT.isScalableVector() == NVT.isScalableVector() &&
9238 "cannot modify scalable vectors in this way");
9239 SDLoc dl(InOp);
9240
9241 // Check if InOp already has the right width.
9242 if (InVT == NVT)
9243 return InOp;
9244
9245 ElementCount InEC = InVT.getVectorElementCount();
9246 ElementCount WidenEC = NVT.getVectorElementCount();
9247 if (WidenEC.hasKnownScalarFactor(RHS: InEC)) {
9248 unsigned NumConcat = WidenEC.getKnownScalarFactor(RHS: InEC);
9249 SmallVector<SDValue, 16> Ops(NumConcat);
9250 SDValue FillVal =
9251 FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: InVT) : DAG.getPOISON(VT: InVT);
9252 Ops[0] = InOp;
9253 for (unsigned i = 1; i != NumConcat; ++i)
9254 Ops[i] = FillVal;
9255
9256 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9257 }
9258
9259 if (InEC.hasKnownScalarFactor(RHS: WidenEC))
9260 return DAG.getExtractSubvector(DL: dl, VT: NVT, Vec: InOp, Idx: 0);
9261
9262 if (NVT.isScalableVector() && InVT.isScalableVector()) {
9263 // Split the input into the largest equal-sized scalable subvectors.
9264 unsigned InNumElts = InVT.getVectorMinNumElements();
9265 unsigned NewNumElts = NVT.getVectorMinNumElements();
9266 unsigned CommonFactor = std::gcd(m: InNumElts, n: NewNumElts);
9267 EVT PartVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT.getVectorElementType(),
9268 EC: ElementCount::getScalable(MinVal: CommonFactor));
9269
9270 SmallVector<SDValue, 16> Ops;
9271 unsigned NumCopiedParts = std::min(a: InNumElts, b: NewNumElts) / CommonFactor;
9272 for (unsigned I = 0; I != NumCopiedParts; ++I)
9273 Ops.push_back(
9274 Elt: DAG.getExtractSubvector(DL: dl, VT: PartVT, Vec: InOp, Idx: I * CommonFactor));
9275
9276 unsigned NumResultParts = NewNumElts / CommonFactor;
9277 if (NumResultParts > NumCopiedParts) {
9278 SDValue FillVal = FillWithZeroes ? DAG.getConstant(Val: 0, DL: dl, VT: PartVT)
9279 : DAG.getPOISON(VT: PartVT);
9280 Ops.append(NumInputs: NumResultParts - NumCopiedParts, Elt: FillVal);
9281 }
9282
9283 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: NVT, Ops);
9284 }
9285
9286 assert(!InVT.isScalableVector() && !NVT.isScalableVector() &&
9287 "Scalable vectors should have been handled already.");
9288
9289 unsigned InNumElts = InEC.getFixedValue();
9290 unsigned WidenNumElts = WidenEC.getFixedValue();
9291
9292 // Fall back to extract and build (+ mask, if padding with zeros).
9293 SmallVector<SDValue, 16> Ops(WidenNumElts);
9294 EVT EltVT = NVT.getVectorElementType();
9295 unsigned MinNumElts = std::min(a: WidenNumElts, b: InNumElts);
9296 unsigned Idx;
9297 for (Idx = 0; Idx < MinNumElts; ++Idx)
9298 Ops[Idx] = DAG.getExtractVectorElt(DL: dl, VT: EltVT, Vec: InOp, Idx);
9299
9300 SDValue UndefVal = DAG.getPOISON(VT: EltVT);
9301 for (; Idx < WidenNumElts; ++Idx)
9302 Ops[Idx] = UndefVal;
9303
9304 SDValue Widened = DAG.getBuildVector(VT: NVT, DL: dl, Ops);
9305 if (!FillWithZeroes)
9306 return Widened;
9307
9308 assert(NVT.isInteger() &&
9309 "We expect to never want to FillWithZeroes for non-integral types.");
9310
9311 SmallVector<SDValue, 16> MaskOps;
9312 MaskOps.append(NumInputs: MinNumElts, Elt: DAG.getAllOnesConstant(DL: dl, VT: EltVT));
9313 MaskOps.append(NumInputs: WidenNumElts - MinNumElts, Elt: DAG.getConstant(Val: 0, DL: dl, VT: EltVT));
9314
9315 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: NVT, N1: Widened,
9316 N2: DAG.getBuildVector(VT: NVT, DL: dl, Ops: MaskOps));
9317}
9318